argos-ruby 1.0.0

data/README.md

@@ -0,0 +1,35 @@
+# argos-ruby
+
+A Ruby library for parsing Argos tracking data
+
+```argos-ruby``` has been developed to parse [Argos](http://www.argos-system.org)
+satellite tracking data collected at the [Norwegian Polar Institute]
+(http://npolar.no/en) over a 25-year period (!) from 1989 to 2013.
+
+Be warned, the Argos file formats have changed over time. No promises are
+made that the library will work outside of Norway :).
+
+Currently, the library parses Argos DS/DIAG files dating from 1991
+and onwards.
+
+## Install
+Now
+$ git clone https://github.com/npolar/argos-ruby.git
+
+Soon
+$ gem install argos-ruby
+
+## Command-line usage
+```sh
+$  curl "https://raw.github.com/npolar/argos-ruby/master/spec/argos/_ds/990660_A.DAT" > /tmp/990660_A.DAT
+$  ./bin/argos-ruby /tmp/990660_A.DAT --filter "lambda {|a| a[:program] == 660 }"
+```
+This will output (the lambda filter is of course optional):
+
+```json
+[{"program":660,"platform":14747,"lines":2,"sensors":32,"satellite":"K","lc":null,"positioned":null,"latitude":null,"longitude":null,"altitude":null,"headers":5,"measured":"1999-12-16T00:46:49Z","identical":1,"sensor_data":["92","128","130","132"],"technology":"argos","type":"ds","filename":"/tmp/990660_A.DAT","source":"3a39e0bd0b944dca4f4fbf17bc0680704cde2994","warn":["missing-position","sensors-count-mismatch"],"parser":"argos-ruby-1.0.0","id":"f2c82a5ca1330b312925949a15ac300d07452a12"}]
+
+```
+Links
+* [http://api.npolar.no/tracking/?q=&filter-technology=argos](http://api.npolar.no/tracking/?q=&filter-technology=argos)
+* [Argos users manual v1.5](http://www.argos-system.org/files/pmedia/public/r363_9_argos_users_manual-v1.5.pdf) (PDF)

data/argos-ruby.gemspec

@@ -0,0 +1,21 @@
+# encoding: utf-8
+# https://github.com/radar/guides/blob/master/gem-development.md
+require File.expand_path(File.dirname(__FILE__)+"/lib/argos")
+
+Gem::Specification.new do |s|
+  s.name        = "argos-ruby"
+  s.version     = Argos::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Espen Egeland", "Conrad Helgeland"]
+  s.email       = ["data*npolar.no"]
+  s.homepage    = "http://github.com/npolar/argos-ruby"
+  s.summary     = %q{Argos satellite tracking data parsers}
+  s.description = %q{Parses Argos (http://www.argos-system.org/) DS/DAT and DIAG/DIA files.}
+  s.license = "GPL-3.0"
+  s.add_development_dependency "rspec"
+  s.files         = `git ls-files`.split("\n")
+  s.executables      = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/bin/argos-ruby

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# argos-ruby /path/argos/ --dest=/path/dest --level=info 2>> /path/argos-json.log 
+require_relative "../lib/argos"
+require "optparse"
+require "yaml"
+
+param = { format: "json",
+  action: "parse",
+  level: Logger::WARN,
+  dest: nil,
+  filter: nil,
+}
+
+optparser = OptionParser.new(ARGV) do |opts|
+  #opts.banner ""
+
+  opts.on_tail("--version", "-v", "Library version") do
+    puts Argos.library_version
+    exit
+  end
+
+  opts.on("--action={parse|source}", "-a", "Action") do |action|
+    param[:action] = action
+  end
+
+  opts.on("--dest=destination", "-a", "Destination directory") do |dest|
+    param[:dest] = dest
+  end
+
+  opts.on("--format=FORMAT", "-f", "Format") do |format|
+    param[:format] = format
+  end
+
+  opts.on("--filter=FILTER", "Filter lambda") do |filter|
+    param[:filter] = filter
+  end
+
+  opts.on("--level=LEVEL", "-l=LEVEL", "Log level: debug|info|warn|error|fatal") do |level|
+
+    param[:level] = case level
+      when /debug|0/i
+        Logger::DEBUG
+      when /info|1/i
+        Logger::INFO
+      when /warn|2/i
+        Logger::WARN
+      when /error|3/i
+        Logger::ERROR
+      when /fatal|4/i
+        Logger::FATAL
+      else
+        param[:level]
+    end
+  end
+
+end
+optparser.parse!
+glob = ARGV[0]||Dir.pwd
+
+begin
+
+log = Logger.new(STDERR)
+log.level = param[:level]
+
+
+if not Argos.argos? glob
+  glob = glob.gsub(/\/$/, "")
+  if glob != /\*/
+    glob += "/**/*"
+  end
+end
+
+log.debug "#{File.realpath(__FILE__)} #{param}"
+bundle = Digest::SHA1.hexdigest(glob+param[:filter].to_s)
+result = []
+
+Dir[glob].select {|f|
+  Argos.argos? f
+}.map {|filename|
+  
+  argos = Argos.factory(Argos.type(filename))
+  argos.filename = filename
+  argos.filter = param[:filter]
+  argos.log = log
+
+  case param[:action]
+    when "source"
+      result << Argos.source(argos).merge(glob: glob, bundle: bundle)
+    when "parse"
+      arr = argos.parse(filename)
+      unless param[:dest].nil?
+        jsonfile = "#{param[:dest].gsub(/\$/, "")}/#{argos.source}.json"
+        File.open(jsonfile, "w") { |file| file.write(arr.to_json) }
+      end
+      result += arr
+    when "messages"
+      argos.parse(filename)
+      result += argos.messages
+  end
+}
+
+ds_count = result.select {|argos| argos[:type] == "ds"}.size
+diag_count = result.select {|argos| argos[:type] == "diag"}.size
+# err count
+# warn count
+
+if ["parse", "messages"].include? param[:action]
+  log.info "Documents: #{result.size}, ds: #{ds_count}, diag: #{diag_count}, bundle: #{bundle}, glob: #{glob}"
+elsif "source" == param[:action]
+  result = result.select {|s| s[:size] > 0 }
+  sum_count = result.map {|s| s[:size]}.inject { |sum, c| sum + c }
+  result = result.map {|s|
+    s[:total] = sum_count
+    s
+  }
+  log.info "Documents (Σcount): #{sum_count}, sources: #{result.size} (ds: #{ds_count}, diag: #{diag_count}), bundle: #{bundle}, glob: #{glob}"
+end
+
+if param[:dest].nil?
+  case param[:format]
+    when "json"
+      puts result.to_json
+    when /y(a)?ml/ 
+      puts result.to_yaml
+    when /(ruby|rb)/
+      puts result
+  end
+end
+
+exit(0)
+
+rescue => e
+  Logger.new(STDERR).fatal e
+  exit(1)
+end

data/lib/argos/command.rb

@@ -0,0 +1,58 @@
+require "optparse"
+
+module Argos
+  class Command
+
+    attr_accessor :param
+
+    PARAM_DEFAULT = { format: "json",
+      action: "parse",
+      level: Logger::WARN,
+      dest: nil,
+      filter: nil,
+    }
+
+    def initialize
+      @param = PARAM_DEFAULT
+    end
+
+    def param(argv=ARGV)
+      optparser = OptionParser.new(ARGV) do |opts|
+    
+        opts.on_tail("--version", "-v", "Library version") do
+          puts Argos.library_version
+          exit
+        end
+      
+        opts.on("--action=ACTION", "-a", "Action") do |action|
+          param[:action] = action
+        end
+      
+        opts.on("--filter=FILTER", "-f", "Filter") do |filter|
+          param[:filter] = filter
+        end
+      
+        opts.on("--level=LEVEL", "-l=LEVEL", "Log level") do |level|
+      
+          param[:level] = case level
+            when /debug|0/i
+              Logger::DEBUG
+            when /info|1/i
+              Logger::INFO
+            when /warn|2/i
+              Logger::WARN
+            when /error|3/i
+              Logger::ERROR
+            when /fatal|4/i
+              Logger::FATAL
+            else
+              param[:level]
+          end
+        end
+      
+      end
+      optparser.parse!
+    end
+
+  end
+end

data/lib/argos/diag.rb

@@ -0,0 +1,316 @@
+module Argos
+
+  # Argos DIAG file parser
+  #
+  # Usage
+  #
+  #   diag = Argos::Diag.new
+  #   diag.log = Logger.new(STDERR)
+  #   puts diag.parse(filename).to_json
+  #
+  # For information about Argos, see: http://www.argos-system.org
+  #
+  # @author Espen Egeland
+  # @author Conrad Helgeland
+  class Diag < Array
+    include Argos
+  
+    LOCATION_CLASS = ["3", "2", "1", "0", "A", "B", "Z"]
+
+    attr_accessor :log, :filename, :programs
+
+    attr_reader :filename, :filter, :filtername, :sha1, :valid, :filesize
+  
+    START_REGEX = /^\s*\d{5,6}\s+Date : \d{2}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}/
+    $start_diag ='^\s*\d{5,6}\s+Date : \d{2}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}'
+  
+    # Diag format 1
+    #  02168  Date : 21.06.94 08:43:16  LC : Z  IQ : 00
+    #  Lat1 : ???????  Lon1 : ????????  Lat2 : ???????  Lon2 : ????????
+    #  Nb mes : 002  Nb mes>-120dB : 000  Best level : -125 dB
+    #  Pass duration : 113s   NOPC : 0
+    #  Calcul freq : 401 650000.0 Hz   Altitude :    0 m
+    #  24551           00          137
+    #$FORMAT_1 = '\s*\d{5,6} +Date : (0[1-9]|[12][0-9]|3[01])\.(0[1-9]|1[012])\.\d{2} ([0-1][0-9]|2[0-3]):([0-5][0-9]:[0-5][0-9]) +LC : (3|2|1|0|A|B|Z) +IQ : *\d{2} *Lat1 : +((\d+\.\d{3}[NS])|\?)+ +Lon1 : +((\d+\.\d{3}[EW])|\?+) +Lat2 : +((\d+\.\d{3}[NS])|\?+) +Lon2 : +((\d+\.\d{3}[EW])|\?+) +Nb mes : (\d{3})? +Nb mes>-120dB : (\d{3})? +Best level : (-\d{3})? *dB +Pass duration : *(\d+|\?+) *s? +NOPC : +([0-4]|\?) +Calcul freq : +\d{3} (\d+\.\d+)? *Hz +Altitude : +(\d+)? m( +(\d{2,5}|\w{2}))*$'
+    $FORMAT_1 =' *\d{5,6} +Date : (0[1-9]|[12][0-9]|3[01])\.(0[1-9]|1[012])\.\d{2} ([0-1][0-9]|2[0-3]):([0-5][0-9]:[0-5][0-9]) +LC : (3|2|1|0|A|B|Z) +IQ : *\d{2} *Lat1 : +((\d+\.\d{3}[NS])|\?+) +Lon1 : +((\d+\.\d{3}[EW])|\?+) +Lat2 : +((\d+\.\d{3}[NS])|\?)+ +Lon2 : +((\d+\.\d{3}[EW])|\?+) +Nb mes : (\d{3})? +Nb mes>-120dB : (\d{3})? +Best level : (-\d{3})? *dB +Pass duration : *(\d+|\?+) *s? +NOPC : *([0-4]|\?)? +Calcul freq : +\d{3} (\d+\.\d+)? *Hz +Altitude : +(\d+)? m( +.+E[+-]\d+)?( +(\d{2,5}|\w{2}))*'
+  
+  
+    # 09689  Date : 01.02.90 08:35:35  LC : 0  LI : -7
+    # Lat1 : 76.304N  Lon1 :  18.925E  Lat2 : 75.769N  Lon2 :  21.554E
+    # Nb mes : 004  Nb mes>-120Db : 000  Best level : -130 Db
+    # Pass duration : 426s   Dist track :   0
+    # Calcul freq : 401 649507.6Khz   Altitude :    0 m
+    # -.81408E+1           03           70
+    $FORMAT_2 = '\s*\d{5,6} +Date : (0[1-9]|[12][0-9]|3[01])\.(0[1-9]|1[012])\.\d{2} ([0-1][0-9]|2[0-3]):([0-5][0-9]:[0-5][0-9]) +LC : (3|2|1|0|A|B|Z) +(LI : *\-?\d+)? *Lat1 : +\d+\.\d{3}[NS] +Lon1 : +\d+\.\d{3}[EW] +Lat2 : +\d+\.\d{3}[NS] +Lon2 : +\d+\.\d{3}[EW] +Nb mes : \d{3} +Nb mes>-120(Db|dB) : \d{3} +Best level : -\d{3}? *(Db|dB) +Pass duration : +\d+s +Dist track : +\d+ +Calcul freq : +\d{3} \d+\.\d+ *(Khz|Hz) +Altitude : +\d+ m+(.+E[+-]\d+)*( +(\d{2,5}|\w{2}))*$'
+  
+  
+    # 09689  Date : 01.02.90 08:35:35  LC : 0  IQ : 00
+    # Lat1 : 76.304N  Lon1 :  18.925E  Lat2 : 75.769N  Lon2 :  21.554E
+    # Nb mes : 004  Nb mes>-120Db : 000  Best level : -130 Db
+    # Pass duration : 426s  NOPC :
+    # Calcul freq : 401 649507.6Khz   Altitude :    0 m
+    # 03           70
+    $FORMAT_3 =' *\d{5,6} +Date : (0[1-9]|[12][0-9]|3[01])\.(0[1-9]|1[012])\.\d{2} ([0-1][0-9]|2[0-3]):([0-5][0-9]:[0-5][0-9]) +LC : (3|2|1|0|A|B|Z) +IQ : *\d{2} *Lat1 : +((\d+\.\d{3}[NS])|\?+) +Lon1 : +((\d+\.\d{3}[EW])|\?+) +Lat2 : +((\d+\.\d{3}[NS])|\?)+ +Lon2 : +((\d+\.\d{3}[EW])|\?+) +Nb mes : (\d{3})? +Nb mes>-120dB : (\d{3})? +Best level : (-\d{3})? *dB +Pass duration : *(\d+|\?+) *s? +NOPC : +([0-4]|\?) +Calcul freq : +\d{3} (\d+\.\d+)? *Hz +Altitude : +(\d+)? m +.+E[+-]\d+( +(\d{2,5}|\w{2}))*$'
+  
+  
+    def initialize (filename = nil)
+      @index = 0
+      @error_num = 0
+      @errors_num =0
+      @argos_contacts = Array.new
+      @filename = filename
+      read_file (filename) unless filename == nil
+  
+    end
+
+    def filter?
+      not @filter.nil?
+    end
+    
+    def filter=filter
+      if filter.respond_to? :call
+        @filter = filter
+      elsif filter =~ /lambda|Proc/
+        @filtername = filter
+        @filter = eval(filter)
+      end
+    end
+
+    def get_hash
+      @argos_contacts
+    end
+  
+    def parse(filename=nil)
+      if filename.nil?
+        filename = @filename
+      end
+      
+      self.clear # Needed if you parse multiple times
+      @total = linecount = 0
+      @valid = false
+
+      #filename = File.realpath(filename)
+      @filename = filename
+      if filename.nil? or not File.exists? filename
+        raise ArgumentError, "Missing ARGOS DS file: \"#{filename}\""
+      end
+      @sha1 = Digest::SHA1.file(filename).hexdigest 
+
+
+      valid_file = false
+
+      contact = []
+      file = File.open(filename)
+      @filesize = file.size
+
+      log.debug "Parsing Argos DIAG file #{filename} sha1:#{sha1} (#{filesize} bytes)"
+      if filter?
+        log.info "Using filter: #{filter}"
+      end
+
+
+      linecount = 0
+      match =0
+      startline = 0
+      contact =""
+  
+      File.open(filename, :encoding => "iso-8859-1").each do |line|
+        line = line.to_s.strip
+        linecount+=1
+        if line =~ /#{$start_diag}/
+          match+=1
+          if contact !=""
+            check_format(contact.strip, startline)
+          end
+          contact = line
+          startline = linecount
+        else
+          contact = contact + " " +line
+        end
+      end
+      check_format(contact.strip, startline)
+
+      if filter?
+        total = self.size
+        filtered = self.select{|diag| filter.call(diag)}    
+        log.info "Selected #{filtered.size}/#{total} Argos DIAG segments sha1:#{sha1} #{filename}"
+        self.clear
+        
+        filtered.each do |filtered|
+          self << filtered
+        end
+      else
+        log.info "Parsed #{self.size} Argos DIAG segments sha1:#{sha1} #{filename}"
+      end
+        
+      self
+    end
+  
+  
+  def check_format(contact, line_num)
+
+    if contact =~ /#{$FORMAT_1}/ or contact =~ /#{$FORMAT_2}/ or contact =~ /#{$FORMAT_2}/
+      self << create_diag_hash(contact)
+      true
+    else
+      error = "#{__FILE__}#check_format #{filename}:#{line_num} source:#{@sha1} Invalid format:\n"  + contact
+      log.error error
+      false
+    end
+
+  end
+  
+    #http://www.argos-system.org/files/pmedia/public/r363_9_argos_users_manual-v1.5.pdf p. 48
+    #Nb mes : 025 Number of messages received
+    #Nb mes>-120 dB: 015 Number of messages received by the satellite at a signal strength greater than -120 decibels
+    #Best level : -113 dB Best signal strength, units are dB
+    #Pass duration : 900s Time elapsed between the first and last message received by the 
+    #satellite
+    #NOPC = 4 Number Of Plausibility Checks successful (from 0-4)
+    #Calcul Freq : 401 650000.3 Calculated frequency
+    #Altitude : 213 m Altitude used for location calculation
+    def create_diag_hash(contact="")
+      platform = contact[/^(\d{5,})/,1]
+      location_data = contact[/Date : (\d{2}.\d{2}.\d{2} \d{2}:\d{2}:\d{2})/,1]
+      contact_time = convert_date(location_data) unless location_data ==nil
+      lc = contact[/LC : (3|2|1|0|A|B|Z)/,1]
+      li = contact[/LI : *(\-?\d+)/,1]
+      iq= contact[/IQ : *(\d{2})/,1]
+      lat1= contact[/Lat1 : +(\d+\.\d{3}[NS]|\?+)/,1]
+      lat1 = coordinate_conv(lat1)
+      lon1= contact[/Lon1 : +(\d+\.\d{3}[EW]|\?+)/,1]
+      lon1 = coordinate_conv(lon1)
+      lat2= contact[/Lat2 : +(\d+\.\d{3}[NS]|\?+)/,1]
+      lat2 = coordinate_conv(lat2)
+      lon2= contact[/Lon2 : +(\d+\.\d{3}[EW]|\?+)/,1]
+      lon2 = coordinate_conv(lon2)
+      nb= contact[/Nb mes : (\d{3})/,1]
+      nb120= contact[/Nb mes>-120(Db|dB) : (\d{3})/,2]
+      best_level= contact[/Best level : (-\d{3})/,1]
+      pass_duration= contact[/Pass duration : +(\d+|\?)/,1]
+      dist_track= contact[/Dist track : +(\d+)/,1]
+      nopc= contact[/NOPC : +([0-4]|\?)/,1]
+      nopc = nopc =~/\?+/ ? nil : nopc
+      nopc.to_i unless nopc == nil
+      frequency = contact[/Calcul freq : +(\d{3} \d+\.\d+)/,1]
+      if frequency =~ /[ ]/
+        frequency = frequency.split(" ").join("").to_f
+      end
+      altitude= contact[/Altitude : +(\d+)? m/,1]
+      altitude = altitude.to_i unless altitude == nil
+  
+      data_start = contact.index(" m ")
+      sensor_data = contact[data_start+2,contact.length].split(" ") unless data_start == nil
+     
+      diag = { platform: platform.to_i,
+        measured: contact_time,
+        lc: lc,
+        iq: iq,
+        li: li,
+        latitude: lat1,
+        longitude: lon1,
+        latitude2: lat2,
+        longitude2: lon2,
+        messages: nb.to_i,
+        messages_120dB: nb120.to_i,
+        best_level: best_level.to_i,
+        pass_duration: pass_duration.to_i,
+        dist_track: dist_track,
+        nopc: nopc.to_i,
+        frequency: frequency,
+        altitude: altitude,
+        sensor_data: sensor_data,
+        technology: "argos",
+        type: type,
+        filename: filename,
+        source: "#{sha1}"
+      }
+
+      idbase = diag.clone
+      idbase.delete :filename
+      id = Digest::SHA1.hexdigest(idbase.to_json)
+
+      diag[:parser] = Argos.library_version
+      diag[:id] = id
+      diag
+    end
+  
+  
+    def coordinate_conv (co)
+      if co =~ /\?+/
+        co = nil
+      elsif co =~/N$/ || co =~/E$/
+        co=co.chop.to_f
+      elsif co =~/S$/ || co =~/W$/
+        co = co.chop.to_f
+        co = co * -1
+      end
+    end
+  
+  
+    def convert_date(date)
+      timestamp = DateTime.strptime(date, '%d.%m.%y %H:%M:%S').iso8601.to_s
+      timestamp['+00:00'] = "Z"
+      timestamp
+    end
+  
+    def type
+      "diag"
+    end
+
+    def messages
+      self
+    end
+
+    def programs
+      @programs ||= []
+    end
+
+    def start
+      first[:measured]
+    end
+
+    def stop
+      last[:measured]
+    end
+
+
+    def source
+      sha1
+    end
+
+  end
+end
+
+
+#52061  Date : 03.10.09 08:41:12  LC : 0  IQ : 58
+#      Lat1 : 76.651N  Lon1 :  20.930W  Lat2 : 76.168N  Lon2 :  18.539W
+#      Nb mes : 006  Nb mes>-120dB : 000  Best level : -123 dB
+#      Pass duration : 441s  NOPC :  2
+#      Calcul freq : 401 677543.2 Hz   Altitude :    0 m
+#              40           47           00           00
+#              00           00           00          240
+#              00           00           00           00
+#              00           00          185
+#52061  Date : 03.10.09 09:47:05  LC : Z  IQ : 38
+#      Lat1 : 76.998N  Lon1 :  22.398W  Lat2 : 74.370N  Lon2 :   10.000
+#      Nb mes : 003  Nb mes>-120dB : 000  Best level : -129 dB
+#      Pass duration : 100s  NOPC :  0
+#      Calcul freq : 401 677501.2 Hz   Altitude :    0 m
+#              40           35           00           00
+#              00           00           00          240
+#              00           00           00           00
+#              64           00          197
+#52061  Date : 03.10.09 10:23:35  LC : Z  IQ : 00
+#      Lat1 : ???????  Lon1 : ????????  Lat2 : ???????  Lon2 : ????????
+#      Nb mes : 001  Nb mes>-120dB : 000  Best level : -125 dB
+#      Pass duration : ???s  NOPC :  ?
+#      Calcul freq : 401 677531.1 Hz   Altitude :    0 m
+#             168           64           30
+
+
+
+
+#200910Oct.DIA
+#: Line: 26132 Invalid format:
+#52061  Date : 03.10.09 09:47:05  LC : Z  IQ : 38 Lat1 : 76.998N  Lon1 :  22.398W  Lat2 : 74.370N  Lon2 :   10.000 Nb mes : 003  Nb mes>-120dB : 000  Best level : -129 dB Pass duration : 100s  NOPC :  0 Calcul freq : 401 677501.2 Hz   Altitude :    0 m 40           35           00           00 00           00           00          240 00           00           00           00 64           00          197