argos-ruby 1.0.1 → 1.0.2

data/README.md

@@ -2,21 +2,17 @@
 
 A Ruby library for parsing Argos tracking data
 
-```argos-ruby``` has been developed to parse [Argos](http://www.argos-system.org)
-satellite tracking data collected at the [Norwegian Polar Institute]
-(http://npolar.no/en) over a 25-year period (!) from 1989 to 2013.
+[argos-ruby](https://github.com/npolar/argos-ruby) is used to parse [Argos](http://www.argos-system.org)
+satellite tracking data files collected at the [Norwegian Polar Institute]
+(http://npolar.no/en) since 1989.
 
 Be warned, the Argos file formats have changed over time. No promises are
 made that the library will work outside of Norway :).
 
-Currently, the library parses Argos DS/DIAG files dating from 1991
+Currently, the library parses Argos DS/DIAG files dating from August 1990
 and onwards.
 
 ## Install
-Now
-$ git clone https://github.com/npolar/argos-ruby.git
-
-Soon
 $ gem install argos-ruby
 
 ## Command-line usage

data/bin/argos-ruby

@@ -75,11 +75,17 @@ end
 log.debug "#{File.realpath(__FILE__)} #{param}"
 bundle = Digest::SHA1.hexdigest(glob+param[:filter].to_s)
 result = []
+argosfiles = 0
 
-Dir[glob].select {|f|
-  Argos.argos? f
-}.map {|filename|
+Dir[glob].reject {|f| File.directory? f}.select {|f|
+  argosfiles = argosfiles + 1
+  if not Argos.argos? f  
+    log.warn "Not Argos: #{f}"
+  end
   
+  Argos.argos? f  
+}.map {|filename|
+  log.debug "*"*80
   argos = Argos.factory(Argos.type(filename))
   argos.filename = filename
   argos.filter = param[:filter]
@@ -87,9 +93,7 @@ Dir[glob].select {|f|
 
   case param[:action]
     when "source"
-     
       result << Argos.source(argos).merge(glob: glob, bundle: bundle)
-      
     when "parse"
       
       arr = argos.parse(filename)
@@ -112,6 +116,8 @@ diag_count = result.select {|argos| argos[:type] == "diag"}.size
 
 if ["parse", "messages"].include? param[:action]
   log.info "Documents: #{result.size}, ds: #{ds_count}, diag: #{diag_count}, bundle: #{bundle}, glob: #{glob}"
+  
+  
 elsif "source" == param[:action]
   result = result.select {|s| s[:size] > 0 }
   sum_count = result.map {|s| s[:size]}.inject { |sum, c| sum + c }
@@ -119,13 +125,16 @@ elsif "source" == param[:action]
     s[:total] = sum_count
     s
   }
+  if argosfiles != result.size
+    log.fatal "Argos file count: #{argosfiles} differs from parsed files: #{result.size}"
+  end
   log.info "Documents (Σcount): #{sum_count}, sources: #{result.size} (ds: #{ds_count}, diag: #{diag_count}), bundle: #{bundle}, glob: #{glob}"
 end
 
 if param[:dest].nil?
   case param[:format]
     when "json"
-      puts result.to_json
+      puts JSON.pretty_generate(result)
     when /y(a)?ml/ 
       puts result.to_yaml
     when /(ruby|rb)/
@@ -138,4 +147,4 @@ exit(0)
 rescue => e
   Logger.new(STDERR).fatal e
   exit(1)
-end
+end 

data/lib/argos/diag.rb

@@ -17,9 +17,9 @@ module Argos
   
     LOCATION_CLASS = ["3", "2", "1", "0", "A", "B", "Z"]
 
-    attr_accessor :log, :filename, :programs, :multiplicates
+    attr_accessor :log, :filename, :programs
 
-    attr_reader :filename, :filter, :filtername, :sha1, :valid, :filesize
+    attr_reader :filename, :filter, :filtername, :sha1, :valid, :filesize, :updated, :multiplicates, :errors
   
     START_REGEX = /^\s*\d{5,6}\s+Date : \d{2}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}/
     $start_diag ='^\s*\d{5,6}\s+Date : \d{2}.\d{2}.\d{2} \d{2}:\d{2}:\d{2}'
@@ -53,14 +53,8 @@ module Argos
     $FORMAT_3 =' *\d{5,6} +Date : (0[1-9]|[12][0-9]|3[01])\.(0[1-9]|1[012])\.\d{2} ([0-1][0-9]|2[0-3]):([0-5][0-9]:[0-5][0-9]) +LC : (3|2|1|0|A|B|Z) +IQ : *\d{2} *Lat1 : +((\d+\.\d{3}[NS])|\?+) +Lon1 : +((\d+\.\d{3}[EW])|\?+) +Lat2 : +((\d+\.\d{3}[NS])|\?)+ +Lon2 : +((\d+\.\d{3}[EW])|\?+) +Nb mes : (\d{3})? +Nb mes>-120dB : (\d{3})? +Best level : (-\d{3})? *dB +Pass duration : *(\d+|\?+) *s? +NOPC : +([0-4]|\?) +Calcul freq : +\d{3} (\d+\.\d+)? *Hz +Altitude : +(\d+)? m +.+E[+-]\d+( +(\d{2,5}|\w{2}))*$'
   
   
-    def initialize (filename = nil)
-      @index = 0
-      @error_num = 0
-      @errors_num =0
-      @argos_contacts = Array.new
-      @filename = filename
-      read_file (filename) unless filename == nil
-  
+    def initialize
+      @errors = []  
     end
 
     def filter?
@@ -76,10 +70,7 @@ module Argos
       end
     end
 
-    def get_hash
-      @argos_contacts
-    end
-  
+    # @return []
     def parse(filename=nil)
       if filename.nil?
         filename = @filename
@@ -102,6 +93,7 @@ module Argos
       contact = []
       file = File.open(filename)
       @filesize = file.size
+      @updated = file.mtime
 
       log.debug "Parsing Argos DIAG file #{filename} sha1:#{sha1} (#{filesize} bytes)"
       if filter?
@@ -145,10 +137,11 @@ module Argos
   
       @multiplicates = group_by { |e| e }.select { |k, v| v.size > 1 }.map(&:first)
       if multiplicates.any?
-        log.warn "Multiplicates (source sha1 #{sha1} #{filename}): #{multiplicates.to_json}"
+        #log.warn "#{multiplicates.size} multiplicates in source sha1 #{sha1} #{filename}): #{multiplicates.map {|a|a[:id]} }"
         self.uniq!
         log.info "Unique DIAG messages: #{self.size} sha1: #{sha1} #{filename}"
       end
+      self.sort_by! {|diag| diag[:measured]}
     
       self
     end
@@ -160,7 +153,8 @@ module Argos
       self << create_diag_hash(contact)
       true
     else
-      error = "#{__FILE__}#check_format #{filename}:#{line_num} source:#{@sha1} Invalid format:\n"  + contact
+      error = "#{filename}:#{line_num} sha1:#{@sha1} Invalid format:\n"  + contact
+      @errors << error
       log.error error
       false
     end
@@ -229,7 +223,7 @@ module Argos
         sensor_data: sensor_data,
         technology: "argos",
         type: type,
-        filename: filename,
+        location: "file://"+filename,
         source: "#{sha1}"
       }
 

data/lib/argos/ds.rb

@@ -19,7 +19,7 @@ module Argos
 
     attr_writer :log, :filename
 
-    attr_reader :filename, :filter, :filtername, :valid, :filesize, :sha1, :messages, :multiplicates
+    attr_reader :filename, :filter, :filtername, :valid, :filesize, :updated, :sha1, :messages, :multiplicates, :errors
 
     START_REGEX = /^\d{5} \d{5,6} +\d+ +\d+/
 
@@ -27,7 +27,8 @@ module Argos
 
     LOCATION_CLASS = [nil, "0","1","2","3","A","B","G","Z"]
 
-    def self.ds? filename
+    def initialize
+      @errors = []
     end
 
     def filter?
@@ -78,7 +79,8 @@ module Argos
       contact = []
       file = File.open(filename)
       @filesize = file.size
-
+      @updated = file.mtime
+      
       log.debug "Parsing ARGOS DS file #{filename} source:#{sha1} (#{filesize} bytes)"
       if filter?
         log.debug "Using filter: #{@filtername.nil? ? filter : @filtername }"
@@ -149,11 +151,11 @@ module Argos
 
       @multiplicates = group_by { |e| e }.select { |k, v| v.size > 1 }.map(&:first)
       if multiplicates.any?
-        log.warn "Multiplicates (source sha1 #{sha1} #{filename}): #{multiplicates.to_json}"
+        log.warn "#{multiplicates.size} multiplicates in source sha1 #{sha1} #{filename}): #{multiplicates.map {|a|a[:id]} }"
         self.uniq!
         log.info "Unique DS messages: #{self.size} sha1: #{sha1} #{filename}"
       end
-      
+      self.sort_by! {|ds| ds[:measured]}
       self
     end
   
@@ -303,11 +305,15 @@ module Argos
       unfolded
     end
 
+    # Merges a DS header hash into each measurement
+    # @return [Array] Measurements with header and static metadata merged in
     def merge(ds, measurement)
       m = ds.select {|k,v| k != :measurements and k != :errors and k != :warn }     
       m = m.merge(measurement)          
-      m = m.merge({ technology: "argos",
-        type: type, filename: filename, source: sha1
+      m = m.merge ({ technology: "argos",
+        type: type,
+        location: "file://"+filename,
+        source: sha1
       })
 
       if not ds[:errors].nil? and ds[:errors].any?
@@ -325,6 +331,27 @@ module Argos
         m[:warn] << "sensors-count-mismatch"
       end
 
+      # Create id as SHA1 hash of measurement minus stuff that may vary (like filename)
+      # 
+      # Possible improvement for is to base id on a static list of keys
+      # :program,
+      # :platform,
+      # :lines,
+      # :sensors,
+      # :satellite,
+      # :lc,
+      # :positioned,
+      # :latitude,
+      # :longitude,
+      # :altitude,
+      # :headers,
+      # :measured,
+      # :identical,
+      # :sensor_data,
+      # :technology,
+      # :type,
+      # :source
+
       idbase = m.clone
       idbase.delete :errors
       idbase.delete :filename

data/lib/argos.rb

@@ -16,7 +16,7 @@ require_relative "argos/diag"
 # 
 # For information about Argos, see: http://www.argos-system.org
 module Argos
-  VERSION = "1.0.1"
+  VERSION = "1.0.2"
   # Detect Argos type ("ds" or "diag" or nil)
   #
   # @param filename [String] Argos (DS or DIAG) file
@@ -116,15 +116,20 @@ module Argos
       west: argos.longitudes.min,
       latitude_mean: latitude_mean,
       longitude_mean: longitude_mean,
-      filename: argos.filename,
-      filesize: argos.filesize,
+      location: "file://"+argos.filename,
+      bytes: argos.filesize,
+      updated: argos.updated.xmlschema,
       messages: argos.messages.size,
       filter: argos.filtername.nil? ? argos.filter : argos.filtername,
-      size: argos.size
+      size: argos.size,
+      parser: library_version
     }
     if argos.multiplicates.any?
       source[:multiplicates] = argos.multiplicates.map {|a| a[:id]}
     end
+    if argos.errors.any?
+      source[:errors] = argos.errors
+    end
     source
     
   end

data/spec/argos/_diag/error.diag

@@ -0,0 +1,7 @@
+ 14759  Date : 28.08.91 13:47:39  LC : 1                                        
+      Lat1 : 76.990N  Lon1 :  22.232E  Lat2 : 80.748N  Lon2 :  37.670E          
+      Nb mes : 005  Nb mes>-120dB : 004  Best level : -116 dB                   
+      Pass duration : 292s   Dist track :   3                                   
+      Calcul freq : 401 649728.9 Hz   Altitude :    0 m                         
+      -.15686E+1           00           16           50                         
+ 14759  Date : 28.08.91 15:10:23  LC : 1                                        

data/spec/argos/diag_spec.rb

@@ -28,6 +28,10 @@ module Argos
       it "should return unique elements" do
         @diag.parse(diagfile("dup.diag")).size.should == 1
       end
+      
+      it "should store errors" do
+        @diag.parse(diagfile("error.diag")).errors[0].should =~ /Invalid format/
+      end
   
       describe "#size" do
         it "should == number messages" do
@@ -55,8 +59,8 @@ module Argos
         :dist_track => nil,
         :technology=>"argos",
         :type=>"diag",
-        :filename=>VALID_DIAG,
-        :id => "a4e59580432b7b621f66c0cdc3087127554acd1d",
+        :location=>"file://"+VALID_DIAG,
+        :id => "ea2243d95df7d70a02a4fb66bc8876f10fdabf12",
         :parser => "argos-ruby-#{Argos::VERSION}",
         :source =>"f53ae3ab454f3e210347439aa440c084f775f9a4"}
         end

data/spec/argos/ds_spec.rb

@@ -61,7 +61,7 @@ module Argos
          :sensor_data,
          :technology,
          :type,
-         :filename,
+         :location,
          :source,
          :parser,
          :id]}
@@ -90,8 +90,8 @@ module Argos
 :positioned=>"1999-12-30T18:41:56Z", :latitude=>79.828,
 :longitude=>22.319, :altitude=>0.0, :headers=>12,
 :measured=>"1999-12-30T18:43:52Z", :identical=>3, :sensor_data=>["78", "00", "00"],
-:technology=>"argos", :type=>"ds", :filename=>VALID_DS, :parser => "argos-ruby-#{Argos::VERSION}",
-:source=>"3a39e0bd0b944dca4f4fbf17bc0680704cde2994", :id=>"4369c31c191bd55a998e6293ff4639da3984a95d"}]
+:technology=>"argos", :type=>"ds", :location=>"file://"+VALID_DS, :parser => "argos-ruby-#{Argos::VERSION}",
+:source=>"3a39e0bd0b944dca4f4fbf17bc0680704cde2994", :id=>"c78642c2bbfd4188df8940cd966b40cef33f2881"}]
           end
         end
       end
@@ -110,11 +110,11 @@ module Argos
 :latitude=>nil, :longitude=>nil, :altitude=>nil, :headers=>5,
 :measured=>"1999-12-16T00:46:49Z", :identical=>1,
 :sensor_data=>["92", "128", "130", "132"], :technology=>"argos",
-:type=>"ds", :filename=>VALID_DS,
+:type=>"ds", :location=> "file://"+VALID_DS,
 :source=>"3a39e0bd0b944dca4f4fbf17bc0680704cde2994",
 :warn=>["missing-position", "sensors-count-mismatch"],
 :parser=>"argos-ruby-#{Argos::VERSION}",
-:id=>"f2c82a5ca1330b312925949a15ac300d07452a12"}]
+:id=>"e81e2f255b62fa88e20351de8d5b2de9bf6b3028"}]
       end
 
     end

data/spec/argos_spec.rb

@@ -0,0 +1,67 @@
+require 'spec_helper.rb'
+
+module Argos
+  
+  describe Argos do
+    describe "source" do
+      context "DS 990660_A.DAT" do
+        ds = Ds.new
+        ds.log = Logger.new("/dev/null")
+        ds.parse dsfile("990660_A.DAT")
+        it do
+          Argos.source(ds).should ==  {:id=>"3a39e0bd0b944dca4f4fbf17bc0680704cde2994",
+:technology=>"argos",
+:collection=>"tracking",
+:type=>"ds",
+:programs=>[660, 9660],
+:platforms=>[2167, 2168, 2170, 2173, 2174, 2176, 2179, 2180, 2188, 2189, 8567, 8571, 8574, 8576, 8578, 8579, 9678, 9686, 9692, 9693, 9694, 9696, 9698, 9699, 10618, 10619, 10622, 10783, 11104, 14747, 14765],
+:start=>"1999-12-01T10:23:48Z",
+:stop=>"1999-12-31T19:18:58Z",
+:north=>79.866,
+:east=>55.068,
+:south=>75.822,
+:west=>22.309,
+:latitude_mean=>78.394,
+:longitude_mean=>33.5,
+:location=>"file://#{dsfile("990660_A.DAT")}",
+:bytes=>107584,
+:messages=>449,
+:filter=>nil,
+:size=>843,
+:updated => "2013-10-21T19:02:32+02:00",
+:parser=>Argos.library_version }
+        end
+      end
+      context "DS 990660_A.DIA" do
+        diag = Diag.new
+        diag.log = Logger.new("/dev/null")
+        diag.parse diagfile("990660_A.DIA")
+        it do
+          Argos.source(diag).should == {:id=>"f53ae3ab454f3e210347439aa440c084f775f9a4",
+:technology=>"argos",
+:collection=>"tracking",
+:type=>"diag",
+:programs=>[],
+:platforms=>[2167, 2168, 2170, 2173, 2174, 2176, 2179, 2180, 2188, 2189, 8567, 8571, 8574, 8576, 8578, 8579, 9678, 9686, 9692, 9693, 9694, 9696, 9698, 9699, 10618, 10619, 10622, 10783, 11104, 14747, 14765],
+:start=>"1999-12-01T10:24:54Z",
+:stop=>"1999-12-31T19:18:58Z",
+:north=>87.011,
+:east=>65.27,
+:south=>65.417,
+:west=>-58.627,
+:latitude_mean=>78.124,
+:longitude_mean=>30.752,
+:location=>"file://#{diagfile("990660_A.DIA")}",
+:bytes=>222056,
+:updated=>"2013-10-21T20:31:55+02:00",
+:messages=>448,
+:filter=>nil,
+:size=>448,
+:parser=>Argos.library_version}
+        end
+      end
+      
+    end
+    
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: argos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-10 00:00:00.000000000 Z
+date: 2014-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -48,12 +48,14 @@ files:
 - lib/argos/exception.rb
 - spec/argos/_diag/990660_A.DIA
 - spec/argos/_diag/dup.diag
+- spec/argos/_diag/error.diag
 - spec/argos/_diag/valid_2009-03-02_DB_DIAG.txt
 - spec/argos/_ds/990660_A.DAT
 - spec/argos/_ds/dup.ds
 - spec/argos/_ds/sensor_mismatch_ds.txt
 - spec/argos/diag_spec.rb
 - spec/argos/ds_spec.rb
+- spec/argos_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/npolar/argos-ruby
 licenses: