argos-ruby 1.1.3.1 → 1.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YzBiNzk2MjU0YWQ0NGViYzk2ZjMzMjZhMTBjYjgzNjY4M2U2MzdkMg==
-  data.tar.gz: !binary |-
-    ZjNhYTliZDM2ZWNjOTg4OWFlMzEyYjZiMmYzMjVkZDY0MDc4MmUzMA==
+SHA1:
+  metadata.gz: c8fc28465dbe802bf2961bea1d3913654d4969b9
+  data.tar.gz: fb761e384ab9d9761bd601c641fcbd87e0d2525a
 SHA512:
-  metadata.gz: !binary |-
-    ZDI3YzNiYTlhMmY2MmU4NGMwOTQxZThjNzc5NjgwZjc2NDAzNjgxNzYzNDQw
-    ZTVmZWM5ZTc3ZmM0OWNkZDE2ZTNiZGI0ZTc2NzFmY2UzOTM5ZGRjYWYxOGM4
-    NmVmOWU3ODAxZDdlOWVkNmJlODg3MDVhNGJmMWQ0NTA1NzJlZjY=
-  data.tar.gz: !binary |-
-    MjlkNmM4MDhmMDZhNGIwY2FlNWU2MTYxNmQ1MWJlMTUxNDEyZDYxMjY0MThh
-    MmZjNGU3NTgyOGE4MjZhNTUwNGVlZDI1OTYyYjEwYjQ3MmQxMWIxMWE0NTQz
-    NjQyYWI4Y2EwNDFjMWUwNzg1MjEyMjExZGUzOWZjYTdiMzQ3NGI=
+  metadata.gz: 74ac8e53f830ec8119d0339e9ea12c98f09640baf7992b1e3288c92d23f80ae13198a0bf32206ff0b047b2610c48403271d3f54db44145022f720f07297678ea
+  data.tar.gz: c4894a56bdae7a75c581f3dff27da1c2c91a88a1a67f58745c2755a6ba6bc5cbd7731d10d747715d9adfe12b9dfbaa3dbf4bc9eefbab734ca897583805551106

data/Gemfile

@@ -4,12 +4,9 @@ gem "yajl-ruby", git: "git://github.com/brianmario/yajl-ruby.git"
 gem "uuidtools"
 gem "hashie", git:  "git://github.com/intridea/hashie.git"
 #gem "json-schema", git:  "git://github.com/hoxworth/json-schema.git"
-gem "savon", github: 'savonrb/savon'
+gem "savon", github: "savonrb/savon", branch: "version3"
 
 group :development, :test do
-  gem "thin"
-  gem "shotgun"
   gem "rspec"
   gem "simplecov"
-  gem "ruby-prof"
 end

data/README.md

@@ -1,57 +1,46 @@
 # argos-ruby
 
-A Ruby library and command-line tool accessing data from the [Argos](http://www.argos-system.org) tracking system.
+[argos-ruby](https://github.com/npolar/argos-ruby) contains tools for working with
+[Argos](http://www.argos-system.org) satellite tracking data and services:
 
-## argos-soap (webservice client)
+* Parsers for Argos legacy ASCII (DS/DAT and DIAG/DIA files)
+* SOAP web service client
+* Web service download tool
+* JSON conversion from ASCII and XML
 
-  $ argos-soap --operations
-```json
-["getCsv","getStreamXml","getKml","getXml","getXsd","getPlatformList","getObsCsv","getObsXml"]
-```
-**All programs**
-Data from all programs and all platforms are fetched by default
-
-$ argos-soap --operation=getXml > [getXml.xml](https://github.com/npolar/argos-ruby/blob/master/spec/argos/_soap/getXml.xml)
-
-$ argos-soap --operation=getXml --format=json > [getXml.json](https://github.com/npolar/argos-ruby/blob/master/spec/argos/_soap/getXml.json)
-
-$ argos-soap --operation=getKml > [getKml.xml](https://github.com/npolar/argos-ruby/blob/master/spec/argos/_soap/getKml.xml)
-
-$ argos-soap --operation=getCsv --format=text > [getCsv.csv](https://github.com/npolar/argos-ruby/blob/master/spec/argos/_soap/getCsv.csv)
-
-**Select program(s)**
-argos-soap --operation getXml --programNumber 9660
-
-**Select platformId(s)**
-$ argos-soap --operation getXml --platformId 81308
+Developed by staff at the [Norwegian Polar Data Centre](http://data.npolar.no), [Norwegian Polar Institute](http://npolar.no).
 
+## Webservice
 
-**Authentication**
-Either set your credentials on the command line
-  $ argos-soap --operation=getPlatformList --username=myusername --password=mypassword
-Or set environmental variables ARGOS_SOAP_USERNAME and ARGOS_SOAP_PASSWORD
-
-## Argos file parsing
-### DS/DIAG to JSON
-  $ # 
+```sh
+$ argos-soap --download archive/tracking/CLS  --username=USERNAME --password=PASSWORD --debug
+$ argos-soap -o getXml --username=USERNAME --password=PASSWORD
+```
+See [argos-soap](https://github.com/npolar/argos-ruby/wiki/argos-soap) for more usage examples.
 
-## About
+### Legacy file parsing
 
-[argos-ruby](https://github.com/npolar/argos-ruby) was developed to parse [Argos](http://www.argos-system.org)
-satellite tracking data text files collected by the [Norwegian Polar Institute]
-(http://npolar.no/en) since 1989. Be warned, the Argos text file formats have changed over time. No promises are
-made that the library will work outside of Norway :).
+```sh
+$ argos-ascii spec/argos/_ds/*.DAT
+$ argos-ascii --action=source "spec/argos/_d*"
+$ argos-ascii --filter='lambda {|d| d[:program] == 9660 and d[:platform] == 2189 }' spec/argos/_ds/990660_A.DAT
+```
+The **source** action provides a metadata summary, list of programs, platforms, etc.
 
-Currently, the library parses Argos DS/DIAG files dating from August 1990 and onwards.
+### JSON (via XSLT)
+```sh
+$ xsltproc lib/argos/_xslt/argos-json.xslt spec/argos/_soap/getXml.xml 
+```
 
 ## Install
+```sh
 $ gem install argos-ruby
+```
 
 ## Links
-
 * https://github.com/npolar/api.npolar.no/wiki/Tracking-API-JSON
 * [Argos User's Manual](http://www.argos-system.org/manual/)
-* [Argos Web Service Interface Specification v1.5](http://www.argos-system.org/manual/argos_webservices-1_4.pdf)
+* [Argos Web Service Interface Specification v1.4](http://www.argos-system.org/manual/argos_webservices-1_4.pdf)
 * http://alaska.usgs.gov/science/biology/spatial/
 * http://gis-lab.info/programs/argos/argos-manual-eng.html
-
+* [CLS](http://www.cls.fr/welcome_en.html) - operates the Argos system

data/argos-ruby.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.summary     = %q{Argos satellite tracking library and command-line tools}
   s.description = %q{Argos (http://www.argos-system.org/) webservice client and Argos legacy file (DS/DAT and DIAG/DIA) parser.}
   s.license = "GPL-3.0"
-  s.add_development_dependency "rspec"
+  s.add_development_dependency 'rspec', '~> 3'
   s.files         = `git ls-files`.split("\n")
   s.executables      = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/bin/argos-soap

@@ -4,10 +4,12 @@
 # Ruby-based command line client for Argos' Soap webservice
 #
 # For more information:
-# $ ./bin/argos-soap --help
+# $ argos-soap --help
 # https://github.com/npolar/argos-ruby/blob/master/README.md
 
-require_relative "../lib/argos"
-require_relative "../lib/argos/soap_command"
+Dir.chdir(__dir__) do
+  require_relative "../lib/argos"
+  require_relative "../lib/argos/soap_command"
+end
+Argos::SoapCommand.run
 
-Argos::SoapCommand.run

data/lib/argos/_xsd/argos-data.xsd

@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>  
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+<!-- W3C specifications   http://www.w3.org/TR/xmlschema-0  -->
+
+<!-- DATA DECLARATION -->
+<xs:element name="data">
+  <xs:complexType>
+    <xs:sequence>
+      <xs:element name="program" type="program" minOccurs="0" maxOccurs="unbounded"/>
+      <xs:element name="errors" type="errors" minOccurs="0"/>
+    </xs:sequence>
+    <xs:attribute name="version" type="xs:string"/>
+  </xs:complexType>
+</xs:element>
+
+<!-- PROGRAM DECLARATION -->
+<xs:complexType name="program">
+  <xs:sequence>
+    <xs:element name="programNumber" type="xs:int" minOccurs="0"/>
+    <xs:element name ="platform" type="platform" minOccurs="0" maxOccurs="unbounded"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- PLATFORM DECLARATION -->
+<xs:complexType name="platform">
+  <xs:sequence>
+    <xs:element name="platformId" type="xs:int" minOccurs="0"/>
+    <xs:element name="platformType" type="xs:string" minOccurs="0"/>
+    <xs:element name="platformName" type="xs:string" minOccurs="0"/>
+    <xs:element name="platformModel" type="xs:string" minOccurs="0"/>
+    <xs:element name="platformHexId" type="xs:string" minOccurs="0"/>
+    <xs:element name ="satellitePass" type="satellitePass" minOccurs="0" maxOccurs="unbounded"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- SATELLITE PASS DECLARATION -->
+<xs:complexType name="satellitePass">
+  <xs:sequence>
+    <xs:element name="satellite" type="xs:string" minOccurs="0"/>
+    <xs:element name="bestMsgDate" type="xs:dateTime" minOccurs="0"/>
+    <!-- unit : second -->
+    <xs:element name="duration" type="xs:int" minOccurs="0"/>
+    <xs:element name="nbMessage" type="xs:int" minOccurs="0"/>
+    <xs:element name="message120" type="xs:int" minOccurs="0"/>
+    <!-- unit : dBm -->
+    <xs:element name="bestLevel" type="xs:int" minOccurs="0"/>
+	<!-- unit : Hertz -->
+    <xs:element name="frequency" type="xs:double" minOccurs="0"/>
+    <xs:element name="location" type="location" minOccurs="0"/>
+    <xs:element name ="message" type="message" minOccurs="0" maxOccurs="unbounded"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- LOCATION DECLARATION -->
+<xs:complexType name="location">
+  <xs:sequence>
+  	<!-- GMT -->
+    <xs:element name="locationDate" type="xs:dateTime" minOccurs="0"/>
+    <!-- range : -90°..+90° -->
+    <xs:element name="latitude" type="xs:double" minOccurs="0"/>
+    <!-- range : -180°..+180° -->
+    <xs:element name="longitude" type="xs:double" minOccurs="0"/>
+    <!-- unit : meter -->
+    <xs:element name="altitude" type="xs:double" minOccurs="0"/>
+    <xs:element name="locationClass" type="xs:string" minOccurs="0"/>
+    <!-- unit : meter/second -->
+    <xs:element name="gpsSpeed" type="xs:float" minOccurs="0"/>
+    <!-- range : 0°..+360°, clockwise -->
+    <xs:element name="gpsHeading" type="xs:float" minOccurs="0"/>
+    <xs:element name="diagnostic" type="diagnostic" minOccurs="0"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- DIAGNOSTIC DECLARATION -->
+<xs:complexType name="diagnostic">
+  <xs:sequence>
+  	<!-- range : -90°..+90° -->
+    <xs:element name="latitude2" type="xs:float" minOccurs="0"/>
+    <!-- range : -180°..+180° -->
+    <xs:element name="longitude2" type="xs:float" minOccurs="0"/>
+    <!-- unit : meter -->
+    <xs:element name="altitude2" type="xs:float" minOccurs="0"/>
+    <xs:element name="index" type="xs:int" minOccurs="0"/>
+    <xs:element name="nopc" type="xs:int" minOccurs="0"/>
+    <!-- unit : meter -->
+    <xs:element name="errorRadius" type="xs:float" minOccurs="0"/>
+    <!-- unit : meter -->
+    <xs:element name="semiMajor" type="xs:float" minOccurs="0"/>
+    <!-- unit : meter -->
+    <xs:element name="semiMinor" type="xs:float" minOccurs="0"/>
+    <!-- range 0°..180°, clockwise -->
+    <xs:element name="orientation" type="xs:float" minOccurs="0"/>
+    <xs:element name="hdop" type="xs:string" minOccurs="0"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- MESSAGE DECLARATION -->
+<xs:complexType name="message">
+  <xs:sequence>
+  	<!-- GMT -->
+    <xs:element name="bestDate" type="xs:dateTime" minOccurs="0"/>
+    <xs:element name="compression" type="xs:int" minOccurs="0"/>
+    <xs:element name ="collect" type="collect" minOccurs="0" maxOccurs="unbounded"/>
+    <xs:element name ="format" type="format" minOccurs="0" maxOccurs="unbounded"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- COLLECT DECLARATION -->
+<xs:complexType name="collect">
+  <xs:sequence>
+    <xs:element name="type" type="xs:string" minOccurs="0"/>
+    <xs:element name="alarm" type="xs:string" minOccurs="0"/>
+    <xs:element name="concatenated" type="xs:string" minOccurs="0"/>
+    <!-- GMT -->
+    <xs:element name="date" type="xs:dateTime" minOccurs="0"/>
+    <!-- unit : dBm -->
+    <xs:element name="level" type="xs:float" minOccurs="0"/>
+	<!-- unit : Hertz -->
+    <xs:element name="doppler" type="xs:double" minOccurs="0"/>
+    <xs:element name="rawData" type="xs:string" minOccurs="0"/>
+  </xs:sequence>
+</xs:complexType>
+
+
+<!-- FORMAT DECLARATION -->
+<xs:complexType name="format">
+  <xs:sequence>
+    <xs:element name="formatOrder" type="xs:int" minOccurs="0"/>
+    <xs:element name="formatName" type="xs:string" minOccurs="0"/>
+    <xs:element name ="sensor" type="sensor" minOccurs="0" maxOccurs="unbounded"/>
+  </xs:sequence>
+</xs:complexType>
+
+<!-- SENSOR DECLARATION -->
+<xs:complexType name="sensor">
+  <xs:sequence>
+    <xs:element name="order" type="xs:int" minOccurs="0"/>
+    <xs:element name="name" type="xs:string" minOccurs="0"/>
+    <xs:element name="valueType" type="xs:string" minOccurs="0"/>
+    <xs:element name="value" type="xs:string" minOccurs="0"/>
+    <xs:element name="valueStr" type="xs:string" minOccurs="0"/>
+  </xs:sequence>
+</xs:complexType>
+
+
+<!-- ERRORS DECLARATION -->
+<xs:complexType name="errors">
+  <xs:sequence>
+    <xs:element name ="error" type="xs:string" minOccurs="0" maxOccurs="unbounded">
+      
+    </xs:element>
+  </xs:sequence>
+</xs:complexType>
+
+</xs:schema>
+
+

data/lib/argos/_xslt/argos-json.xslt

@@ -0,0 +1,81 @@
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+ <xsl:output omit-xml-declaration="yes" indent="yes"/>
+ <xsl:strip-space elements="*"/>
+<!--
+  Transform Argos data XML to Tracking JSON (http://api.npolar.no/schema/tracking)
+
+  Usage:
+    xsltproc argos-json.xslt argos.xml
+    
+    xsltproc argos-json.xslt - -stringparam filename "argos-2014-06-11-platform-129654.xml" argos-2014-06-11-platform-129654.xml
+    
+    @todo param name system to force gps on known gps devices?
+    @todo test with errors
+    
+-->
+  <xsl:param name="filename" select="''"/>
+  <xsl:param name="npolar" select="'true'"/>
+
+
+<xsl:template match="/">  
+[<xsl:for-each select="/data[number(@version) &lt; 2]/program/platform/satellitePass/message">
+  
+  <xsl:variable name="programNumber" select="ancestor::program/programNumber" />
+  <xsl:variable name="platform" select="ancestor::program/platform" />
+  <xsl:variable name="satellitePass" select="parent::satellitePass" />
+  <xsl:variable name="location" select="$satellitePass/location" />
+  
+{ "program": "<xsl:value-of select="$programNumber"/>",
+  "platform": "<xsl:value-of select="$platform/platformId"/>",
+  "platform_type": "<xsl:value-of select="$platform/platformType"/>",
+  "platform_model": "<xsl:value-of select="$platform/platformModel"/>",
+  "platform_name": "<xsl:value-of select="$platform/platformName"/>",
+  "platform_id": "<xsl:value-of select="$platform/platformHexId"/>",  
+  "latitude": <xsl:choose><xsl:when test="number($location/latitude) = $location/latitude"><xsl:value-of select="$location/latitude"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "longitude": <xsl:choose><xsl:when test="number($location/longitude) = $location/longitude"><xsl:value-of select="$location/longitude"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "altitude": <xsl:choose><xsl:when test="number($location/altitude) = $location/altitude"><xsl:value-of select="$location/altitude"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "measured": "<xsl:value-of select="$satellitePass/bestMsgDate"/>",
+  "satellite": "<xsl:value-of select="$satellitePass/satellite"/>",
+  "duration": <xsl:value-of select="$satellitePass/duration"/>,
+  "messages": <xsl:value-of select="$satellitePass/nbMessage"/>,
+  "messages_120dB": <xsl:choose><xsl:when test="number($satellitePass/message120) = $satellitePass/message120"><xsl:value-of select="$satellitePass/message120"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "best_level": <xsl:choose><xsl:when test="number($satellitePass/bestLevel) = $satellitePass/bestLevel"><xsl:value-of select="$satellitePass/bestLevel"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "frequency": <xsl:choose><xsl:when test="number($satellitePass/frequency) = $satellitePass/frequency"><xsl:value-of select="$satellitePass/frequency"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>, 
+  "positioned": <xsl:choose><xsl:when test="$location/locationDate != ''">"<xsl:value-of select="$location/locationDate"/>"</xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "lc": <xsl:choose><xsl:when test="$location/locationClass != ''">"<xsl:value-of select="$location/locationClass"/>"</xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,<xsl:choose><xsl:when test="$location/diagnostic">
+  "lc_index": <xsl:choose><xsl:when test="number($location/diagnostic/index) = $location/diagnostic/index"><xsl:value-of select="$location/diagnostic/index"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "latitude2": <xsl:value-of select="$location/diagnostic/latitude2"/>,
+  "longitude2": <xsl:value-of select="$location/diagnostic/longitude2"/>,
+  "altitude2": <xsl:value-of select="$location/diagnostic/altitude2"/>,
+  "nopc": <xsl:value-of select="$location/diagnostic/nopc"/>,
+  "error_radius": <xsl:value-of select="$location/diagnostic/errorRadius"/>,
+  "semi_major": <xsl:value-of select="$location/diagnostic/semiMajor"/>,
+  "semi_minor": <xsl:value-of select="$location/diagnostic/semiMinor"/>,
+  "speed": <xsl:choose><xsl:when test="number($location/diagnostic/gpsSpeed) = $location/diagnostic/gpsSpeed"><xsl:value-of select="$location/diagnostic/gpsSpeed"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "heading": <xsl:choose><xsl:when test="number($location/diagnostic/gpsHeading) = $location/diagnostic/gpsHeading"><xsl:value-of select="$location/diagnostic/gpsHeading"/></xsl:when><xsl:otherwise>null</xsl:otherwise></xsl:choose>,
+  "orientation": <xsl:value-of select="$location/diagnostic/orientation"/>,
+  "hdop": <xsl:value-of select="$location/diagnostic/hdop"/>,</xsl:when></xsl:choose>
+  "row": <xsl:value-of select="position()"/>,
+  "total": <xsl:value-of select="count(//message)"/>,
+  "file": "<xsl:value-of select="$filename"/>",
+  "identical": <xsl:value-of select="compression"/>,
+  "sensor_data": [<xsl:for-each select="format/sensor/value">"<xsl:value-of select="."/>"<xsl:choose><xsl:when test="position() &lt; last()">,</xsl:when></xsl:choose></xsl:for-each>],
+  "sensor_hex": "<xsl:value-of select="collect/rawData"/>",
+  "sensor": [<xsl:for-each select="format/sensor">{"name": "<xsl:value-of select="name"/>", "value": "<xsl:value-of select="value|valueStr"/>", "value_type": "<xsl:value-of select="valueType"/>", "order": <xsl:value-of select="order"/> }<xsl:choose><xsl:when test="position() &lt; last()">, </xsl:when></xsl:choose></xsl:for-each>],
+  "doppler": <xsl:value-of select="collect/doppler"/>,
+  "level": <xsl:value-of select="collect/level"/>,
+  "collects": <xsl:value-of select="count(collect)"/>,
+  "formats": <xsl:value-of select="count(format)"/>,
+  "collect": [<xsl:for-each select="collect">{ "level": <xsl:value-of select="level"/>, "type": "<xsl:value-of select="type"/>", "alarm": "<xsl:value-of select="alarm"/>", "concatenated": "<xsl:value-of select="concatenated"/>", "hex": "<xsl:value-of select="rawData"/>", "doppler": <xsl:value-of select="doppler"/>, "measured": "<xsl:value-of select="date"/>" }<xsl:choose><xsl:when test="position() &lt; last()">,</xsl:when></xsl:choose></xsl:for-each>],
+  "format": [<xsl:for-each select="format">{ "order": <xsl:value-of select="formatOrder"/>, "name": "<xsl:value-of select="formatName"/>" }<xsl:choose><xsl:when test="position() &lt; last()">,</xsl:when></xsl:choose></xsl:for-each>]<xsl:choose><xsl:when test="$npolar = 'true'">,
+  "schema": "http://api.npolar.no/schema/tracking",
+  "collection": "tracking",
+  "technology": "argos",
+  "system": "<xsl:choose><xsl:when test="number($location/diagnostic/gpsSpeed) = $location/diagnostic/gpsSpeed">gps</xsl:when><xsl:otherwise>argos</xsl:otherwise></xsl:choose>",
+  "parser": "argos-json.xslt",
+  "type": "xml"</xsl:when></xsl:choose><xsl:choose><xsl:when test="count(/data/errors/error) &gt; 0">,
+  "warn": [<xsl:for-each select="/data/errors/error">"<xsl:value-of select="."/> (argos webservice error code <xsl:value-of select="@code"/>)"<xsl:choose><xsl:when test="position() &lt; last()">,</xsl:when></xsl:choose></xsl:for-each>]</xsl:when></xsl:choose>}<xsl:choose><xsl:when test="position() &lt; last()">,</xsl:when></xsl:choose>
+</xsl:for-each>]
+</xsl:template>
+
+</xsl:stylesheet>

data/lib/argos/download.rb

@@ -1,6 +1,8 @@
 require "fileutils"
 require "logger"
 require "time"
+require "digest"
+require "nokogiri"
 
 module Argos
   
@@ -11,17 +13,22 @@ module Argos
   # /{archive}/{year}/program-{programNumber}/platform-{platformId}/argos-{YYYY-MM-DD}-platform-{platformId}.[xml|csv]
   #
   # Command line:
-  # $ ./bin/argos-soap --download /mnt/tracking/ws-argos.cls.fr --username=xyz --password=123
+  # $ ./bin/argos-soap --download /tmp/tracking/ws-argos.cls.fr --username=xyz --password=123 --debug
+  # Crontab example:
+  # 27 3 * * * source /home/argos/.rvm/environments/default && argos-soap --download /var/ws-argos.cls.fr --username=user --password=pw
+
 
   class Download
     
     def self.download(username, password, archive, log, days=20)
       
+      log.debug "Starting download of Argos XML to #{archive}"
+      
       soap = Argos::Soap.new({username: username, password: password})
       soap.log = log
       
       programs = soap.programs
-      
+      log.debug "Programs for #{username}: #{programs.to_json}"
       year = DateTime.now.year
   
       soap.getPlatformList["data"]["program"].each do |program|
@@ -29,9 +36,8 @@ module Argos
         soap.programNumber = programNumber
       
         platforms = soap.platforms
-        log.debug "Program: #{program["programNumber"]}, #{platforms.size} platform(s)"
         
-        program["platform"].select {|platform|
+        active = program["platform"].select {|platform|
           
           lastCollectDate = DateTime.parse(platform["lastCollectDate"])
           lastLocationDate = DateTime.parse(platform["lastLocationDate"])
@@ -40,13 +46,24 @@ module Argos
           
           (lastCollectDate > twentydays or lastLocationDate > twentydays)
           
-          }.each_with_index do | platform, idx |
+        }
+        
+        
+        inactive = program["platform"] - active
+        
+        active.each_with_index do |a,m|
+          log.debug "Active [#{m+1}/#{active.size}]: #{a.reject{|k,v| k =~ /location/i }.to_json}"
+        end
+        inactive.each_with_index do |i,n|
+          log.debug "Inactive [#{n+1}/#{inactive.size}]: #{i.reject{|k,v| k =~ /location/i }.to_json}"
+        end
+        active.each_with_index do | platform, idx |
           
       
           platformId = platform["platformId"]
           soap.platformId = platformId
           
-          log.debug "Program: #{programNumber}, platform: #{platformId}, lastCollectDate: #{platform["lastCollectDate"]}"
+          log.debug "About to download program: #{programNumber}, platform: #{platformId} [#{idx+1}/#{active.size}], lastCollectDate: #{platform["lastCollectDate"]}"
         
           20.downto(1) do |daysago|
             
@@ -58,27 +75,48 @@ module Argos
               soap.period = {startDate: "#{date}T00:00:00Z", endDate: "#{date}T23:59:59.999Z"}
               soap.getXml
               
-              
               FileUtils.mkdir_p(destination)
               filename = destination+"/argos-#{date}-platform-#{platformId}.xml"
-                
-              File.open(filename, "wb") { |file| file.write(soap.xml)}
-              log.debug "Saved #{filename}"
               
-              soap.getCsv
+              if File.exists? filename
+                existing_sha1 = Digest::SHA1.file(filename).hexdigest
+                existing_errors = Argos::Soap.new.validate(File.read(filename))
+                if existing_errors.any?
+                  log.error "Validation error for existing data #{filename} (#{File.size(filename)} bytes): #{existing_errors.uniq.to_json}"
+                end 
+                #log.debug "Keeping existing file #{filename} from #{File.mtime(filename)}, fresh data is identical"
+              end
+              
+              new_xml_ng = Nokogiri::XML(soap.xml, nil, nil, Nokogiri::XML::ParseOptions::NOBLANKS | Nokogiri::XML::ParseOptions::NOCDATA | Nokogiri::XML::ParseOptions::STRICT)
+              new_xml = new_xml_ng.canonicalize
+              new_sha1 = Digest::SHA1.hexdigest(new_xml)
               
-              File.open(filename.gsub(/xml$/, "csv"), "wb") { |file| file.write(soap.text)}
+              if existing_sha1.nil? or existing_errors.any? or existing_sha1 != new_sha1
+              
+                if errors = Argos::Soap.new.validate(new_xml_ng).any?
+                  raise "Failed XML schema validation"
+                else
+                  File.open(filename, "wb") { |file| file.write(new_xml)}
+                  log.debug "Validated and saved new data: #{filename}"
+                end
                 
+              end
+              log.debug "Day -#{daysago}: #{date}: #{new_xml_ng.xpath("//message").size} message(s) (program #{programNumber}, platform #{platformId})"
             rescue Argos::NodataException
               # noop
+              log.debug "Day -#{daysago}: #{date}: No data for (program #{programNumber}, platform #{platformId})"
             rescue => e
               log.error e
             end
             
           end
+          
+          log.debug "Completed download of #{platformId}"
+          
         end
+        log.debug "Completed download for #{username}"
       end
     end
     
   end
-end
+end

data/lib/argos/soap.rb

@@ -18,8 +18,7 @@ end
 module Argos
   class Soap
 
-    # A "simple" Soap client for Argos-system satellite tracking webservice
-    # http://wanderingbarque.com/nonintersecting/2006/11/15/the-s-stands-for-simple/   
+    # A [simple](http://wanderingbarque.com/nonintersecting/2006/11/15/the-s-stands-for-simple/) Soap client for the Argos satellite tracking webservice operated by CLS
     
     # client [Savon] (version 3)
     # request [String] Soap:Envelope (XML request body)
@@ -244,7 +243,6 @@ module Argos
       platformListPrograms.each do |program|
         platforms  += program["platform"].map {|p| p["platformId"].to_i}
       end
-      
       platforms
     end
     
@@ -283,6 +281,18 @@ module Argos
       client.services
     end
     
+    def schema
+      Nokogiri::XML::Schema(File.read("#{__dir__}/_xsd/argos-data.xsd"))
+    end
+    
+    def validate(xml)
+      if xml.is_a? String
+        xml = Nokogiri.XML(xml)
+      end
+      schema.validate(xml)
+      
+    end
+    
     # @return [String]
     def text
       @text||=""
@@ -336,6 +346,7 @@ module Argos
       end
       
       # Handle errors
+
       ng = Nokogiri.XML(xml)
       ng.xpath("/data/errors/error").each do | error |
         if error.key?("code")
@@ -351,11 +362,18 @@ module Argos
         end
       end
       
-      # Validation
-      # Validation against XSD does not work: ["Element 'data': No matching global declaration available for the validation root."]
-      # schema = Nokogiri::XML::Schema(File.read("/tmp/argos-data.xsd"))
-      # v = schema.validate(ng)
+      # Validation - only :getXml
+      if [:getXml].include? op_sym
+        # Validation against getXSD schema does not work: ["Element 'data': No matching global declaration available for the validation root."]
+        # See https://github.com/npolar/argos-ruby/commit/219e4b3761e5265f8f9e8b924bcfc23607902428 for the fix
+        schema = Nokogiri::XML::Schema(File.read("#{__dir__}/_xsd/argos-data.xsd"))
+        v = schema.validate(ng)
+        if v.any?
+          log.debug "#{v.size} errors: #{v.map{|v|v.to_s}.uniq.to_json}"
+        end
+      end
       
+
       # Convert XML to Hash
       nori = Nori.new
       nori.parse(xml)

data/lib/argos/soap_command.rb

@@ -117,7 +117,7 @@ module Argos
           end
         end
         
-        opts.on("--download=archive", "Download") do |archive|
+        opts.on("--download=/path/to/archive", "Download") do |archive|
           @param[:archive] = archive
         end
 
@@ -178,7 +178,7 @@ module Argos
         else
           raise ArgumentError, "Unspported operation: #{param[:operation]}"  
         end
-        puts output
+        puts output.to_s
         
       rescue NodataException
         log.debug output
@@ -216,8 +216,7 @@ module Argos
       when :raw
         soap.raw
       when :xml
-        # Return XML with parameter and time in trailing comment
-        soap.xml+xml_trailer
+        soap.xml+"\n"
       else
         raise ArgumentError, "Unknown format: #{format}"
       end
@@ -236,9 +235,9 @@ module Argos
       [:getCsv, :getStreamXml, :getKml, :getXml, :getXsd, :getPlatformList, :getObsCsv, :getObsXml] 
     end
     
-    def xml_trailer
-      "<!-- #{Argos.library_version} #{param.reject {|k,v| k =~ /^password$/}.to_json} now: #{Time.now.utc.xmlschema} -->"
-    end
-    
+    #def xml_trailer
+    #  "<!-- #{Argos.library_version} #{param.reject {|k,v| k =~ /^password$/}.to_json} now: #{Time.now.utc.xmlschema} -->\n"
+    #end
+    #
   end
 end

data/lib/argos.rb

@@ -25,7 +25,7 @@ require_relative "argos/download"
 # 
 # For information about Argos, see: http://www.argos-system.org
 module Argos
-  VERSION = "1.1.3.1"
+  VERSION = "1.2.0"
   
   def self.library_version
     "argos-ruby-#{VERSION}"