rgovdata 0.1.0

data/lib/rgovdata/data/us/registry.yml

@@ -0,0 +1,12 @@
+--- !ruby/object:RGovData::ServiceListing
+realm: :us
+key: eqs7day-M1
+name: "Worldwide M1+ Earthquakes, Past 7 Days"
+description: "Real-time, worldwide earthquake list for the past 7 days"
+publisher: US Geological Survey
+keywords: ANSS, geologist, plate, real time, environment, catalog, federal data downloadaftershock, intensity scale, magnitude scale, seismogram, mercalli, hypocenter, federal datasets, hazard, geophysics, epicenter, subduction, GSN, quake, magnitude, seismograph, foreshock, seismic, focal mechanism, seismology, tectonics, intensity, Waveforms, tsunami, seismicity, fault, richter, Volcano, temblor, earthquake, aftershock, seismologist, landslide
+license: public
+info_uri: http://explore.data.gov/Geography-and-Environment/Worldwide-M1-Earthquakes-Past-7-Days/7tag-iwnu
+uri: http://earthquake.usgs.gov/earthquakes/catalogs/eqs7day-M1.txt
+type: :csv
+

data/lib/rgovdata/service.rb

@@ -0,0 +1,10 @@
+require 'rgovdata/catalog/dn'
+require 'rgovdata/service/listing'
+require 'rgovdata/service/service'
+require 'rgovdata/service/odata_service'
+require 'rgovdata/service/file_service'
+require 'rgovdata/service/csv_service'
+require 'rgovdata/service/dataset/dataset'
+require 'rgovdata/service/dataset/odata_dataset'
+require 'rgovdata/service/dataset/file_dataset'
+require 'rgovdata/service/dataset/csv_dataset'

data/lib/rgovdata/service/csv_service.rb

@@ -0,0 +1,3 @@
+# This is the catalog class that describes a CSV file-based service
+class RGovData::CsvService < RGovData::FileService
+end

data/lib/rgovdata/service/dataset/csv_dataset.rb

@@ -0,0 +1,43 @@
+require 'csv'
+
+# This is the catalog class that describes a CSV file service DataSet
+class RGovData::CsvDataSet < RGovData::FileDataSet
+
+  # Returns array of attributes that describe the records of the specific entity
+  # => overrides RGovData::Dn.attributes
+  def attributes
+    records unless @attributes # forces a load
+    @attributes
+  end
+
+  # Returns the value of the named +attribute+ from a recordset +row+
+  # => overrides RGovData::DataSet.attribute_value
+  def attribute_value(row,attribute)
+    row[attribute.to_s]
+  end
+
+  # Loads the native dataset (URI or File)
+  # => overrides RGovData::DataSet.load_instance
+  def load_instance
+    if uri =~ /^.+:\/\//
+      URI.parse( uri )
+    else
+      File.new(uri, "r")
+    end
+  end
+  protected :load_instance
+
+  # Loads the native record set
+  # => overrides RGovData::DataSet.load_records
+  def load_records
+    csv = CSV.new(open(native_instance),{:headers=>:first_row}).read
+    @attributes = csv.headers
+    if limit.present?
+      csv.entries[0,limit]
+    else
+      csv.entries
+    end
+  end
+  protected :load_records
+
+end

data/lib/rgovdata/service/dataset/dataset.rb

@@ -0,0 +1,91 @@
+# This is the catalog class that describes a generic Service DataSet
+class RGovData::DataSet
+  include RGovData::Dn
+  attr_reader :options
+  attr_reader :service
+  attr_reader :native_service
+
+  class << self
+    def load_datasets(service)
+      dataset_class = "RGovData::#{service.type.to_s.capitalize}DataSet".constantize
+      ds = []
+      service.dataset_keys.each do |dataset|
+        ds << dataset_class.new({:dataset_key=>dataset},service)
+      end
+      ds
+    end
+  end
+
+  def initialize(options,service)
+    @options = if options.is_a?(Hash)
+      OpenStruct.new(options)
+    else
+      OpenStruct.new
+    end
+    @service = service.dup # avoid circular dependencies
+    @native_service = @service.try(:native_instance)
+  end
+
+  # attribute accessors
+  def realm           ; service.realm               ; end
+  def service_key     ; service.service_key         ; end
+  def dataset_key     ; options.dataset_key         ; end
+  def uri             ; service.uri                 ; end
+  # Returns the record limit currently imposed
+  def limit           ; options.limit               ; end
+  # Set the record limit to +value+
+  def limit=(value)
+    options.limit = value
+  end
+
+  # Returns array of attributes that describe the specific entity
+  # => overrides RGovData::Dn.meta_attributes
+  def meta_attributes
+    [:id,:realm,:service_key,:dataset_key]
+  end
+
+  # Returns the native dataset key
+  alias_method :native_dataset_key, :dataset_key
+
+  # Returns the native dataset instance
+  # If +reload+ is true, it re-initializes
+  def native_instance(reload = false)
+    @native_instance = if reload
+      load_instance
+    else
+      @native_instance || load_instance
+    end
+  end
+
+  # Returns the records
+  # If +reload+ is true, it re-initializes and re-runs the query
+  def records(reload = false)
+    @records = if reload
+      load_records
+    else
+      @records || load_records
+    end
+  end
+
+  # Returns the value of the named +attribute+ from a recordset +row+
+  # Purpose is to encapsulate differences in addressing attribute values
+  def attribute_value(row,attribute)
+    row.send(attribute)
+  end
+  
+  # Loads the native dataset
+  # => override this in specific dataset classes as required
+  def load_instance
+    nil
+  end
+  protected :load_instance
+
+  # Loads the native record set
+  # => override this in specific dataset classes as required
+  def load_records
+    nil
+  end
+  protected :load_records
+
+  
+end

data/lib/rgovdata/service/dataset/file_dataset.rb

@@ -0,0 +1,46 @@
+require 'uri'
+require 'open-uri'
+
+# This is the catalog class that describes a generic file service DataSet
+# Currently only handles text files
+class RGovData::FileDataSet < RGovData::DataSet
+
+  # Returns array of attributes that describe the records of the specific entity
+  # Generic FileDataSets don't have attributes, returns a single selector for the row
+  # => overrides RGovData::Dn.attributes
+  def attributes
+    ['row']
+  end
+
+  # Returns the value of the named +attribute+ from a recordset +row+
+  # Generic FileDataSets don't have attributes, so always return full row
+  # => overrides RGovData::DataSet.attribute_value
+  def attribute_value(row,attribute)
+    row
+  end
+
+  # Loads the native dataset (URI or File)
+  # => overrides RGovData::DataSet.load_instance
+  def load_instance
+    if uri =~ /^.+:\/\//
+      URI.parse( uri )
+    else
+      File.new(uri, "r")
+    end
+  end
+  protected :load_instance
+
+  # Loads the native record set
+  # => overrides RGovData::DataSet.load_records
+  def load_records
+    # open(native_instance,"UserAgent" => "Mozilla/5.0")
+    strio = StringIO.new(open(native_instance).read)
+    if limit.present?
+      strio.to_a[0,limit]
+    else
+      strio
+    end
+  end
+  protected :load_records
+
+end

data/lib/rgovdata/service/dataset/odata_dataset.rb

@@ -0,0 +1,31 @@
+# This is the catalog class that describes an OData Service DataSet
+class RGovData::OdataDataSet < RGovData::DataSet
+  
+  # Returns the attribute names based on class meta-data
+  # => overrides RGovData::Dn.attributes
+  def attributes
+    @attributes ||= native_service.class_metadata[entity_name].keys
+  end
+
+  # Returns the reated OData entity name for this DataSet
+  # TODO: currently, this is a hack, as ruby_odata doesn't yet return the collection EntityType
+  def entity_name
+    dataset_key.gsub(/Set$/,'')
+  end
+
+  # Loads the native OData::QueryBuilder
+  # => overrides RGovData::DataSet.load_instance
+  def load_instance
+    native_service.send(native_dataset_key)
+  end
+  protected :load_instance
+
+  # Loads the native record set
+  # => overrides RGovData::DataSet.load_records
+  def load_records
+    native_instance(true)
+    native_instance.top(limit) if limit.present?
+    Array(service.native_instance.execute)
+  end
+  protected :load_records
+end

data/lib/rgovdata/service/file_service.rb

@@ -0,0 +1,10 @@
+# This is the catalog class that describes a generic file-based service
+class RGovData::FileService < RGovData::Service
+
+  # Returns an array of DataSets (keys) for the service
+  # => overrides RGovData::Service.dataset_keys
+  def dataset_keys
+    [type]
+  end
+
+end

data/lib/rgovdata/service/listing.rb

@@ -0,0 +1,47 @@
+# A ServiceListing is the metadata describing a specific service
+# It encapsulates access to the underlying service
+class RGovData::ServiceListing
+  attr_accessor :realm         # realm for the service
+  attr_accessor :key           # unique service name or id (within realm)
+  attr_accessor :name          # human name of the service
+  attr_accessor :description   # human description of the service
+  attr_accessor :keywords      # keywords for the service
+  attr_accessor :publisher     # service publisher name
+  attr_accessor :license       # license covering the service if any
+  attr_accessor :info_uri      # url to a web page about the service if any
+  attr_accessor :uri           # url to the service interface
+  attr_accessor :type          # service type [:odata,:csv,:file]
+  attr_accessor :credentialset # name of the credential set required
+
+  include RGovData::Dn
+
+  # Returns the service for this listing
+  def service
+    @service ||= RGovData::Service.get_instance(self)
+  end
+
+  # Returns the service key
+  alias_method :service_key, :key
+
+  # Returns an array of DataSets for the service
+  # => delegate to service
+  def datasets
+    service.try(:datasets)
+  end
+  alias_method :records, :datasets
+
+  # Returns the dataset(s) matching +key+
+  # => delegate to service
+  def get_dataset(key)
+    service.try(:get_dataset,key)
+  end
+  # Returns the first dataset matching +key+
+  # => delegate to service
+  def find(id)
+    service.try(:find,id)
+  end
+  # Alias for find
+  alias_method :find_by_id, :find
+  
+
+end

data/lib/rgovdata/service/odata_service.rb

@@ -0,0 +1,50 @@
+require 'ruby_odata'
+
+# This is the catalog class that describes an OData Service
+class RGovData::OdataService < RGovData::Service
+
+  # Returns an array of DataSets (keys) for the service
+  def dataset_keys
+    # @dataset_keys ||= native_instance.classes.keys
+    @dataset_keys ||= native_instance.instance_variable_get(:@collections)
+  end
+
+  # Returns the native service object if applicable
+  # By default, returns self
+  def native_instance
+    @native_instance ||= load_service
+  end
+
+  # Identifies, loads, and returns the native service instance
+  def load_service
+    clear
+    # currently forcing SSL verification off (seems to be required for projectnimbus)
+    # TODO: this should probably be a setting in the ServiceListing
+    rest_options = {:verify_ssl=>false}
+    if credentialset && credentialset != 'projectnimbus'
+      credentials = config.credentialsets[credentialset]
+      # merge basic auth
+      rest_options.merge!({ :username => credentials['username'], :password => credentials['password'] })
+    end
+    svc = OData::Service.new(uri, rest_options)
+    if credentialset && credentialset == 'projectnimbus'
+      credentials = config.credentialsets[credentialset]
+      # some special funk to insert headers for projectnimbus authentication
+      actual_rest_options = svc.instance_variable_get(:@rest_options)
+      rest_options = actual_rest_options.merge({:headers => {
+        'AccountKey' => credentials['AccountKey'], 'UniqueUserID' => credentials['UniqueUserID']
+      }})
+      svc.instance_variable_set(:@rest_options,rest_options)
+    end
+    svc
+  end
+  protected :load_service
+
+  # Clears current state
+  # TODO: move to Dn?
+  def clear
+    @datasets = @dataset_keys = @native_instance = nil
+  end
+  protected :clear
+end
+

data/lib/rgovdata/service/service.rb

@@ -0,0 +1,93 @@
+require 'ostruct'
+
+# A Service describes a specific service
+# It encapsulates access to the underlying service implementation
+class RGovData::Service
+  include RGovData::CommonConfig
+  include RGovData::Dn
+  
+  attr_accessor :options
+  attr_reader :native_instance    # the underlying native service object (if applicable)
+
+  class << self
+    # Returns the appropriate Service class for the given uri and type
+    # +options may be a RGovData::ServiceListing or a Hash
+    # If +options+ is a hash, it requires the following members:
+    # +uri+
+    # +type+
+    # +credentialset+
+    def get_instance(options={})
+      type = (options.class <= RGovData::ServiceListing) ? options.type : options[:type]
+      service_class = "RGovData::#{type.to_s.capitalize}Service".constantize
+      service_class.new(options)
+    rescue # invalid or not a supported type
+      nil
+    end
+  end
+
+  # +new+ requires
+  # +options may be a RGovData::ServiceListing or a Hash
+  # If +options+ is a hash, it requires the following members:
+  # +uri+
+  # +type+
+  # +transport+
+  # +credentialset+
+  def initialize(options)
+    @options = if options.is_a?(Hash)
+      OpenStruct.new(options)
+    elsif options.class <= RGovData::ServiceListing
+      options.dup # avoid circular refs
+    else
+      OpenStruct.new
+    end
+  end
+
+  # attribute accessors
+  def realm         ; options.realm         ; end
+  def service_key   ; options.service_key   ; end
+  def uri           ; options.uri           ; end
+  def type          ; options.type          ; end
+  def transport     ; options.transport     ; end
+  def credentialset ; options.credentialset ; end
+
+  # Returns array of attributes that describe the specific entity
+  # => overrides RGovData::Dn.meta_attributes
+  def meta_attributes
+    [:id,:realm,:service_key,:uri,:type,:transport,:credentialset]
+  end
+
+  # Returns the native service object if applicable
+  # By default, returns self
+  def native_instance
+    @native_instance || self
+  end
+  
+  # Returns an array of DataSets for the service
+  # => may need to be overridden for a specific service type
+  def datasets
+    dataset_class = "RGovData::#{type.to_s.capitalize}DataSet".constantize
+    @datasets ||= dataset_class.load_datasets(self)
+  rescue
+    []
+  end
+
+  # Returns an array of DataSets (keys) for the service
+  # => needs to be overridden for each service type
+  def dataset_keys
+    []
+  end
+
+  # Returns the dataset(s) matching +key+
+  def get_dataset(key)
+    return nil unless datasets && !datasets.empty?
+    matches = datasets.select {|s| s.dataset_key =~ /#{key}/}
+    matches.count == 1 ? matches.first : matches
+  end
+  # Returns the first dataset matching +key+
+  def find(id)
+    Array(get_dataset(id)).first
+  end
+  # Alias for find
+  alias_method :find_by_id, :find
+
+end