rgovdata 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use 1.9.2-p136@rgovdata --create
+

data/CHANGELOG

@@ -0,0 +1,7 @@
+0.1.0 Initial Release
+=====================
+* internal registry - limited set of SG and US resources
+* support for OData, csv, and generic text file sources
+* basic command line client
+* rails-compatible (see rgovdata.com site)
+* ruby script examples

data/Gemfile

@@ -0,0 +1,18 @@
+source 'http://rubygems.org'
+
+gem 'activesupport', '>= 3.0.3'
+gem 'i18n', '>= 0.5.0'
+gem 'ruby_odata', '~> 0.0.10'
+gem 'getoptions', '~> 0.3'
+
+group :development do
+  gem 'bundler', '~> 1.0.0'
+  gem 'jeweler', '~> 1.6.4'
+  gem 'rcov', '>= 0'
+  gem 'rdoc', '~> 3.11'
+end
+
+group :development, :test do
+  gem 'rake', '~> 0.9.2.2'
+  gem 'rspec', '~> 2.7.0', :require => 'spec'
+end

data/Gemfile.lock

@@ -0,0 +1,48 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (3.0.7)
+    diff-lcs (1.1.3)
+    getoptions (0.3)
+    git (1.2.5)
+    i18n (0.6.0)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    json (1.6.1)
+    mime-types (1.17.2)
+    nokogiri (1.5.0)
+    rake (0.9.2.2)
+    rcov (0.9.11)
+    rdoc (3.11)
+      json (~> 1.4)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
+    ruby_odata (0.0.10)
+      activesupport (>= 2.3.5)
+      nokogiri (>= 1.4.2)
+      rest-client (>= 1.5.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (>= 3.0.3)
+  bundler (~> 1.0.0)
+  getoptions (~> 0.3)
+  i18n (>= 0.5.0)
+  jeweler (~> 1.6.4)
+  rake (~> 0.9.2.2)
+  rcov
+  rdoc (~> 3.11)
+  rspec (~> 2.7.0)
+  ruby_odata (~> 0.0.10)

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Paul Gallagher
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,114 @@
+= RGovData
+
+RGovData is a ruby library for really simple access to government data.
+It aims to make consuming government data sets as easy as "gem install rgovdata",
+letting you focus on what you are trying to achieve with the data, and happily
+ignore all the messy underlying details of transport protocols, authentication and so on.
+
+It can be used as a command line tool, a library for ruby projects,
+and/or a library for rails projects.
+The problem it is attempting to solve is
+
+More information is available at
+link:http://rgovdata.com
+along with examples of it's use.
+
+Taking on the challenge of providing access to "all government data" is a somewhat quixotic quest
+for a single individual, but it could work with a community to support it!
+The library is open sourced under an MIT license, and the project hosted on
+{GitHub}[https://github.com/tardate/rgovdata]. If you'd like to get involved, see the
+"Contributing to RGovData" section below.
+
+== Requirements
+
+* ruby 1.9
+* optional: rails 3.0.x
+
+== Objectives and Implementation Status
+
+The following are the broad goals of the RGovData library, along with a simple statement of the
+current implementation status:
+
+* Support for discovery of data sets [status: limited to an internal registry at present.
+  But it has a framework for adding discovery services as and when they are available ]
+* Support all countries that are actively publishing government data sets [status: very limited;
+  currently just supporting some examples from SG and US]
+* Support the range of data formats used in government data sets [status: currently limited to CSV and OData,
+  but with a framework to add more when available ]
+* Command line tool for discovery and access to data [status: basic features available]
+* Provide a common abstraction and simple API for data discovery and access from ruby projects [status: implemented]
+* Provide specific support for Rails ORM technologies (ActiveModel/ActiveRecord/ActiveResource) to
+  make it easy and natural to use in Rails proejcts [status: no specific support yet, however the base ruby API works fine in Rails]
+* Provide a transparent caching mechanism for non-realtime data sets (e.g. csv files) [status: not yet implemented. If you
+  require caching or batch downloading, it is something you currently must implement yourself]
+
+Note that the current version is a very early implementation. It is likely that interfaces
+and capabilities may be refactored or changed in subsequent versions, and not necessarily preserving
+backward compatibility.
+
+
+== Getting Started
+
+For more details and examples, see
+link:http://rgovdata.com
+
+
+=== Installation - Basic Gem and Command Line Usage
+
+Make sure you have a working ruby installation, then simply:
+  $ gem install rgovdata
+
+When the installation is complete, try the command line:
+  $ rgd
+  rgovdata client v0.1.0. Type 'help' for info...
+  rgd://sg>
+
+
+=== Installation - Rails
+
+Add rgovdata to your Gemfile and run bundler:
+
+  $ cat Gemfile
+  ...
+  gem 'rgovdata', '~> 0.1.0'
+  ...
+  $ bundle install
+
+
+== Data Encumberance
+
+Although it's all theoretically "our government data", be aware that many of the data sets you
+can get to with RGovData are encumbered by copyright, commercial or other terms of use (yes, I know: wtf!).
+
+It is up to *you* to ensure that your use of data complies with all the applicable restrictions.
+RGovData simply provides a mechanism for getting the data, and explicitly does not provide any rights
+enforcement or protection.
+
+
+== Contributing to RGovData
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so your changes don't get unintentionally broken in a future version.
+* Please try not to mess with the Rakefile, version, gemspec or changelog. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so the mater repository maintainer can cherry-pick around it.
+
+=== Running Tests
+
+RSpec is used for testing, and it is hooked into rake. Note that integration tests are _not_ run by default.
+
+* rake  - just runs unit tests
+* rake spec - same as rake
+* rake spec:integration - only runs integration tests
+* rake spec:all - run all unit and integration tests
+  
+When you do run integration tests, they make live calls on some real services, some of which require authentication.
+Integration tests will use an rgovdata.conf file in the root of the project for configuration.
+
+
+== Copyright
+
+Copyright (c) 2011 Paul Gallagher and open-sourced under an MIT license.
+See LICENSE for further details.

data/Rakefile

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+require 'rspec'
+require 'rspec/core/rake_task'
+
+$LOAD_PATH.unshift('lib')
+require 'rgovdata/version'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "rgovdata"
+  gem.version = RGovData::Version::STRING
+  gem.homepage = "http://github.com/tardate/rgovdata"
+  gem.license = "MIT"
+  gem.summary = %Q{Really simple access to government data for ruby}
+  gem.description = %Q{Consuming government-published data in a ruby or rails application shouldn't require a PhD}
+  gem.email = "gallagher.paul@gmail.com"
+  gem.authors = ["Paul Gallagher"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+desc "Run only RSpec unit test examples"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ["-c", "-f progress"]
+  t.pattern = 'spec/unit/**/*_spec.rb'
+end
+
+desc "Run only RSpec integration test examples"
+RSpec::Core::RakeTask.new(:'spec:integration') do |t|
+  t.rspec_opts = ["-c", "-f progress"]
+  t.pattern = 'spec/integration/**/*_spec.rb'
+end
+
+desc "Run all RSpec test examples"
+RSpec::Core::RakeTask.new(:'spec:all') do |t|
+  t.rspec_opts = ["-c", "-f progress"]
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  rdoc.main = "README.rdoc"
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "rgovdata #{RGovData::Version::STRING}"
+  rdoc.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+end
+

data/bin/rgd

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'rgovdata'
+require 'getoptions'
+
+begin
+  RGovData::Shell.new(GetOptions.new(RGovData::Shell::OPTIONS)).run
+rescue Exception => e
+  STDERR.puts e
+end

data/examples/all_quakes.rb

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+require 'rgovdata'
+
+dataset = RGovData::Catalog.get('//us/eqs7day-M1/csv')
+puts dataset.attributes.join(',')
+dataset.records.each do |row|
+  puts row
+end

data/examples/arbitrary_data.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+require 'rgovdata'
+
+# This demonstrates how to use RGovData with arbitrary services
+# currently not supported by RGovData::Catalog
+
+# The minimum requirement is a +uri+ and +type+
+# You may need to add a +credentialset+ if authentication is required
+options = {
+  :uri => "http://earthquake.usgs.gov/earthquakes/catalogs/eqs7day-M1.txt",
+  :type => :csv
+}
+
+# Directly create the service endpoint
+service = RGovData::Service.get_instance(options)
+
+puts "Manufactured a service: #{service}"
+puts "With uri: #{service.uri}"
+puts "And type: #{service.type}"
+
+# Now we can use the service to access the dataset (only one in this case since it is a CSV file service)
+dataset = service.datasets.first
+
+# And work with the data:
+puts "This dataset has the following attributes: #{dataset.attributes.join(',')}"
+puts "And it has #{dataset.records.count} rows of data"

data/examples/catalog_traversal.rb

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+require 'rgovdata'
+
+# This demonstrates the essential process of traversing the RGovData catalog
+
+# This gets the root catalog
+root_catalog = RGovData::Catalog.new(nil)
+
+puts "The root catalog has a collection of realms: #{root_catalog.records}"
+# => root_catalog.realms is an alias for root_catalog.records at this level
+
+# Take the first realm catalog
+catalog = root_catalog.records.first
+
+puts "The first catalog is: #{catalog}"
+puts "And it contains the following service listings: #{catalog.records}"
+# => catalog.services is an alias for catalog.records at this level
+
+# Take the first service listing
+service_listing = catalog.records.first
+
+puts "The first service_listing is: #{service_listing}"
+puts "=> it represents the underlying service: #{service_listing.service}"
+
+puts "=> it has the following datasets: #{service_listing.records}"
+# => service_listing.datasets is an alias for service_listing.records at this level
+
+# Take the first dataset
+dataset = service_listing.records.first
+
+puts "The first dataset is: #{dataset}"
+puts "=> it has the following attributes: #{dataset.attributes}"
+puts "=> it has the following number records: #{dataset.records.count}"
+

data/examples/earthquakes.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'rgovdata'
+
+quakes = RGovData::Catalog.get('//us/eqs7day-M1/csv').records.count
+puts "Holy Harp Array Batman, there have been #{quakes} M1+ quakes this week!"

data/lib/rgovdata.rb

@@ -0,0 +1,4 @@
+require 'rgovdata/version'
+require 'rgovdata/config'
+require 'rgovdata/service'
+require 'rgovdata/catalog'

data/lib/rgovdata/catalog.rb

@@ -0,0 +1,4 @@
+require 'rgovdata/catalog/dn'
+require 'rgovdata/catalog/catalog'
+require 'rgovdata/catalog/registry_strategy/registry_strategy'
+require 'rgovdata/catalog/registry_strategy/internal_registry'

data/lib/rgovdata/catalog/catalog.rb

@@ -0,0 +1,79 @@
+class RGovData::Catalog
+  attr_accessor :realm
+  include RGovData::Dn
+
+  class << self
+    # Returns the object specified by the +key+
+    # Key specification:
+    # //<realm>/<service-key>/<data-set-name>
+    # All key components are optional - you will get the best matching object for the key spec
+    # //sg - will return RGovData::Catalog for realm=:sg
+    # //sg/nlb - will return RGovData::ServiceListing for the nlb service in SG
+    # /nlb - will return RGovData::ServiceListing for the nlb service in SG (assuming SG is the default realm)
+    # //sg/nlb/Library - will return RGovData::OdataService for the nlb Library service in SG
+    def get(key)
+      key ||= '//'
+      key.gsub!(':','/') # handle alternate encoding
+      keypart = Regexp.new(/(?:\/\/([^\/]+))?(?:\/([^\/]+))?(?:\/([^\/]+))?/).match(key)
+      found = catalog = self.new(keypart[1])
+      if keypart[2]
+        found = service = catalog.get_service(keypart[2])
+        if keypart[3]
+          found = service.get_dataset(keypart[3])
+        end
+      end
+      found
+    end
+  end
+
+  def initialize(default_realm=nil)
+    @realm = default_realm && default_realm.to_sym
+  end
+
+  # Returns available realms
+  def realms
+    # TODO: currently hard-coded
+    [:sg,:us].map{|realm| self.class.new(realm) }
+  end
+
+  # Returns an array of ServiceListings for the current realm
+  def services
+    @services ||= registry_strategy.load_services
+  end
+
+  # Returns the service(s) matching +key+
+  def get_service(key)
+    return nil unless services && !services.empty?
+    matches = services.select {|s| s.key =~ /#{key}/}
+    matches.count == 1 ? matches.first : matches
+  end
+
+  # override realm setter to clear state when realm changed
+  def realm=(value)
+    clear
+    @realm = value
+  end
+
+  # Returns the registry strategy class for the current realm
+  def registry_strategy
+    RGovData::RegistryStrategy.instance_for_realm(realm)
+  end
+  protected :registry_strategy
+  
+  # Generic interface to return the currently applicable record set
+  # => overrides RGovData::Dn.records
+  def records
+    if realm.present?
+      services
+    else
+      realms
+    end
+  end
+
+  # Clears current state
+  # TODO: move to Dn
+  def clear
+    @realm = @services = nil
+  end
+  protected :clear
+end