rboc 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e345c92733e6be5ca2b9f229b2e4992bad5c187
-  data.tar.gz: 3008ae622663bb05437a556aab9b792cfd31a7ed
+  metadata.gz: 05e933b4d3dc2ece95ee01a0928a0f3e0a38ce6a
+  data.tar.gz: bb7a176c5f7e96e9486c65bf38ebb2b3af65a308
 SHA512:
-  metadata.gz: 225a997c45f5cb497c948e5e9c04b53c878e01f5c5a9e671122008aadf96bcf1d3ab7ad374b378a0c044f7c41cd23f28a8d4adc3e1f92c5a7ace53783c1d76db
-  data.tar.gz: 9e8dee4f2cc1deeaf9fdd889e01dd2075dd8aae278cdc1b656596992069ba841ea92afedd2f321bcaee8c108fd1b8202727aea95fedb24df0072de1666165a6b
+  metadata.gz: e6117dbe696ec68046c6471d33d75014e1ed072cd2064de75abf775e15375fc69212c6d0db0a453cd21871551e17828389b384cc851ae01ab9408258f0c32508
+  data.tar.gz: 777e7158a44d4470518ad4d737729c4c9999a9542dd0bee37d02a5c747eca6f12711e8d77d05366d654f67b8984331d56ded9b27b1834fa21792072d9d90ea99

data/lib/rboc.rb

@@ -1,226 +1,135 @@
 require 'curb'
 require 'json'
 require 'uri'
+require 'set'
 
+require 'rboc/census'
 require 'rboc/geo'
 require 'rboc/data'
 
 # A module defining methods for accessing the U.S. Census data API.
 #
+# Census data is divided between a number of files, like the American Community Survey
+# (ACS) 5 year estimates file, the ACS 3 year estimates file, and the 2010 Census
+# summary file. See the {data documentation}[http://www.census.gov/developers/data/] on
+# the Census website for a description of all available files.
+#
+# In +rboc+, the list of available files (using abbreviated names) is contained in
+# +Census::DATA_SETS+.  Each entry in that array corresponds to a class constant in
+# +Census+ assigned to a +Census::DataSet+ instance. A DataSet object contains one or
+# more DataSetVintage objects which represent particular vintage for the given survey.
+# Use the DataSet#vintage_years method to see the vintage years available.
+#
+#     Census::ACS5.vintage_years
+#     # => [2010, 2011, 2012]
+#
+# To access a particular data set vintage, use square brackets.
+#
+#     Census::ACS5[2010].class
+#     # => Census::DataSetVintage
+#
+# To download data, use the +query+ method on a DataSet or DataSetVintage object.
+# Calling #query on the containing DataSet is the same as calling #query on the most
+# recent vintage year. 
+#
+#     Census::ACS5.query(q=Census::Query.new) {|q| ...}
+#     # returns data for most recent vintage year
+#     Census::ACS5[2010].query(q=Census::Query.new) {|q| ...}
+#     # returns data for 2010 vintage year
+#
+# If a block is passed it is called on the Census::Query argument. Queries return
+# Census::ResultSet. For each file there is also a "raw" query method with the same
+# signature:
+#
+#     Census::ACS5.query_raw(q=Census::Query.new) {|q| ...}
+#
+# The raw version returns the unmodified response string, which gives the requested
+# data in JSON format. Note, however, that +#query_raw+ will raise an error if you try to
+# download more than 50 variables (this is a restriction of the Census API).  #query
+# will break your request into chunks and merge them into a single response object.
+#
+# Examples:
+#
+#     # In the following examples I assume the user has installed a key locally, so a
+#     key is not # specified in query parameters.
+#
+#     # Create a query to request the total population for each county in Iowa.
+#     require 'rboc'
+#     my_q = Census::Query.new
+#     my_q.variables = ['B00001_001E'] # this needs to be an array
+#     my_q.geo.summary_level = 'county'
+#     my_q.geo.contained_in = { 'state' => 19 }
+#
+#     # Pass the query to an appropriate Census file, examine the returned column names, and 
+#     # iterate over the results.
+#     result = Census::ACS5.query my_q
+#     result.colnames
+#     # => ["B00001_001E", "state", "county"]
+#     result.each {|row| p row}
+#     # {"B00001_001E" => "1461", "state" => "19", "county" => "001"}
+#     # {"B00001_001E" => "823", "state" => "19", "county" => "003"}
+#     # ...
+#
+#     # You can also iterate over rows without column names
+#     result.rows.each {|row| p row}
+#     # ["1461", "19", "001"]
+#     # ["823", "19", "003"]
+#     # ...
+#
+#     # You can use a block to set query parameters.
+#     result2 = Census::ACS5.query do |q|
+#       q.variables = ['B00001_001E']
+#       q.geo.summary_level = 'county'
+#       q.geo.contained_in = { 'state' => 19 }
+#     end
+#     result2 == result
+#     # => true
+#
+#     # There is a second, chainable syntax for defining query parameters that
+#     # is convenient for one-liners.
+#     result3 = Census::ACS5.query {|q| q.get('B00001_001E').for('county').in('state' => 19)}
+#     result3 = result
+#     # => true
+#
 module Census
 
   # Base URL of the Census data API.
   #
   API_URL = 'http://api.census.gov/data'
 
-  # Path to the installed API key relative to this file.
-  #
-  INSTALLED_KEY_REL_PATH = '../data/installed_key'
-
-  # Path to the installed API key.
+  # Where to store local data
   #
-  INSTALLED_KEY_PATH = File.join(File.dirname(File.expand_path(__FILE__)), INSTALLED_KEY_REL_PATH)
+  LOCAL_DATA_DIR = File.join ENV['HOME'], '.census'
 
-  # Data files accessible through the Census API.
+  # Where cached responses from the Census API. Only data descriptions are stored.
   #
-  FILES = ['acs1', 'acs1_cd', 'acs3', 'acs5', 'sf1', 'sf3']
+  CACHE_DIR = File.join LOCAL_DATA_DIR, 'cache'
 
-  # List valid years of data for each data file.
+  # Path to the installed API key.
   #
-  FILE_VALID_YEARS = {
-    'acs1'    => [2012],
-    'acs1_cd' => [2011],
-    'acs3'    => [2012],
-    'acs5'    => [2011, 2010],
-    'sf1'     => [2010, 2000, 1990],
-    'sf3'     => [2000, 1990]
-  }
-
-  FILE_URL_SUBST = {
-    'acs1' => 'acs1/profile',
-    'acs3' => 'acs3/profile'
-  }
-
-  class CensusApiError < StandardError; end
-  class InvalidQueryError < CensusApiError; end
-  class InvalidKeyError < CensusApiError; end
-  class NoMatchingRecordsError < CensusApiError; end
-  class ServerSideError < CensusApiError; end
+  INSTALLED_KEY_PATH = File.join LOCAL_DATA_DIR, 'installed_key'
 
-  # A class representing a query to the Census API.
+  # Data discoverable API URL
   #
-  class Query
-    attr_accessor :variables, :geo
-
-    def initialize
-      @variables = []
-      @geo = Geography.new
-    end
-
-    def api_key=(key)
-      @api_key = key
-    end
+  DATA_DISCOVERY_URL = 'http://api.census.gov/data.json'
 
-    # Returns the API key to be used for this query. If the key hasn't been set explicitly, this
-    # method attempts to load a key previously installed by Census#install_key!.
-    #
-    def api_key
-      @api_key ||= Census.installed_key
-    end
-
-    # these chainable methods mirror the field names in the HTTP get string
+  self.setup_local_directory!
+  data_sets = JSON.parse self.get_cached_url(DATA_DISCOVERY_URL)
 
-    def get(*vars)
-      @variables = vars
-      self
-    end
-
-    def for(level)
-      @geo.summary_level = level
-      self
-    end
-
-    def in(container)
-      @geo.contained_in = container
-      self
-    end
-
-    def key(key)
-      @api_key = key
-      self
-    end
+  # extract unique file names and valid years
+  data_names = Set.new
+  data_sets.each do |d|
+    name = d['c_dataset'].join('_').upcase
 
-    # Constructs a new Query object with a subset of variables. Creates a shallow copy of this
-    # Query's geography and api key.
-    #
-    def [](rng)
-      variables = @variables[rng]
-      q = Query.new
-      q.variables = variables
-      q.geo = @geo
-      q.api_key = @api_key
-      q
+    if data_names.include? name
+      self.const_get(name).add_vintage DataSetVintage.new(d)
+    else
+      data_names << name
+      ds = DataSet.new
+      ds.add_vintage DataSetVintage.new(d)
+      self.const_set name, ds
     end
-
-    def to_hash
-      h = {}
-      h['key'] = self.api_key
-      h.merge! geo.to_hash
-
-      v = @variables
-      v = v.join(',') if v.is_a? Array
-      h['get'] = v
-
-      h
-    end
-
-    # Returns the query portion of the API GET string.
-    #
-    def to_s
-      URI.encode_www_form self.to_hash
-    end
-  end
-
-  class <<self
-
-    # Writes the given key to a local file. If a key is installed, then you don't have to specify
-    # a key in your query.
-    #
-    def install_key!(key)
-      File.open INSTALLED_KEY_PATH, 'w' do |f|
-        f.write key
-      end
-    end
-
-    def installed_key
-      if File.exists? INSTALLED_KEY_PATH
-        File.read INSTALLED_KEY_PATH
-      else
-        nil
-      end
-    end
-
-    # Constructs the URL needed to perform the query on the given file.
-    #
-    def api_url(year, file, url_file, query)
-      year = year.to_i
-      unless FILE_VALID_YEARS[file].include? year
-        raise ArgumentError, "Invalid year '#{year}' for file '#{file}'"
-      end
-
-      url_file ||= file
-      yield query if block_given?
-      [API_URL, year.to_s,  "#{url_file}?#{query.to_s}"].join('/')
-    end
-
-    # Accesses the data api and returns the unmodified body of the HTTP response.  Raises errors
-    # if the HTTP response code indicates a problem.
-    #
-    def api_raw(year, file, url_file, query)
-      yield query if block_given?
-      url = api_url year, file, url_file, query
-      puts "GET #{url}"
-
-      c = Curl::Easy.new url
-      c.perform
-      r = c.response_code
-
-      if r == 200
-        return c.body_str
-      elsif r == 400
-        raise InvalidQueryError
-      elsif r == 204
-        raise NoMatchingRecordsError
-      elsif r == 500
-        raise ServerSideError
-      elsif r == 302 && (c.head.include?("missing_key") || c.head.include?("invalid_key"))
-        raise InvalidKeyError
-      else
-        raise CensusApiError, "Unexpected HTTP response code: #{r}"
-      end
-    end
-
-    # Accesses the the data api and parses the result into a Census::Data object.
-    #
-    def api_data(year, file, url_file, query)
-      yield query if block_given?
-
-      # download the first 50 or fewer variables
-      json = api_raw year, file, url_file, query[0...50]
-      d = Data.new json
-
-      # download remaining variables 50 at a time
-      offset = 50
-      while offset <= query.variables.length
-        json = api_raw year, file, url_file, query[offset...(offset+50)]
-        json = JSON.parse json
-
-        # sometimes the API returns a descriptive hash (in a single element array) if the
-        # requested columns are invalid
-        raise InvalidQueryError if json.first.is_a? Hash
-
-        d.merge! json
-        offset += 50
-      end
-
-      d
-    end
-
-  end
-
-  def self.api_call(file, url_file)
-
-    define_singleton_method file do |year: FILE_VALID_YEARS[file].first, query: Query.new, &block|
-      api_data year, file, url_file, query, &block
-    end
-
-    define_singleton_method(file+'_raw') do |year: FILE_VALID_YEARS[file].first, query: Query.new, &block|
-      api_raw year, file, url_file, query, &block
-    end
-  end
-
-  FILES.each do |f| 
-    self.api_call f, FILE_URL_SUBST[f]
   end
 
+  DATA_SETS = data_names.sort
 end

data/lib/rboc/census.rb

@@ -0,0 +1,137 @@
+module Census
+  class CensusApiError < StandardError; end
+  class InvalidQueryError < CensusApiError; end
+  class InvalidKeyError < CensusApiError; end
+  class NoMatchingRecordsError < CensusApiError; end
+  class ServerSideError < CensusApiError; end
+
+  # A class representing a query to the Census API.
+  #
+  class Query
+    attr_accessor :variables, :geo
+
+    def initialize
+      @variables = []
+      @geo = Geography.new
+    end
+
+    def api_key=(key)
+      @api_key = key
+    end
+
+    # Returns the API key to be used for this query. If the key hasn't been set explicitly, this
+    # method attempts to load a key previously installed by Census#install_key!.
+    #
+    def api_key
+      @api_key ||= Census.installed_key
+    end
+
+    # these chainable methods mirror the field names in the HTTP get string
+
+    def get(*vars)
+      @variables = vars
+      self
+    end
+
+    def for(level)
+      @geo.summary_level = level
+      self
+    end
+
+    def in(container)
+      @geo.contained_in = container
+      self
+    end
+
+    def key(key)
+      @api_key = key
+      self
+    end
+
+    # Constructs a new Query object with a subset of variables. Creates a shallow copy of this
+    # Query's geography and api key.
+    #
+    def [](rng)
+      variables = @variables[rng]
+      q = Query.new
+      q.variables = variables
+      q.geo = @geo
+      q.api_key = @api_key
+      q
+    end
+
+    def to_hash
+      h = {}
+      h['key'] = self.api_key
+      h.merge! geo.to_hash
+
+      v = @variables
+      v = v.join(',') if v.is_a? Array
+      h['get'] = v
+
+      h
+    end
+
+    # Returns the query portion of the API GET string.
+    #
+    def to_s
+      URI.encode_www_form self.to_hash
+    end
+  end
+
+  class <<self
+
+    # Set up the local directory where cached data and the installed key will be stored.
+    def setup_local_directory!
+      unless Dir.exists? LOCAL_DATA_DIR
+        Dir.mkdir LOCAL_DATA_DIR
+      end
+
+      unless Dir.exists? CACHE_DIR
+        Dir.mkdir CACHE_DIR
+      end
+
+    end
+
+    # Writes the given key to a local file. If a key is installed, then you don't have to specify
+    # a key in your query.
+    #
+    def install_key!(key)
+      File.open INSTALLED_KEY_PATH, 'w' do |f|
+        f.write key
+      end
+    end
+
+    # Returns the currently installed key or +nil+ if no key is installed.
+    #
+    def installed_key
+      if File.exists? INSTALLED_KEY_PATH
+        File.read INSTALLED_KEY_PATH
+      else
+        nil
+      end
+    end
+
+    # Looks for the url basename in the cache directory. If it doesn't exist, then downloads the
+    # file from the web.
+    #
+    def get_cached_url(url)
+      local_file = File.join CACHE_DIR, File.basename(url)
+      if File.exists? local_file
+        File.read local_file
+      else
+        puts "Getting #{url}"
+        c = Curl::Easy.new url
+        c.perform
+        file_content = c.body_str
+        
+        File.open local_file, 'w' do |f|
+          f.write file_content
+        end
+
+        file_content
+      end
+    end
+
+  end
+end