datacatalog-importer 0.3.0 → 0.3.1

data/README.md

@@ -69,7 +69,7 @@ Put the main logic / algorithm / voodoo of your importer in the `run` method. Th
       :catalog_name      => "...",
       :catalog_url       => "http://...",
     }
-    
+
 Note that most of these parameters match up with the properties defined for a [Source in the National Data Catalog API](http://github.com/sunlightlabs/datacatalog-api/blob/master/resources/sources.rb). These parameters are just passed along to the API, which will validate the values.
 
 The remaining parameters (`organization` and `downloads`) are handled by the importer framework:
@@ -79,7 +79,7 @@ The remaining parameters (`organization` and `downloads`) are handled by the imp
 * The downloads array is used to lookup or create the associate download formats for a data source.
 
 You may have noticed the use of `Kronos.parse` above. We highly recommend the use of the [kronos library](http://github.com/djsun/kronos) for the parsing of dates.
-    
+
 ### organization parameter
 
 `@handler.organization()` expects a hash parameter of this shape:

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/datacatalog-importer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.3.0"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
-  s.date = %q{2010-08-25}
+  s.date = %q{2010-08-30}
   s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
   s.email = %q{djames@sunlightfoundation.com}
   s.extra_rdoc_files = [

data/example/lib/puller.rb

@@ -45,7 +45,7 @@ class Puller
   def initialize(handler)
     @handler = handler
   end
-  
+
   def run
     common = {
       :catalog_name => "Example Catalog",

data/lib/handler.rb

@@ -4,7 +4,7 @@ module DataCatalog
   module ImporterFramework
     class Handler
       include Shared
-      
+
       def initialize(options)
         @options = options
         @counter = {}
@@ -13,15 +13,15 @@ module DataCatalog
           @counter[resource] = 1
         end
       end
-      
+
       def source(data)
         write_data(:source, data)
       end
-      
+
       def organization(data)
         write_data(:organization, data)
       end
-      
+
       def write_data(resource, data)
         file = folder(resource) + ("/%08i.yml" % @counter[resource])
         Utility.write_yaml(file, data)

data/lib/puller.rb

@@ -3,14 +3,14 @@ module DataCatalog
     class Puller
 
       REQUIRED = %w(cache_folder puller)
-      
+
       def initialize(options)
         REQUIRED.each do |r|
           raise Error, "option :#{r} is required" unless options[r.intern]
         end
         @options = options
       end
-      
+
       def run
         Utility.report_timing "pull" do
           handler = Handler.new(@options)

data/lib/pusher.rb

@@ -7,9 +7,9 @@ module DataCatalog
   module ImporterFramework
     class Pusher
       include Shared
-      
+
       REQUIRED = %w(api_key base_uri cache_folder)
-      
+
       # These keys should not be passed along directly; they need to be
       # examined so that real ID's can be passed along instead.
       LOOKUP_KEYS = [:organization, :downloads]
@@ -30,7 +30,7 @@ module DataCatalog
           push_sources
         end
       end
-      
+
       protected
 
       def setup_api
@@ -44,7 +44,7 @@ module DataCatalog
           create_or_update_organization(data)
         end
       end
-      
+
       def push_sources
         read_data(:source) do |data|
           link_to_existing_organization!(data, :organization_id)
@@ -59,8 +59,6 @@ module DataCatalog
           end
         end
       end
-      
-      # ---
 
       def read_data(resource)
         folder = folder(resource)
@@ -77,60 +75,53 @@ module DataCatalog
           yield data
         end
       end
-      
-      # ---
-      
+
       def create_or_update_organization(data)
         url, name = data[:url], data[:name]
         raise "#{name} has blank URL" if url.blank?
-        docs = DataCatalog::Organization.all(:url => url)
-        n = docs.length
-        if n == 0
-          puts "Creating Organization: #{name}"
+        org, attempts = lookup_organization({ :url => url, :name => name })
+        if org
+          puts "Updating Organization: #{name}"
           begin
-            DataCatalog::Organization.create(data)
+            DataCatalog::Organization.update(org.id, data)
           rescue DataCatalog::BadRequest => e
-            error("Cannot create Organization", {
+            error("Cannot update Organization with id : #{org.id}", {
               :params => data,
               :errors => e.errors,
             })
           end
         else
-          if n > 1
-            warning("Cannot find unique Source with url : #{url}", {
-              :warning    => "#{n} matches: " + docs.map { |x| x.id }.join(" "),
-              :workaround => "Using #{docs[0].id}"
-            })
-          end
-          puts "Updating Organization: #{name}"
+          puts "Creating Organization: #{name}"
           begin
-            DataCatalog::Organization.update(docs[0].id, data)
+            DataCatalog::Organization.create(data)
           rescue DataCatalog::BadRequest => e
-            error("Cannot update Organization with id : #{docs[0].id}", {
+            error("Cannot create Organization", {
               :params => data,
               :errors => e.errors,
             })
           end
         end
       end
-      
-      def find_organization_by(field, name)
-        docs = DataCatalog::Organization.all(field => name)
+
+      def find_organization_by(field, value)
+        docs = DataCatalog::Organization.all(field => value)
         n = docs.length
         if n == 0
           nil
         else
           if n > 1
-            warning("Cannot find unique Organization with #{field} : #{name}", {
-              :warning    => "#{n} matches: " + docs.map { |x| x.id }.join(" "),
-              :workaround => "Using #{docs[0].id}"
-            })
+            id_matches = docs.map { |x| x.id }.join(" ")
+            warning("Cannot find unique Organization with #{field} : " +
+              "#{value}", {
+                :warning    => "#{n} matches: #{id_matches}",
+                :workaround => "Using #{docs[0].id}"
+              })
           end
           docs[0]
         end
       end
-      
-      # Important: do not modify data
+
+      # Important: do not modify the `data` parameter
       def create_or_update_source(data)
         data = clean_source_data(data)
         docs = DataCatalog::Source.all(:url => data[:url])
@@ -163,8 +154,8 @@ module DataCatalog
           })
         end
       end
-      
-      # Important: do not modify data
+
+      # Note: it is important to not modify the 'data' parameter.
       def create_or_update_download(source, data)
         data = data.merge({:source_id => source.id})
         docs = DataCatalog::Download.all({
@@ -194,63 +185,65 @@ module DataCatalog
             })
           end
         else
-          error("Cannot find unique Download with source_id : #{source.id} and format : #{data[:format]}", {
-            :error => "#{n} matches: " + docs.map { |x| x.id }.join(" ")
-          })
+          id_matches = docs.map { |x| x.id }.join(" ")
+          error("Cannot find unique Download with source_id : #{source.id}" +
+            " and format : #{data[:format]}", {
+              :error => "#{n} matches: #{id_matches}"
+            })
         end
       end
-      
-      # Try to link to an existing organization, first by
-      # using an URL, then by name.
+
+      # Try to link to an existing organization, first by using an URL, then
+      # by name.
       #
-      # Note: modifies data (that is why I use the !)
+      # Note: modifies data (hence the ! in the method name)
       def link_to_existing_organization!(data, organization_id_key)
-        hash = data.delete(:organization)
-        raise "Could not find :organization key" unless hash
+        organization_data = data.delete(:organization)
+        raise "Could not find :organization key" unless organization_data
+        org, attempts = lookup_organization(organization_data)
+        if org
+          data[organization_id_key] = org.id
+        else
+          puts "- Could not find organization (to link) with " +
+            attempts.join(' or ')
+        end
+        true # return value not important
+      end
 
+      def lookup_organization(organization_data)
         keys = [:url, :home_url, :name]
-        unless hash.any? { |key, value| keys.include?(key) }
+        unless organization_data.any? { |key, value| keys.include?(key) }
           raise "Need #{keys.join(' or ')} to lookup an organization"
         end
-
         attempts = []
         organization = nil
         keys.each do |key|
-          value = hash[key]
+          value = organization_data[key]
           organization = if value
             attempts << "#{key} : #{value}"
             find_organization_by(key, value)
           end
           break if organization
         end
-
-        if organization
-          data[organization_id_key] = organization.id
-        else
-          puts "- Could not find organization with #{attempts.join(' or ')}"
-        end
-        true # return value not important
+        [organization, attempts]
       end
-      
-      protected
 
-      # Important: do not modify data
+      # Important: do not modify the `data` parameter
       def clean_source_data(data)
         data = data.reject { |k, v| LOOKUP_KEYS.include?(k) }
-        
         frequency = Frequency.new(data[:frequency])
         data[:frequency] = "other" unless frequency.valid?
         data
       end
-      
+
       def error(text, object)
         report("Error", text, object)
       end
-      
+
       def warning(text, object)
         report("Warning", text, object)
       end
-      
+
       def report(type, text, object)
         puts "- #{type} : #{text}"
         puts "  Uploading Report to API."
@@ -263,7 +256,7 @@ module DataCatalog
       rescue DataCatalog::BadRequest => e
         raise Error, "Could not upload Report to API: #{e.errors.inspect}"
       end
-      
+
     end
   end
 end

data/lib/tasks.rb

@@ -1,7 +1,7 @@
 module DataCatalog
   module ImporterFramework
     class Tasks
-  
+
       def initialize(options)
         define(options)
       end

data/lib/utility.rb

@@ -85,7 +85,7 @@ module DataCatalog
           end
         end
       end
-      
+
       def self.remove_fetch_options(options={})
         [:max_attempts, :retry_delay, :quiet].each do |opt|
           options.delete(opt)
@@ -101,9 +101,9 @@ module DataCatalog
         puts "Elapsed time [#{label}] %.2f s" % diff
         result
       end
-      
+
       # == Parsing ===
-      
+
       def self.parse_file(format, file, options={})
         File.open(file) do |f|
           case format
@@ -122,7 +122,7 @@ module DataCatalog
           end
         end
       end
-      
+
       def self.parse_uri(format, uri, options={})
         data = fetch(uri, options)
         case format
@@ -139,7 +139,7 @@ module DataCatalog
           raise "Unexpected format : #{format.inspect}"
         end
       end
-      
+
       def self.parse_file_or_uri(format, file, uri, options={})
         force_fetch = options.delete(:force_fetch) || false
         if force_fetch || !File.exist?(file)

data/natdat_is_hungry.md

@@ -74,7 +74,7 @@ Put the main logic / algorithm / secret recipe / voodoo of your importer in the
       :catalog_name      => "...",
       :catalog_url       => "http://...",
     }
-    
+
 Note that most of these parameters match up with the properties defined for a [Source in the National Data Catalog API](http://github.com/sunlightlabs/datacatalog-api/blob/master/resources/sources.rb). These parameters are just passed along to the API, which will validate the values.
 
 The remaining parameters (`organization` and `downloads`) are handled by the importer framework:
@@ -84,7 +84,7 @@ The remaining parameters (`organization` and `downloads`) are handled by the imp
 * The downloads array is used to lookup or create the associate download formats for a data source.
 
 You may have noticed the use of `Kronos.parse` above. We highly recommend the use of the [kronos library](http://github.com/djsun/kronos) for the parsing of dates.
-    
+
 **organization parameter**
 
 `@handler.organization()` expects a hash parameter of this shape:

data/spec/spec_helper.rb

@@ -6,5 +6,5 @@ require 'spec'
 require 'spec/autorun'
 
 Spec::Runner.configure do |config|
-  
+
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: datacatalog-importer
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: false
   segments: 
   - 0
   - 3
-  - 0
-  version: 0.3.0
+  - 1
+  version: 0.3.1
 platform: ruby
 authors: 
 - David James
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-25 00:00:00 -04:00
+date: 2010-08-30 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency