datacatalog-importer 0.1.19 → 0.2.0

data/.gitignore

@@ -17,5 +17,7 @@ tmtags
 coverage
 rdoc
 pkg
+example/cache
+example/config.yml
 
 ## PROJECT::SPECIFIC

data/Rakefile

@@ -8,10 +8,10 @@ begin
     gem.summary = %Q{A framework to write National Data Catalog importers}
     gem.description = %Q{This framework makes it easier to write importers for the National Data Catalog.}
     gem.email = "djames@sunlightfoundation.com"
-    gem.homepage = "http://github.com/djsun/datacatalog-importer"
+    gem.homepage = "http://github.com/sunlightlabs/datacatalog-importer"
     gem.authors = ["David James"]
-    gem.add_dependency "nokogiri", ">= 1.4.1"
-    gem.add_dependency "datacatalog", ">= 0.4.14"
+    gem.add_dependency "nokogiri", ">= 1.4.2"
+    gem.add_dependency "datacatalog", ">= 0.4.15"
     gem.add_development_dependency "rspec", ">= 1.2.9"
     # gem is a Gem::Specification...
     # see http://www.rubygems.org/read/chapter/20 for additional settings

data/VERSION

@@ -1 +1 @@
-0.1.19
+0.2.0

data/datacatalog-importer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.1.19"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
-  s.date = %q{2010-05-12}
+  s.date = %q{2010-07-08}
   s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
   s.email = %q{djames@sunlightfoundation.com}
   s.extra_rdoc_files = [
@@ -24,7 +24,12 @@ Gem::Specification.new do |s|
      "Rakefile",
      "VERSION",
      "datacatalog-importer.gemspec",
+     "example/README.md",
+     "example/config.example.yml",
+     "example/lib/puller.rb",
+     "example/rakefile.rb",
      "lib/datacatalog-importer.rb",
+     "lib/handler.rb",
      "lib/importer.rb",
      "lib/puller.rb",
      "lib/pusher.rb",
@@ -36,10 +41,10 @@ Gem::Specification.new do |s|
      "spec/spec_helper.rb",
      "spec/utility_spec.rb"
   ]
-  s.homepage = %q{http://github.com/djsun/datacatalog-importer}
+  s.homepage = %q{http://github.com/sunlightlabs/datacatalog-importer}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.3.7}
   s.summary = %q{A framework to write National Data Catalog importers}
   s.test_files = [
     "spec/spec_helper.rb",
@@ -50,18 +55,18 @@ Gem::Specification.new do |s|
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.1"])
-      s.add_runtime_dependency(%q<datacatalog>, [">= 0.4.14"])
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.2"])
+      s.add_runtime_dependency(%q<datacatalog>, [">= 0.4.15"])
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
     else
-      s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
-      s.add_dependency(%q<datacatalog>, [">= 0.4.14"])
+      s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
+      s.add_dependency(%q<datacatalog>, [">= 0.4.15"])
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
     end
   else
-    s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
-    s.add_dependency(%q<datacatalog>, [">= 0.4.14"])
+    s.add_dependency(%q<nokogiri>, [">= 1.4.2"])
+    s.add_dependency(%q<datacatalog>, [">= 0.4.15"])
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
   end
 end

data/example/README.md

@@ -0,0 +1,5 @@
+# README
+
+This is an example of how to write an importer that depends on the National Data Catalog importer framework.
+
+Please note that it is a bare bones example. It simply demonstrates how to setup your rakefile and just the basic moving parts.

data/example/config.example.yml

@@ -0,0 +1,3 @@
+local:
+  api_key: "0000000000000000000000000000000000000000"
+  base_uri: "http://localhost:3000"

data/example/lib/puller.rb

@@ -0,0 +1,71 @@
+class Puller
+
+  ORGS = [
+    {
+      :name        => "Budget Office",
+      :url         => "http://example.gov/orgs/budget-office",
+      :description => "Prepares the executive budget..."
+    },
+    {
+      :name        => "Environmental Agency",
+      :url         => "http://example.gov/orgs/environmental-agency",
+      :description => "Tracks environmental compliance..."
+    },
+    {
+      :name        => "Inspector General",
+      :url         => "http://example.gov/orgs/inspector-general",
+      :description => "Inspects..."
+    }
+  ]
+
+  SOURCES = [
+    {
+      :title             => "School District Performance",
+      :url               => "http://example.gov/data-sets/209",
+      :description       => "Comparative school performance...",
+      :frequency         => "annual",
+      :source_type       => "dataset",
+    },
+    {
+      :title             => "Economic Development",
+      :url               => "http://example.gov/data-sets/210",
+      :description       => "Economic indicators for...",
+      :frequency         => "monthly",
+      :source_type       => "dataset",
+    },
+    {
+      :title             => "Superfund Projects",
+      :url               => "http://example.gov/apis/5",
+      :description       => "API for environmental cleanup...",
+      :frequency         => "monthly",
+      :source_type       => "api",
+    },
+  ]
+
+  def initialize(handler)
+    @handler = handler
+  end
+  
+  def run
+    common = {
+      :catalog_name => "Example Catalog",
+      :catalog_url  => "http://example.gov",
+    }
+    ORGS.each do |o|
+      @handler.organization(
+        o.merge(common).merge({
+          :org_type     => "governmental",
+        })
+      )
+    end
+    SOURCES.each do |s|
+      @handler.source(
+        s.merge(common).merge({
+          :license      => "public domain",
+          :license_url  => "http://example.gov/license",
+        })
+      )
+    end
+  end
+
+end

data/example/rakefile.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'yaml'
+require File.dirname(__FILE__) + '/../lib/datacatalog-importer'
+require File.dirname(__FILE__) + '/lib/puller'
+
+def setup
+  config_file = File.dirname(__FILE__) + '/config.yml'
+  config = YAML.load_file(config_file)
+  env = ENV['IMPORTER_ENV']
+  raise "IMPORTER_ENV undefined" unless env
+  raise "IMPORTER_ENV invalid" unless config[env]
+  DataCatalog::ImporterFramework::Tasks.new({
+    :api_key      => config[env]['api_key'],
+    :base_uri     => config[env]['base_uri'],
+    :cache_folder => File.dirname(__FILE__) + '/cache/parsed',
+    :name         => "Example Catalog",
+    :uri          => "http://example.datacatalog.gov",
+    :puller       => Puller,
+  })
+end
+
+setup

data/lib/datacatalog-importer.rb

@@ -6,6 +6,7 @@
 # strange conflicts.
 require 'generator.rb'
 
+require File.dirname(__FILE__) + '/handler'
 require File.dirname(__FILE__) + '/importer'
 require File.dirname(__FILE__) + '/puller'
 require File.dirname(__FILE__) + '/pusher'

data/lib/handler.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/shared'
+
+module DataCatalog
+  module ImporterFramework
+    class Handler
+      include Shared
+      
+      def initialize(options)
+        @options = options
+        @counter = {}
+        [:source, :organization].each do |resource|
+          FileUtils.mkdir_p(folder(resource))
+          @counter[resource] = 1
+        end
+      end
+      
+      def source(data)
+        write_data(:source, data)
+      end
+      
+      def organization(data)
+        write_data(:organization, data)
+      end
+      
+      def write_data(resource, data)
+        file = folder(resource) + ("/%08i.yml" % @counter[resource])
+        Utility.write_yaml(file, data)
+        @counter[resource] += 1
+      end
+
+    end
+  end
+end

data/lib/puller.rb

@@ -1,64 +1,23 @@
-require File.dirname(__FILE__) + '/shared'
-
 module DataCatalog
   module ImporterFramework
     class Puller
-      include Shared
 
-      REQUIRED = %w(cache_folder pullers)
+      REQUIRED = %w(cache_folder puller)
       
       def initialize(options)
         REQUIRED.each do |r|
           raise Error, "option :#{r} is required" unless options[r.intern]
         end
         @options = options
-        @counter = {
-          :source       => 1,
-          :organization => 1,
-        }
       end
       
       def run
-        Utility.report_timing "pull source" do
-          pull_resource(:source)
-        end
-        Utility.report_timing "pull organization" do
-          pull_resource(:organization)
+        Utility.report_timing "pull" do
+          handler = Handler.new(@options)
+          puller = @options[:puller].new(handler)
+          puller.run
         end
       end
-      
-      protected
-
-      # Note on HTTP Throttling
-      #
-      # It might make sense to throttle HTTP calls in
-      # * pull_organizations
-      # * pull_sources
-      #
-      # However, doing a simple sleep(TIME_DELAY) is too blunt.
-      # It makes sense when an HTTP call is made; however, it does
-      # not make sense when the importer uses a local cache.
-      #
-      # An alternative is to wrap HTTP calls in this Importer library.
-      # It could add a little bit of delay to HTTP calls that are made
-      # too rapidly.
-      #
-      def pull_resource(resource)
-        unless importer_class = @options[:pullers][resource]
-          raise Error, "options[:pullers][:#{resource}] is required"
-        end
-        importer = importer_class.new
-        FileUtils.mkdir_p(folder(resource))
-        while (data = importer.fetch) do
-          write_data(resource, data)
-        end
-      end
-      
-      def write_data(resource, data)
-        file = folder(resource) + ("/%08i.yml" % @counter[resource])
-        Utility.write_yaml(file, data)
-        @counter[resource] += 1
-      end
 
     end
   end

data/lib/shared.rb

@@ -1,6 +1,7 @@
 module DataCatalog
   module ImporterFramework
     module Shared
+
       def folder(resource)
         unless @options
           raise Error, "@options is undefined"
@@ -10,6 +11,7 @@ module DataCatalog
         end
         File.join(@options[:cache_folder], resource.to_s)
       end
+
     end
   end
 end

data/lib/tasks.rb

@@ -14,7 +14,7 @@ module DataCatalog
           puts "Pulling data from the #{options[:name]}..."
           puller = Puller.new({
             :cache_folder => options[:cache_folder],
-            :pullers      => options[:pullers],
+            :puller       => options[:puller],
           })
           puller.run
         end

data/lib/utility.rb

@@ -56,7 +56,7 @@ module DataCatalog
 
       def self.headers
         {
-          "UserAgent" => "National Data Catalog Importer/0.1.19",
+          "UserAgent" => "National Data Catalog Importer/0.2.0",
         }
       end
 

data/spec/spec_helper.rb

@@ -1,5 +1,6 @@
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rubygems'
 require 'datacatalog-importer'
 require 'spec'
 require 'spec/autorun'

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: datacatalog-importer
 version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 19
-  version: 0.1.19
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - David James
@@ -14,44 +15,50 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-12 00:00:00 -04:00
+date: 2010-07-08 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 1
         - 4
-        - 1
-        version: 1.4.1
+        - 2
+        version: 1.4.2
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: datacatalog
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 17
         segments: 
         - 0
         - 4
-        - 14
-        version: 0.4.14
+        - 15
+        version: 0.4.15
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 13
         segments: 
         - 1
         - 2
@@ -76,7 +83,12 @@ files:
 - Rakefile
 - VERSION
 - datacatalog-importer.gemspec
+- example/README.md
+- example/config.example.yml
+- example/lib/puller.rb
+- example/rakefile.rb
 - lib/datacatalog-importer.rb
+- lib/handler.rb
 - lib/importer.rb
 - lib/puller.rb
 - lib/pusher.rb
@@ -88,7 +100,7 @@ files:
 - spec/spec_helper.rb
 - spec/utility_spec.rb
 has_rdoc: true
-homepage: http://github.com/djsun/datacatalog-importer
+homepage: http://github.com/sunlightlabs/datacatalog-importer
 licenses: []
 
 post_install_message: 
@@ -97,23 +109,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: A framework to write National Data Catalog importers