datacatalog-importer 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Sunlight Labs
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "datacatalog-importer"
+    gem.summary = %Q{A framework to write National Data Catalog importers}
+    gem.description = %Q{This framework makes it easier to write importers for the National Data Catalog.}
+    gem.email = "djames@sunlightfoundation.com"
+    gem.homepage = "http://github.com/djsun/datacatalog-importer"
+    gem.authors = ["David James"]
+    gem.add_dependency "nokogiri", ">= 1.4.1"
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    # gem is a Gem::Specification...
+    # see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+# require 'rake/rdoctask'
+# Rake::RDocTask.new do |rdoc|
+#   version = File.exist?('VERSION') ? File.read('VERSION') : ""
+# 
+#   rdoc.rdoc_dir = 'rdoc'
+#   rdoc.title = "datacatalog-importer #{version}"
+#   rdoc.rdoc_files.include('README*')
+#   rdoc.rdoc_files.include('lib/**/*.rb')
+# end
+# 
+# require 'spec/rake/spectask'
+# Spec::Rake::SpecTask.new(:spec) do |spec|
+#   spec.libs << 'lib' << 'spec'
+#   spec.spec_files = FileList['spec/**/*_spec.rb']
+# end
+# 
+# Spec::Rake::SpecTask.new(:rcov) do |spec|
+#   spec.libs << 'lib' << 'spec'
+#   spec.pattern = 'spec/**/*_spec.rb'
+#   spec.rcov = true
+# end
+
+task :spec => :check_dependencies
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/datacatalog-importer.gemspec

@@ -0,0 +1,65 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{datacatalog-importer}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["David James"]
+  s.date = %q{2010-02-05}
+  s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
+  s.email = %q{djames@sunlightfoundation.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.md"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "datacatalog-importer.gemspec",
+     "lib/datacatalog-importer.rb",
+     "lib/importer.rb",
+     "lib/puller.rb",
+     "lib/pusher.rb",
+     "lib/shared.rb",
+     "lib/sort_yaml_hash.rb",
+     "lib/tasks.rb",
+     "lib/utility.rb",
+     "spec/datacatalog-importer_spec.rb",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/djsun/datacatalog-importer}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{A framework to write National Data Catalog importers}
+  s.test_files = [
+    "spec/datacatalog-importer_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.1"])
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+  end
+end
+

data/lib/datacatalog-importer.rb

@@ -0,0 +1,14 @@
+# The following line is a workaround.
+# See https://rails.lighthouseapp.com/projects/8994/tickets/2942-enumerableenumeratornext-causes-stack-level-too-deep-since-activesupport-233
+# 
+# It allows Ruby 1.8.7's Enumerable#each to work properly. Otherwise,
+# the 'active_support' gem loads the 'json' gem and there will be
+# strange conflicts.
+require 'generator.rb'
+
+require File.dirname(__FILE__) + '/importer'
+require File.dirname(__FILE__) + '/puller'
+require File.dirname(__FILE__) + '/pusher'
+require File.dirname(__FILE__) + '/sort_yaml_hash'
+require File.dirname(__FILE__) + '/tasks'
+require File.dirname(__FILE__) + '/utility'

data/lib/importer.rb

@@ -0,0 +1,7 @@
+module DataCatalog
+  module Importer
+
+    class Error < RuntimeError; end
+
+  end
+end

data/lib/puller.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + '/shared'
+
+module DataCatalog
+  module Importer
+    class Puller
+      include DataCatalog::Importer::Shared
+
+      REQUIRED = %w(cache_folder pullers)
+      
+      def initialize(options)
+        REQUIRED.each do |r|
+          raise Error, "option :#{r} is required" unless options[r.intern]
+        end
+        @options = options
+        @counter = {
+          :source       => 1,
+          :organization => 1,
+        }
+      end
+      
+      def run
+        Utility.report_timing "pull source" do
+          pull_resource(:source)
+        end
+        Utility.report_timing "pull organization" do
+          pull_resource(:organization)
+        end
+      end
+      
+      protected
+
+      # Note on HTTP Throttling
+      #
+      # It might make sense to throttle HTTP calls in
+      # * pull_organizations
+      # * pull_sources
+      #
+      # However, doing a simple sleep(TIME_DELAY) is too blunt.
+      # It makes sense when an HTTP call is made; however, it does
+      # not make sense when the importer uses a local cache.
+      #
+      # An alternative is to wrap HTTP calls in this Importer library.
+      # It could add a little bit of delay to HTTP calls that are made
+      # too rapidly.
+      #
+      def pull_resource(resource)
+        unless importer_class = @options[:pullers][resource]
+          raise Error, "options[:pullers][:#{r}] is required"
+        end
+        importer = importer_class.new
+        FileUtils.mkdir_p(folder(resource))
+        while (data = importer.fetch) do
+          write_data(resource, data)
+        end
+      end
+      
+      def write_data(resource, data)
+        file = folder(resource) + ("/%08i.yml" % @counter[resource])
+        Utility.write_yaml(file, data)
+        @counter[resource] += 1
+      end
+
+    end
+  end
+end

data/lib/pusher.rb

@@ -0,0 +1,138 @@
+require 'datacatalog'
+require File.dirname(__FILE__) + '/shared'
+
+module DataCatalog
+  module Importer
+    class Pusher
+      include DataCatalog::Importer::Shared
+      
+      REQUIRED = %w(api_key base_uri cache_folder)
+
+      def initialize(options)
+        REQUIRED.each do |r|
+          raise Error, "option :#{r} is required" unless options[r.intern]
+        end
+        @options = options
+      end
+
+      def run
+        setup_api
+        Utility.report_timing "push organizations" do
+          push_organizations
+        end
+        Utility.report_timing "push sources" do
+          push_sources
+        end
+      end
+      
+      protected
+
+      def setup_api
+        DataCatalog.api_key  = @options[:api_key]
+        DataCatalog.base_uri = @options[:base_uri]
+      end
+
+      def push_organizations
+        read_data(:organization) do |data|
+          create_or_update_organization(data)
+        end
+      end
+      
+      def push_sources
+        read_data(:source) do |data|
+          source = create_or_update_source(data)
+          data[:downloads].each do |download_data|
+            create_or_update_download(source, download_data)
+          end
+        end
+      end
+      
+      # ---
+
+      def read_data(resource)
+        folder = folder(resource)
+        unless File.exist?(folder)
+          raise "Directory does not exist: #{folder}"
+        end
+        wildcard = folder + "/*.yml"
+        files = Dir.glob(wildcard)
+        if files.empty?
+          raise "No files found with: #{wildcard}"
+        end
+        files.sort.each do |f|
+          data = YAML::load_file(f)
+          yield data
+        end
+      end
+      
+      # ---
+      
+      def create_or_update_organization(data)
+        docs = DataCatalog::Organization.all(:url => data[:url])
+        n = docs.length
+        case n
+        when 0
+          puts "Creating Organization: #{data[:name]}"
+          DataCatalog::Organization.create(data)
+        when 1
+          puts "Updating Organization: #{data[:name]}"
+          DataCatalog::Organization.update(docs[0].id, data)
+        else
+          multiple_matches("Organization", n, data)
+        end
+      end
+      
+      # Important: do not modify data
+      def create_or_update_source(data)
+        data = data.reject do |key, value|
+          [:organization, :downloads].include?(key)
+        end
+        docs = DataCatalog::Source.all(:url => data[:url])
+        n = docs.length
+        case n
+        when 0
+          puts "Creating Source: #{data[:title]}"
+          DataCatalog::Source.create(data)
+        when 1
+          puts "Updating Source: #{data[:title]}"
+          DataCatalog::Source.update(docs[0].id, data)
+        else
+          multiple_matches("Source", n, data)
+        end
+      end
+      
+      # Important: do not modify data
+      def create_or_update_download(source, data)
+        data = data.merge({:source_id => source.id})
+        docs = DataCatalog::Download.all({
+          :source_id => source.id,
+          :format    => data[:format],
+        })
+        n = docs.length
+        case n
+        when 0
+          puts "- Creating Download: #{data[:format]}"
+          DataCatalog::Download.create(data)
+        when 1
+          puts "- Updating Download: #{data[:format]}"
+          DataCatalog::Download.update(docs[0].id, data)
+        else
+          multiple_matches("Download", n, data)
+        end
+      end
+      
+      # ---
+      
+      def multiple_matches(model, n, data)
+        puts "? : #{n} matches for #{model}"
+        DataCatalog::Report.create({
+          :status => "new",
+          :text   => "Cannot automatically update #{model}; there " +
+                     "are #{n} matches for url : #{data[:url]}",
+          :object => data.inspect,
+        })
+      end
+      
+    end
+  end
+end

data/lib/shared.rb

@@ -0,0 +1,17 @@
+module DataCatalog
+  module Importer
+    module Shared
+
+      def folder(resource)
+        unless @options
+          raise Error, "@options is undefined"
+        end
+        unless cache_folder = @options[:cache_folder]
+          raise Error, "option :cache_folder is required"
+        end
+        File.join(@options[:cache_folder], resource.to_s)
+      end
+      
+    end
+  end
+end

data/lib/sort_yaml_hash.rb

@@ -0,0 +1,22 @@
+require 'yaml'
+
+class Hash
+
+  def to_yaml(opts = {})
+    YAML::quick_emit(object_id, opts) do |out|
+      out.map(taguri, to_yaml_style) do |map|
+        sorted = self.sort { |a, b| a[0] <=> b[0] }
+        sorted.each { |k, v| map.add(k, v) }
+      end
+    end
+  end
+
+end
+
+class Symbol
+  
+  def <=>(other)
+    self.to_s <=> other.to_s
+  end
+  
+end

data/lib/tasks.rb

@@ -0,0 +1,36 @@
+module DataCatalog
+  module Importer
+    class Tasks
+  
+      def initialize(options)
+        define(options)
+      end
+
+      protected
+
+      def define(options)
+        desc "Pull data from the #{options[:name]}"
+        task :pull do
+          puts "Pulling data from the #{options[:name]}..."
+          puller = DataCatalog::Importer::Puller.new({
+            :cache_folder => options[:cache_folder],
+            :pullers      => options[:pullers],
+          })
+          puller.run
+        end
+
+        desc "Push data to the Data Catalog API"
+        task :push do
+          desc "Pushing data to the Data Catalog API..."
+          pusher = DataCatalog::Importer::Pusher.new({
+            :api_key      => options[:api_key],
+            :base_uri     => options[:base_uri],
+            :cache_folder => options[:cache_folder],
+          })
+          pusher.run
+        end
+      end
+
+    end
+  end
+end

data/lib/utility.rb

@@ -0,0 +1,128 @@
+require 'nokogiri'
+require 'open-uri'
+
+module DataCatalog
+
+  class Utility
+    
+    def self.absolute_url(page_url, url)
+      Utility.plain_string(URI.parse(page_url).merge(url).to_s)
+    end
+
+    def self.single_line_clean(s)
+      plain_string(
+        s.gsub(/[\r\n\t]/, " ").gsub(/[\x80-\xFF]/, "").squeeze(" ").strip)
+    end
+
+    def self.multi_line_clean(s)
+      plain_string(
+        s.gsub(/[\x80-\xFF]/, "").squeeze(" ").strip)
+    end
+
+    def self.fetch(uri)
+      puts "Fetching #{uri}..."
+      io = open(uri, headers)
+      io.read
+    end
+
+    def self.headers
+      {
+        "UserAgent" => "National Data Catalog Importer/0.1.0",
+      }
+    end
+
+    def self.parse_file(filename)
+      File.open(filename) do |f|
+        Nokogiri::HTML::Document.parse(f)
+      end
+    end
+    
+    def self.parse_file_or_uri(uri, file, options={})
+      if options[:force_fetch] || !File.exist?(file)
+        document = parse_uri(uri)
+        File.open(file, "w") { |f| f.write(document) }
+      end
+      parse_file(file) # Why always parse the file? See Note 001, below.
+    end
+
+    def self.parse_uri(uri)
+      puts "Fetching #{uri}..."
+      open(uri, headers) do |io|
+        Nokogiri::HTML::Document.parse(io)
+      end
+    end
+
+    # ActiveSupport 2.3.5 adds @_rails_html_safe aggressively.
+    # This method removes it so you can output clean YAML.
+    def self.plain_string(s)
+      if s.instance_variable_defined?(:@_rails_html_safe)
+        s.send(:remove_instance_variable, :@_rails_html_safe)
+      end
+      s
+    end
+
+    def self.report_timing(label)
+      puts "Starting: [#{label}]"
+      t0 = Time.now
+      result = yield
+      t1 = Time.now
+      diff = t1 - t0
+      puts "Elapsed time [#{label}] %.2f s" % diff
+      result
+    end
+
+    def self.setup_api(api_key, base_uri)
+      DataCatalog.api_key  = api_key
+      DataCatalog.base_uri = base_uri
+    end
+
+    def self.sort_uris(uris)
+      uris.sort do |uri_a, uri_b|
+        uid(uri_a) <=> uid(uri_b)
+      end
+    end
+
+    # Converts a URI into a Unique ID
+    #
+    #   For example:
+    #   Utility.uid("http://data.octo.dc.gov/Metadata.aspx?id=137)
+    #   => 137
+    def self.uid(uri)
+      last_part = uri.split("?id=").last
+      result = last_part.to_i
+      if result == 0
+        message = unindent <<-BLOCK
+          Could not make a non-zero Unique ID for this URI:
+          #{uri}
+        BLOCK
+        raise Error, message
+      end
+      result
+    end
+
+    def self.write_yaml(filename, contents)
+      File.open(filename, "w") do |f|
+        YAML::dump(contents, f)
+      end
+    end
+
+  end
+
+end
+
+# == Note 001 ==
+#
+# I have experienced a strange bug. When fetching a URI from the Web,
+# I get a document that has a link that contains a pipe ("|"). This is a
+# problem because calling URI.parse on a string with a pipe raises an
+# exception:
+#
+#   http://ouc.dc.gov/ouc/cwp/view,a,3,q,552751,oucNav,|32048|.asp
+#   bad URI(is not URI?)
+#
+# Workaround: I write the fetched document to disk first, and then read
+# the document from disk. For some reason the URL gets read from disk as:
+#
+#   http://ouc.dc.gov/ouc/cwp/view,a,3,q,552751,oucNav,%7C32048%7C.asp
+#
+# It would be nice to figure out why this is necessary.

data/spec/datacatalog-importer_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "DatacatalogImporter" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'datacatalog-importer'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification 
+name: datacatalog-importer
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- David James
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-05 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+description: This framework makes it easier to write importers for the National Data Catalog.
+email: djames@sunlightfoundation.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- datacatalog-importer.gemspec
+- lib/datacatalog-importer.rb
+- lib/importer.rb
+- lib/puller.rb
+- lib/pusher.rb
+- lib/shared.rb
+- lib/sort_yaml_hash.rb
+- lib/tasks.rb
+- lib/utility.rb
+- spec/datacatalog-importer_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/djsun/datacatalog-importer
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A framework to write National Data Catalog importers
+test_files: 
+- spec/datacatalog-importer_spec.rb
+- spec/spec_helper.rb