datacatalog-importer 0.1.1 → 0.1.2

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/datacatalog-importer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.1.1"
+  s.version = "0.1.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
-  s.date = %q{2010-02-19}
+  s.date = %q{2010-03-02}
   s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
   s.email = %q{djames@sunlightfoundation.com}
   s.extra_rdoc_files = [

data/lib/puller.rb

@@ -45,7 +45,7 @@ module DataCatalog
       #
       def pull_resource(resource)
         unless importer_class = @options[:pullers][resource]
-          raise Error, "options[:pullers][:#{r}] is required"
+          raise Error, "options[:pullers][:#{resource}] is required"
         end
         importer = importer_class.new
         FileUtils.mkdir_p(folder(resource))

data/lib/utility.rb

@@ -1,3 +1,4 @@
+require 'fastercsv'
 require 'nokogiri'
 require 'open-uri'
 
@@ -30,28 +31,51 @@ module DataCatalog
         "UserAgent" => "National Data Catalog Importer/0.1.0",
       }
     end
+    
+    def self.parse_csv_from_file(filename, extra_header_rows=0)
+      File.open(filename) do |f|
+        extra_header_rows.times { f.gets } # ignore these rows
+        FasterCSV.parse(f)
+      end
+    end
 
-    def self.parse_file(filename)
+    def self.parse_html_from_file(filename)
       File.open(filename) do |f|
         Nokogiri::HTML::Document.parse(f)
       end
     end
     
-    def self.parse_file_or_uri(uri, file, options={})
+    def self.parse_csv_from_file_or_uri(uri, file, options={})
       if options[:force_fetch] || !File.exist?(file)
-        document = parse_uri(uri)
+        document = parse_csv_from_uri(uri)
         File.open(file, "w") { |f| f.write(document) }
       end
-      parse_file(file) # Why always parse the file? See Note 001, below.
+      parse_csv_from_file(file, options[:extra_header_rows] || 0)
+      # Why always parse the file? Consistency with parse_html_from_file_or_uri.
     end
-
-    def self.parse_uri(uri)
+    
+    def self.parse_html_from_file_or_uri(uri, file, options={})
+      if options[:force_fetch] || !File.exist?(file)
+        document = parse_html_from_uri(uri)
+        File.open(file, "w") { |f| f.write(document) }
+      end
+      parse_html_from_file(file) # Why always parse the file? See Note 001, below.
+    end
+    
+    def self.parse_csv_from_uri(uri)
       puts "Fetching #{uri}..."
       open(uri, headers) do |io|
         Nokogiri::HTML::Document.parse(io)
       end
     end
 
+    def self.parse_html_from_uri(uri)
+      puts "Fetching #{uri}..."
+      open(uri, headers) do |io|
+        FasterCSV.parse(io)
+      end
+    end
+
     # ActiveSupport 2.3.5 adds @_rails_html_safe aggressively.
     # This method removes it so you can output clean YAML.
     def self.plain_string(s)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: datacatalog-importer
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - David James
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-19 00:00:00 -05:00
+date: 2010-03-02 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency