datacatalog-importer 0.1.10 → 0.1.11

data/VERSION

@@ -1 +1 @@
-0.1.10
+0.1.11

data/datacatalog-importer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.1.10"
+  s.version = "0.1.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
-  s.date = %q{2010-03-17}
+  s.date = %q{2010-04-05}
   s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
   s.email = %q{djames@sunlightfoundation.com}
   s.extra_rdoc_files = [

data/lib/utility.rb

@@ -6,9 +6,13 @@ module DataCatalog
 
   class Utility
     
+    # == URLs ==
+    
     def self.absolute_url(page_url, url)
       Utility.plain_string(URI.parse(page_url).merge(url).to_s)
     end
+    
+    # == Cleaning ==
 
     def self.single_line_clean(s)
       plain_string(
@@ -20,10 +24,20 @@ module DataCatalog
         s.gsub(/[\x80-\xFF]/, "").squeeze(" ").strip)
     end
 
-    def self.fetch(uri)
-      puts "Fetching #{uri}..."
-      io = open(uri, headers)
-      io.read
+    # ActiveSupport 2.3.5 adds @_rails_html_safe aggressively.
+    # This method removes it so you can output clean YAML.
+    def self.plain_string(s)
+      if s.instance_variable_defined?(:@_rails_html_safe)
+        s.send(:remove_instance_variable, :@_rails_html_safe)
+      end
+      s
+    end
+    
+    # == API ===
+    
+    def self.setup_api(api_key, base_uri)
+      DataCatalog.api_key  = api_key
+      DataCatalog.base_uri = base_uri
     end
 
     def self.headers
@@ -31,7 +45,27 @@ module DataCatalog
         "UserAgent" => "National Data Catalog Importer/0.1.6",
       }
     end
+        
+    # == Various ==
     
+    def self.fetch(uri)
+      puts "Fetching #{uri}..."
+      io = open(uri, headers)
+      io.read
+    end
+    
+    def self.report_timing(label)
+      puts "Starting: [#{label}]"
+      t0 = Time.now
+      result = yield
+      t1 = Time.now
+      diff = t1 - t0
+      puts "Elapsed time [#{label}] %.2f s" % diff
+      result
+    end
+    
+    # == CSV ==
+
     # { :headers => true } is a common option
     def self.parse_csv_from_file(filename, options={})
       extra_header_rows = options.delete(:extra_header_rows) || 0
@@ -41,12 +75,13 @@ module DataCatalog
       end
     end
 
-    def self.parse_html_from_file(filename)
-      File.open(filename) do |f|
-        Nokogiri::HTML::Document.parse(f)
-      end
+    def self.parse_csv_from_uri(uri, options={})
+      puts "Fetching #{uri}..."
+      data = open(uri, headers)
+      puts data.inspect
+      FasterCSV.parse(data, options)
     end
-    
+
     def self.parse_csv_from_file_or_uri(uri, file, options={})
       force_fetch = options.delete(:force_fetch) || false
       if force_fetch || !File.exist?(file)
@@ -56,19 +91,12 @@ module DataCatalog
       parse_csv_from_file(file, options)
     end
     
-    def self.parse_html_from_file_or_uri(uri, file, options={})
-      if options[:force_fetch] || !File.exist?(file)
-        document = parse_html_from_uri(uri)
-        File.open(file, "w") { |f| f.write(document) }
+    # == HTML ==
+
+    def self.parse_html_from_file(filename)
+      File.open(filename) do |f|
+        Nokogiri::HTML::Document.parse(f)
       end
-      parse_html_from_file(file) # Why always parse the file? See Note 001, below.
-    end
-    
-    def self.parse_csv_from_uri(uri, options={})
-      puts "Fetching #{uri}..."
-      data = open(uri, headers)
-      puts data.inspect
-      FasterCSV.parse(data, options)
     end
 
     def self.parse_html_from_uri(uri)
@@ -78,30 +106,39 @@ module DataCatalog
       end
     end
 
-    # ActiveSupport 2.3.5 adds @_rails_html_safe aggressively.
-    # This method removes it so you can output clean YAML.
-    def self.plain_string(s)
-      if s.instance_variable_defined?(:@_rails_html_safe)
-        s.send(:remove_instance_variable, :@_rails_html_safe)
+    def self.parse_html_from_file_or_uri(uri, file, options={})
+      if options[:force_fetch] || !File.exist?(file)
+        document = parse_html_from_uri(uri)
+        File.open(file, "w") { |f| f.write(document) }
       end
-      s
+      parse_html_from_file(file) # Why always parse the file? See Note 001, below.
     end
+    
+    # == XML
 
-    def self.report_timing(label)
-      puts "Starting: [#{label}]"
-      t0 = Time.now
-      result = yield
-      t1 = Time.now
-      diff = t1 - t0
-      puts "Elapsed time [#{label}] %.2f s" % diff
-      result
+    def self.parse_xml_from_file(filename)
+      File.open(filename) do |f|
+        Nokogiri::XML::Document.parse(f)
+      end
     end
 
-    def self.setup_api(api_key, base_uri)
-      DataCatalog.api_key  = api_key
-      DataCatalog.base_uri = base_uri
+    def self.parse_xml_from_uri(uri)
+    	puts "Fetching #{uri}..."
+    	Nokogiri::XML(open(uri))
     end
-
+    
+    def self.parse_xml_from_file_or_uri(uri, file, options={})
+    	if options[:force_fetch] || !File.exist?(file)
+    		document = parse_xml_from_uri(uri)
+    		File.open(file, "w") { |f| f.write(document) }
+    	end
+    	parse_xml_from_file(file) # Why always parse the file? See Note 001, below.
+    end
+    
+    # == YAML
+    
+    # To load YAML use: YAML::load_file(filename)
+    
     def self.write_yaml(filename, contents)
       File.open(filename, "w") do |f|
         YAML::dump(contents, f)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 10
-  version: 0.1.10
+  - 11
+  version: 0.1.11
 platform: ruby
 authors: 
 - David James
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-17 00:00:00 -04:00
+date: 2010-04-05 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency