datacatalog-importer 0.1.15 → 0.1.16

data/VERSION

@@ -1 +1 @@
-0.1.15
+0.1.16

data/datacatalog-importer.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.1.15"
+  s.version = "0.1.16"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]

data/lib/utility.rb

@@ -5,13 +5,13 @@ require 'open-uri'
 module DataCatalog
   module ImporterFramework
     class Utility
-    
+
       # == URLs ==
-    
+
       def self.absolute_url(page_url, url)
         Utility.plain_string(URI.parse(page_url).merge(url).to_s)
       end
-    
+
       # == Cleaning ==
 
       def self.single_line_clean(s)
@@ -32,9 +32,9 @@ module DataCatalog
         end
         s
       end
-    
+
       # == API ===
-    
+
       def self.setup_api(api_key, base_uri)
         DataCatalog.api_key  = api_key
         DataCatalog.base_uri = base_uri
@@ -42,10 +42,10 @@ module DataCatalog
 
       def self.headers
         {
-          "UserAgent" => "National Data Catalog Importer/0.1.6",
+          "UserAgent" => "National Data Catalog Importer/0.1.16",
         }
       end
-        
+
       # == Various ==
 
       # For background on rescuing net/http errors, see:
@@ -66,7 +66,7 @@ module DataCatalog
           end
         end
       end
-    
+
       def self.report_timing(label)
         puts "Starting: [#{label}]"
         t0 = Time.now
@@ -76,7 +76,7 @@ module DataCatalog
         puts "Elapsed time [#{label}] %.2f s" % diff
         result
       end
-    
+
       # == CSV ==
 
       # { :headers => true } is a common option
@@ -89,9 +89,7 @@ module DataCatalog
       end
 
       def self.parse_csv_from_uri(uri, options={})
-        puts "Fetching #{uri}..."
-        data = open(uri, headers)
-        puts data.inspect
+        data = fetch(uri)
         FasterCSV.parse(data, options)
       end
 
@@ -103,7 +101,7 @@ module DataCatalog
         end
         parse_csv_from_file(file, options)
       end
-    
+
       # == HTML ==
 
       def self.parse_html_from_file(filename)
@@ -113,10 +111,8 @@ module DataCatalog
       end
 
       def self.parse_html_from_uri(uri)
-        puts "Fetching #{uri}..."
-        open(uri, headers) do |io|
-          Nokogiri::HTML::Document.parse(io)
-        end
+        data = fetch(uri)
+        Nokogiri::HTML::Document.parse(data)
       end
 
       def self.parse_html_from_file_or_uri(uri, file, options={})
@@ -126,7 +122,7 @@ module DataCatalog
         end
         parse_html_from_file(file) # Why always parse the file? See Note 001, below.
       end
-    
+
       # == XML
 
       def self.parse_xml_from_file(filename)
@@ -136,22 +132,22 @@ module DataCatalog
       end
 
       def self.parse_xml_from_uri(uri)
-      	puts "Fetching #{uri}..."
-      	Nokogiri::XML(open(uri))
+        puts "Fetching #{uri}..."
+        Nokogiri::XML(open(uri))
       end
-    
+
       def self.parse_xml_from_file_or_uri(uri, file, options={})
-      	if options[:force_fetch] || !File.exist?(file)
-      		document = parse_xml_from_uri(uri)
-      		File.open(file, "w") { |f| f.write(document) }
-      	end
-      	parse_xml_from_file(file) # Why always parse the file? See Note 001, below.
+        if options[:force_fetch] || !File.exist?(file)
+          document = parse_xml_from_uri(uri)
+          File.open(file, "w") { |f| f.write(document) }
+        end
+        parse_xml_from_file(file) # Why always parse the file? See Note 001, below.
       end
-    
+
       # == YAML
-    
+
       # To load YAML use: YAML::load_file(filename)
-    
+
       def self.write_yaml(filename, contents)
         File.open(filename, "w") do |f|
           YAML::dump(contents, f)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 15
-  version: 0.1.15
+  - 16
+  version: 0.1.16
 platform: ruby
 authors: 
 - David James