datacatalog-importer 0.2.2 → 0.2.3

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.2.3

data/datacatalog-importer.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.2.2"
+  s.version = "0.2.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
@@ -40,6 +40,8 @@ Gem::Specification.new do |s|
      "natdat_is_hungry.md",
      "spec/spec.opts",
      "spec/spec_helper.rb",
+     "spec/test.csv",
+     "spec/test.html",
      "spec/test.json",
      "spec/test.xml",
      "spec/utility_spec.rb"

data/lib/utility.rb

@@ -57,7 +57,7 @@ module DataCatalog
 
       def self.headers
         {
-          "UserAgent" => "National Data Catalog Importer/0.2.2",
+          "UserAgent" => "National Data Catalog Importer/0.2.3",
         }
       end
 
@@ -67,10 +67,10 @@ module DataCatalog
       # * http://jerith.livejournal.com/40063.html
       # * http://lindsaar.net/2007/12/9/rbuf_filltimeout-error
       def self.fetch(uri, options={})
-        max_attempts = options[:max_attempts] || 3
-        retry_delay  = options[:retry_delay] || 5
-        quiet        = options[:quiet] || false
-        attempts = 0
+        max_attempts = options.delete(:max_attempts) || 3
+        retry_delay  = options.delete(:retry_delay)  || 5
+        quiet        = options.delete(:quiet)        || false
+        attempts     = 0
         loop do
           begin
             puts "Fetching #{uri}..." unless quiet
@@ -85,6 +85,12 @@ module DataCatalog
           end
         end
       end
+      
+      def self.remove_fetch_options(options={})
+        [:max_attempts, :retry_delay, :quiet].each do |opt|
+          options.delete(opt)
+        end
+      end
 
       def self.report_timing(label)
         puts "Starting: [#{label}]"
@@ -99,9 +105,9 @@ module DataCatalog
       # == CSV ==
 
       # { :headers => true } is a common option
-      def self.parse_csv_from_file(filename, options={})
+      def self.parse_csv_from_file(file, options={})
         extra_header_rows = options.delete(:extra_header_rows) || 0
-        File.open(filename) do |f|
+        File.open(file) do |f|
           extra_header_rows.times { f.gets } # ignore these rows
           FasterCSV.parse(f, options)
         end
@@ -112,19 +118,22 @@ module DataCatalog
         FasterCSV.parse(data, options)
       end
 
-      def self.parse_csv_from_file_or_uri(uri, file, options={})
+      def self.parse_csv_from_file_or_uri(file, uri, options={})
         force_fetch = options.delete(:force_fetch) || false
         if force_fetch || !File.exist?(file)
-          document = fetch(uri, options)
-          File.open(file, "w") { |f| f.write(document) }
+          data = fetch(uri, options)
+          File.open(file, "w") { |f| f.write(data) }
+        else
+          remove_fetch_options(options)
         end
+        # Why always parse the file? See Note 001, below.
         parse_csv_from_file(file, options)
       end
 
       # == HTML ==
 
-      def self.parse_html_from_file(filename)
-        File.open(filename) do |f|
+      def self.parse_html_from_file(file)
+        File.open(file) do |f|
           Nokogiri::HTML::Document.parse(f)
         end
       end
@@ -134,10 +143,13 @@ module DataCatalog
         Nokogiri::HTML::Document.parse(data)
       end
 
-      def self.parse_html_from_file_or_uri(uri, file, options={})
-        if options[:force_fetch] || !File.exist?(file)
-          document = parse_html_from_uri(uri)
-          File.open(file, "w") { |f| f.write(document) }
+      def self.parse_html_from_file_or_uri(file, uri, options={})
+        force_fetch = options.delete(:force_fetch) || false
+        if force_fetch || !File.exist?(file)
+          data = fetch(uri, options)
+          File.open(file, "w") { |f| f.write(data) }
+        else
+          remove_fetch_options(options)
         end
         # Why always parse the file? See Note 001, below.
         parse_html_from_file(file)
@@ -145,8 +157,8 @@ module DataCatalog
 
       # == JSON
 
-      def self.parse_json_from_file(filename)
-        File.open(filename) do |f|
+      def self.parse_json_from_file(file)
+        File.open(file) do |f|
           JSON.parse(f.read)
         end
       end
@@ -156,10 +168,13 @@ module DataCatalog
         JSON.parse(data)
       end
 
-      def self.parse_json_from_file_or_uri(uri, file, options={})
-        if options[:force_fetch] || !File.exist?(file)
-          document = parse_json_from_uri(uri)
-          File.open(file, "w") { |f| f.write(document) }
+      def self.parse_json_from_file_or_uri(file, uri, options={})
+        force_fetch = options.delete(:force_fetch) || false
+        if force_fetch || !File.exist?(file)
+          data = fetch(uri, options)
+          File.open(file, "w") { |f| f.write(data) }
+        else
+          remove_fetch_options(options)
         end
         # Why always parse the file? See Note 001, below.
         parse_json_from_file(file)
@@ -167,8 +182,8 @@ module DataCatalog
 
       # == XML
 
-      def self.parse_xml_from_file(filename)
-        File.open(filename) do |f|
+      def self.parse_xml_from_file(file)
+        File.open(file) do |f|
           Nokogiri::XML::Document.parse(f)
         end
       end
@@ -178,10 +193,13 @@ module DataCatalog
         Nokogiri::XML::Document.parse(data)
       end
 
-      def self.parse_xml_from_file_or_uri(uri, file, options={})
-        if options[:force_fetch] || !File.exist?(file)
-          document = parse_xml_from_uri(uri)
-          File.open(file, "w") { |f| f.write(document) }
+      def self.parse_xml_from_file_or_uri(file, uri, options={})
+        force_fetch = options.delete(:force_fetch) || false
+        if force_fetch || !File.exist?(file)
+          data = fetch(uri, options)
+          File.open(file, "w") { |f| f.write(data) }
+        else
+          remove_fetch_options(options)
         end
         # Why always parse the file? See Note 001, below.
         parse_xml_from_file(file)
@@ -189,10 +207,10 @@ module DataCatalog
 
       # == YAML
 
-      # To load YAML use: YAML::load_file(filename)
+      # To load YAML use: YAML::load_file(file)
 
-      def self.write_yaml(filename, contents)
-        File.open(filename, "w") do |f|
+      def self.write_yaml(file, contents)
+        File.open(file, "w") do |f|
           YAML::dump(contents, f)
         end
       end

data/spec/test.csv

@@ -0,0 +1 @@
+Metro Center,Dupont Circle

data/spec/test.html

@@ -0,0 +1,8 @@
+<html>
+  <body>
+    <ol>
+      <li>Metro Center</li>
+      <li>Dupont Circle</li>
+    </ol>
+  </body>
+</html>

data/spec/utility_spec.rb

@@ -105,57 +105,193 @@ describe "Utility" do
     end
   end
 
-  # == XML
+  describe "csv" do
+    describe "parse_csv_from_file" do
+      it "should work" do
+        file = File.dirname(__FILE__) + '/test.csv'
+        result = U.parse_csv_from_file(file)
+        result.should == [["Metro Center", "Dupont Circle"]]
+      end
+    end
 
-  describe "parse_xml_from_file" do
-    it "should work" do
-      file = File.dirname(__FILE__) + '/test.xml'
-      parsed = U.parse_xml_from_file(file)
-      result = parsed.xpath('.//stations/station').map(&:content)
-      result.should == ["Metro Center", "Dupont Circle"]
+    describe "parse_csv_from_uri" do
+      it "should work" do
+        readable = Object.new
+        readable.stub(:read).and_return(%(Metro Center,Dupont Circle))
+        U.stub(:open).and_return(readable)
+        result = U.parse_csv_from_uri("fake", :quiet => true)
+        result.should == [["Metro Center", "Dupont Circle"]]
+      end
+    end
+
+    describe "parse_csv_from_file_or_uri" do
+      it "should work when file present" do
+        file = File.dirname(__FILE__) + '/test.csv'
+        result = U.parse_csv_from_file_or_uri(file, "fake", :quiet => true)
+        result.should == [["Metro Center", "Dupont Circle"]]
+      end
+
+      it "should work when file missing but url ok" do
+        readable = Object.new
+        readable.stub(:read).and_return(%(Metro Center,Dupont Circle))
+        U.stub(:open).and_return(readable)
+        file = File.dirname(__FILE__) + "/missing.csv"
+        FileUtils.rm(file) if File.exists?(file)
+        result = U.parse_csv_from_file_or_uri(file, "fake", :quiet => true)
+        FileUtils.rm(file) if File.exists?(file)
+        result.should == [["Metro Center", "Dupont Circle"]]
+      end
     end
   end
 
-  describe "parse_xml_from_uri" do
-    before do
-      @readable = Object.new
-      @readable.stub(:read).and_return(%(
-        <stations>
-          <station>Metro Center</station>
-          <station>Dupont Circle</station>
-        </stations>
-      ))
+  describe "html" do
+    describe "parse_html_from_file" do
+      it "should work" do
+        file = File.dirname(__FILE__) + '/test.html'
+        parsed = U.parse_html_from_file(file)
+        result = parsed.css('li').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
     end
 
-    it "should work" do
-      U.stub(:open).and_return(@readable)
-      parsed = U.parse_xml_from_uri("fake", :quiet => true)
-      result = parsed.xpath('.//stations/station').map(&:content)
-      result.should == ["Metro Center", "Dupont Circle"]
+    describe "parse_html_from_uri" do
+      it "should work" do
+        readable = Object.new
+        readable.stub(:read).and_return(%(
+          <html>
+            <body>
+              <ol>
+                <li>Metro Center</li>
+                <li>Dupont Circle</li>
+              </ol>
+            </body>
+          </html>
+        ))
+        U.stub(:open).and_return(readable)
+        parsed = U.parse_html_from_uri("fake", :quiet => true)
+        result = parsed.css('li').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
+    end
+
+    describe "parse_html_from_file_or_uri" do
+      it "should work when file present" do
+        file = File.dirname(__FILE__) + '/test.html'
+        parsed = U.parse_html_from_file_or_uri(file, "fake", :quiet => true)
+        result = parsed.css('li').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
+
+      it "should work when file missing but url ok" do
+        readable = Object.new
+        readable.stub(:read).and_return(%(
+          <stations>
+            <station>Metro Center</station>
+            <station>Dupont Circle</station>
+          </stations>
+        ))
+        U.stub(:open).and_return(readable)
+        file = File.dirname(__FILE__) + "/missing.html"
+        FileUtils.rm(file) if File.exists?(file)
+        parsed = U.parse_html_from_file_or_uri(file, "fake", :quiet => true)
+        FileUtils.rm(file) if File.exists?(file)
+        result = parsed.xpath('.//stations/station').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
     end
   end
 
-  # == JSON
+  describe "json" do
+    describe "parse_json_from_file" do
+      it "should work" do
+        file = File.dirname(__FILE__) + '/test.json'
+        U.parse_json_from_file(file).should ==
+          { "stations" => ["Metro Center", "Dupont Circle"] }
+      end
+    end
 
-  describe "parse_json_from_file" do
-    it "should work" do
-      file = File.dirname(__FILE__) + '/test.json'
-      U.parse_json_from_file(file).should ==
-        { "stations" => ["Metro Center","Dupont Circle"] }
+    describe "parse_json_from_uri" do
+      it "should work" do
+        readable = Object.new
+        readable.stub(:read).and_return(
+          %({"stations":["Metro Center","Dupont Circle"]}))
+        U.stub(:open).and_return(readable)
+        result = U.parse_json_from_uri("fake", :quiet => true)
+        result.should == { "stations" => ["Metro Center", "Dupont Circle"] }
+      end
+    end
+
+    describe "parse_json_from_file_or_uri" do
+      it "should work when file present" do
+        file = File.dirname(__FILE__) + '/test.json'
+        result = U.parse_json_from_file_or_uri(file, "fake", :quiet => true)
+        result.should == { "stations" => ["Metro Center", "Dupont Circle"] }
+      end
+
+      it "should work when file missing but url ok" do
+        readable = Object.new
+        readable.stub(:read).and_return(
+          %({"stations":["Metro Center","Dupont Circle"]}))
+        U.stub(:open).and_return(readable)
+        file = File.dirname(__FILE__) + "/missing.json"
+        FileUtils.rm(file) if File.exists?(file)
+        result = U.parse_json_from_file_or_uri(file, "fake", :quiet => true)
+        FileUtils.rm(file) if File.exists?(file)
+        result.should == { "stations" => ["Metro Center", "Dupont Circle"] }
+      end
     end
   end
 
-  describe "parse_json_from_uri" do
-    before do
-      @readable = Object.new
-      @readable.stub(:read).and_return(
-        %({"stations":["Metro Center","Dupont Circle"]}))
+  describe "xml" do
+    describe "parse_xml_from_file" do
+      it "should work" do
+        file = File.dirname(__FILE__) + '/test.xml'
+        parsed = U.parse_xml_from_file(file)
+        result = parsed.xpath('.//stations/station').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
     end
 
-    it "should work" do
-      U.stub(:open).and_return(@readable)
-      result = U.parse_json_from_uri("fake", :quiet => true)
-      result.should == { "stations" => ["Metro Center","Dupont Circle"] }
+    describe "parse_xml_from_uri" do
+      it "should work" do
+        readable = Object.new
+        readable.stub(:read).and_return(%(
+          <stations>
+            <station>Metro Center</station>
+            <station>Dupont Circle</station>
+          </stations>
+        ))
+        U.stub(:open).and_return(readable)
+        parsed = U.parse_xml_from_uri("fake", :quiet => true)
+        result = parsed.xpath('.//stations/station').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
+    end
+
+    describe "parse_xml_from_file_or_uri" do
+      it "should work when file present" do
+        file = File.dirname(__FILE__) + '/test.xml'
+        parsed = U.parse_xml_from_file_or_uri(file, "fake", :quiet => true)
+        result = parsed.xpath('.//stations/station').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
+
+      it "should work when file missing but url ok" do
+        readable = Object.new
+        readable.stub(:read).and_return(%(
+          <stations>
+            <station>Metro Center</station>
+            <station>Dupont Circle</station>
+          </stations>
+        ))
+        U.stub(:open).and_return(readable)
+        file = File.dirname(__FILE__) + "/missing.xml"
+        FileUtils.rm(file) if File.exists?(file)
+        parsed = U.parse_xml_from_file_or_uri(file, "fake", :quiet => true)
+        FileUtils.rm(file) if File.exists?(file)
+        result = parsed.xpath('.//stations/station').map(&:content)
+        result.should == ["Metro Center", "Dupont Circle"]
+      end
     end
   end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: datacatalog-importer
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: false
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 3
+  version: 0.2.3
 platform: ruby
 authors: 
 - David James
@@ -99,6 +99,8 @@ files:
 - natdat_is_hungry.md
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/test.csv
+- spec/test.html
 - spec/test.json
 - spec/test.xml
 - spec/utility_spec.rb