datacatalog-importer 0.2.3 → 0.3.0

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.3.0

data/datacatalog-importer.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.2.3"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]

data/lib/utility.rb

@@ -57,7 +57,7 @@ module DataCatalog
 
       def self.headers
         {
-          "UserAgent" => "National Data Catalog Importer/0.2.3",
+          "UserAgent" => "National Data Catalog Importer/0.3.0",
         }
       end
 
@@ -101,99 +101,46 @@ module DataCatalog
         puts "Elapsed time [#{label}] %.2f s" % diff
         result
       end
-
-      # == CSV ==
-
-      # { :headers => true } is a common option
-      def self.parse_csv_from_file(file, options={})
-        extra_header_rows = options.delete(:extra_header_rows) || 0
-        File.open(file) do |f|
-          extra_header_rows.times { f.gets } # ignore these rows
-          FasterCSV.parse(f, options)
-        end
-      end
-
-      def self.parse_csv_from_uri(uri, options={})
-        data = fetch(uri, options)
-        FasterCSV.parse(data, options)
-      end
-
-      def self.parse_csv_from_file_or_uri(file, uri, options={})
-        force_fetch = options.delete(:force_fetch) || false
-        if force_fetch || !File.exist?(file)
-          data = fetch(uri, options)
-          File.open(file, "w") { |f| f.write(data) }
-        else
-          remove_fetch_options(options)
-        end
-        # Why always parse the file? See Note 001, below.
-        parse_csv_from_file(file, options)
-      end
-
-      # == HTML ==
-
-      def self.parse_html_from_file(file)
-        File.open(file) do |f|
-          Nokogiri::HTML::Document.parse(f)
-        end
-      end
-
-      def self.parse_html_from_uri(uri, options={})
-        data = fetch(uri, options)
-        Nokogiri::HTML::Document.parse(data)
-      end
-
-      def self.parse_html_from_file_or_uri(file, uri, options={})
-        force_fetch = options.delete(:force_fetch) || false
-        if force_fetch || !File.exist?(file)
-          data = fetch(uri, options)
-          File.open(file, "w") { |f| f.write(data) }
-        else
-          remove_fetch_options(options)
-        end
-        # Why always parse the file? See Note 001, below.
-        parse_html_from_file(file)
-      end
-
-      # == JSON
-
-      def self.parse_json_from_file(file)
+      
+      # == Parsing ===
+      
+      def self.parse_file(format, file, options={})
         File.open(file) do |f|
-          JSON.parse(f.read)
+          case format
+          when :csv
+            extra_header_rows = options.delete(:extra_header_rows) || 0
+            extra_header_rows.times { f.gets } # ignore these rows
+            FasterCSV.parse(f, options)
+          when :xml
+            Nokogiri::XML::Document.parse(f)
+          when :json
+            JSON.parse(f.read)
+          when :html
+            Nokogiri::HTML::Document.parse(f)
+          else
+            raise "Unexpected format : #{format.inspect}"
+          end
         end
       end
-
-      def self.parse_json_from_uri(uri, options={})
+      
+      def self.parse_uri(format, uri, options={})
         data = fetch(uri, options)
-        JSON.parse(data)
-      end
-
-      def self.parse_json_from_file_or_uri(file, uri, options={})
-        force_fetch = options.delete(:force_fetch) || false
-        if force_fetch || !File.exist?(file)
-          data = fetch(uri, options)
-          File.open(file, "w") { |f| f.write(data) }
+        case format
+        when :csv
+          # TODO: support extra_header_rows option
+          FasterCSV.parse(data, options)
+        when :xml
+          Nokogiri::XML::Document.parse(data)
+        when :json
+          JSON.parse(data)
+        when :html
+          Nokogiri::HTML::Document.parse(data)
         else
-          remove_fetch_options(options)
+          raise "Unexpected format : #{format.inspect}"
         end
-        # Why always parse the file? See Note 001, below.
-        parse_json_from_file(file)
       end
-
-      # == XML
-
-      def self.parse_xml_from_file(file)
-        File.open(file) do |f|
-          Nokogiri::XML::Document.parse(f)
-        end
-      end
-
-      def self.parse_xml_from_uri(uri, options={})
-        data = fetch(uri, options)
-        Nokogiri::XML::Document.parse(data)
-      end
-
-      def self.parse_xml_from_file_or_uri(file, uri, options={})
+      
+      def self.parse_file_or_uri(format, file, uri, options={})
         force_fetch = options.delete(:force_fetch) || false
         if force_fetch || !File.exist?(file)
           data = fetch(uri, options)
@@ -202,7 +149,7 @@ module DataCatalog
           remove_fetch_options(options)
         end
         # Why always parse the file? See Note 001, below.
-        parse_xml_from_file(file)
+        parse_file(format, file, options)
       end
 
       # == YAML

data/spec/utility_spec.rb

@@ -106,28 +106,28 @@ describe "Utility" do
   end
 
   describe "csv" do
-    describe "parse_csv_from_file" do
+    describe "parse_file" do
       it "should work" do
         file = File.dirname(__FILE__) + '/test.csv'
-        result = U.parse_csv_from_file(file)
+        result = U.parse_file(:csv, file)
         result.should == [["Metro Center", "Dupont Circle"]]
       end
     end
 
-    describe "parse_csv_from_uri" do
+    describe "parse_uri" do
       it "should work" do
         readable = Object.new
         readable.stub(:read).and_return(%(Metro Center,Dupont Circle))
         U.stub(:open).and_return(readable)
-        result = U.parse_csv_from_uri("fake", :quiet => true)
+        result = U.parse_uri(:csv, "fake", :quiet => true)
         result.should == [["Metro Center", "Dupont Circle"]]
       end
     end
 
-    describe "parse_csv_from_file_or_uri" do
+    describe "parse_file_or_uri" do
       it "should work when file present" do
         file = File.dirname(__FILE__) + '/test.csv'
-        result = U.parse_csv_from_file_or_uri(file, "fake", :quiet => true)
+        result = U.parse_file_or_uri(:csv, file, "fake", :quiet => true)
         result.should == [["Metro Center", "Dupont Circle"]]
       end
 
@@ -137,7 +137,7 @@ describe "Utility" do
         U.stub(:open).and_return(readable)
         file = File.dirname(__FILE__) + "/missing.csv"
         FileUtils.rm(file) if File.exists?(file)
-        result = U.parse_csv_from_file_or_uri(file, "fake", :quiet => true)
+        result = U.parse_file_or_uri(:csv, file, "fake", :quiet => true)
         FileUtils.rm(file) if File.exists?(file)
         result.should == [["Metro Center", "Dupont Circle"]]
       end
@@ -145,16 +145,16 @@ describe "Utility" do
   end
 
   describe "html" do
-    describe "parse_html_from_file" do
+    describe "parse_file" do
       it "should work" do
         file = File.dirname(__FILE__) + '/test.html'
-        parsed = U.parse_html_from_file(file)
+        parsed = U.parse_file(:html, file)
         result = parsed.css('li').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
       end
     end
 
-    describe "parse_html_from_uri" do
+    describe "parse_uri" do
       it "should work" do
         readable = Object.new
         readable.stub(:read).and_return(%(
@@ -168,16 +168,16 @@ describe "Utility" do
           </html>
         ))
         U.stub(:open).and_return(readable)
-        parsed = U.parse_html_from_uri("fake", :quiet => true)
+        parsed = U.parse_uri(:html, "fake", :quiet => true)
         result = parsed.css('li').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
       end
     end
 
-    describe "parse_html_from_file_or_uri" do
+    describe "parse_file_or_uri" do
       it "should work when file present" do
         file = File.dirname(__FILE__) + '/test.html'
-        parsed = U.parse_html_from_file_or_uri(file, "fake", :quiet => true)
+        parsed = U.parse_file_or_uri(:html, file, "fake", :quiet => true)
         result = parsed.css('li').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
       end
@@ -193,7 +193,7 @@ describe "Utility" do
         U.stub(:open).and_return(readable)
         file = File.dirname(__FILE__) + "/missing.html"
         FileUtils.rm(file) if File.exists?(file)
-        parsed = U.parse_html_from_file_or_uri(file, "fake", :quiet => true)
+        parsed = U.parse_file_or_uri(:html, file, "fake", :quiet => true)
         FileUtils.rm(file) if File.exists?(file)
         result = parsed.xpath('.//stations/station').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
@@ -202,29 +202,29 @@ describe "Utility" do
   end
 
   describe "json" do
-    describe "parse_json_from_file" do
+    describe "parse_file" do
       it "should work" do
         file = File.dirname(__FILE__) + '/test.json'
-        U.parse_json_from_file(file).should ==
+        U.parse_file(:json, file).should ==
           { "stations" => ["Metro Center", "Dupont Circle"] }
       end
     end
 
-    describe "parse_json_from_uri" do
+    describe "parse_uri" do
       it "should work" do
         readable = Object.new
         readable.stub(:read).and_return(
           %({"stations":["Metro Center","Dupont Circle"]}))
         U.stub(:open).and_return(readable)
-        result = U.parse_json_from_uri("fake", :quiet => true)
+        result = U.parse_uri(:json, "fake", :quiet => true)
         result.should == { "stations" => ["Metro Center", "Dupont Circle"] }
       end
     end
 
-    describe "parse_json_from_file_or_uri" do
+    describe "parse_file_or_uri" do
       it "should work when file present" do
         file = File.dirname(__FILE__) + '/test.json'
-        result = U.parse_json_from_file_or_uri(file, "fake", :quiet => true)
+        result = U.parse_file_or_uri(:json, file, "fake", :quiet => true)
         result.should == { "stations" => ["Metro Center", "Dupont Circle"] }
       end
 
@@ -235,7 +235,7 @@ describe "Utility" do
         U.stub(:open).and_return(readable)
         file = File.dirname(__FILE__) + "/missing.json"
         FileUtils.rm(file) if File.exists?(file)
-        result = U.parse_json_from_file_or_uri(file, "fake", :quiet => true)
+        result = U.parse_file_or_uri(:json, file, "fake", :quiet => true)
         FileUtils.rm(file) if File.exists?(file)
         result.should == { "stations" => ["Metro Center", "Dupont Circle"] }
       end
@@ -243,16 +243,16 @@ describe "Utility" do
   end
 
   describe "xml" do
-    describe "parse_xml_from_file" do
+    describe "parse_file" do
       it "should work" do
         file = File.dirname(__FILE__) + '/test.xml'
-        parsed = U.parse_xml_from_file(file)
+        parsed = U.parse_file(:xml, file)
         result = parsed.xpath('.//stations/station').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
       end
     end
 
-    describe "parse_xml_from_uri" do
+    describe "parse_uri" do
       it "should work" do
         readable = Object.new
         readable.stub(:read).and_return(%(
@@ -262,16 +262,16 @@ describe "Utility" do
           </stations>
         ))
         U.stub(:open).and_return(readable)
-        parsed = U.parse_xml_from_uri("fake", :quiet => true)
+        parsed = U.parse_uri(:xml, "fake", :quiet => true)
         result = parsed.xpath('.//stations/station').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
       end
     end
 
-    describe "parse_xml_from_file_or_uri" do
+    describe "parse_file_or_uri" do
       it "should work when file present" do
         file = File.dirname(__FILE__) + '/test.xml'
-        parsed = U.parse_xml_from_file_or_uri(file, "fake", :quiet => true)
+        parsed = U.parse_file_or_uri(:xml, file, "fake", :quiet => true)
         result = parsed.xpath('.//stations/station').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]
       end
@@ -287,7 +287,7 @@ describe "Utility" do
         U.stub(:open).and_return(readable)
         file = File.dirname(__FILE__) + "/missing.xml"
         FileUtils.rm(file) if File.exists?(file)
-        parsed = U.parse_xml_from_file_or_uri(file, "fake", :quiet => true)
+        parsed = U.parse_file_or_uri(:xml, file, "fake", :quiet => true)
         FileUtils.rm(file) if File.exists?(file)
         result = parsed.xpath('.//stations/station').map(&:content)
         result.should == ["Metro Center", "Dupont Circle"]

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: datacatalog-importer
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 19
   prerelease: false
   segments: 
   - 0
-  - 2
   - 3
-  version: 0.2.3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - David James