datacatalog-importer 0.1.17 → 0.1.18

data/VERSION

@@ -1 +1 @@
-0.1.17
+0.1.18

data/datacatalog-importer.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.1.17"
+  s.version = "0.1.18"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
-  s.date = %q{2010-05-05}
+  s.date = %q{2010-05-06}
   s.description = %q{This framework makes it easier to write importers for the National Data Catalog.}
   s.email = %q{djames@sunlightfoundation.com}
   s.extra_rdoc_files = [

data/lib/utility.rb

@@ -8,8 +8,8 @@ module DataCatalog
 
       # == URLs ==
 
-      def self.absolute_url(page_url, url)
-        plain_string(URI.parse(page_url).merge(url).to_s)
+      def self.absolute_url(base_url, url)
+        plain_string(URI.parse(base_url).merge(url).to_s)
       end
       
       def self.normalize_url(url)
@@ -50,7 +50,7 @@ module DataCatalog
 
       def self.headers
         {
-          "UserAgent" => "National Data Catalog Importer/0.1.17",
+          "UserAgent" => "National Data Catalog Importer/0.1.18",
         }
       end
 
@@ -62,17 +62,18 @@ module DataCatalog
       def self.fetch(uri, options={})
         max_attempts = options[:max_attempts] || 3
         retry_delay  = options[:retry_delay] || 5
+        quiet        = options[:quiet] || false
         attempts = 0
         loop do
           begin
-            puts "Fetching #{uri}..."
+            puts "Fetching #{uri}..." unless quiet
             io = open(uri, headers)
             return io.read
           rescue Timeout::Error, StandardError => e
             attempts += 1
-            puts "  Attempt ##{attempts} failed."
-            puts "  Error: #{e.inspect}"
-            break if attempts >= max_attempts
+            puts "  Attempt ##{attempts} failed." unless quiet
+            puts "  Error: #{e.inspect}" unless quiet
+            return nil if attempts >= max_attempts
             sleep(retry_delay)
           end
         end

data/spec/utility_spec.rb

@@ -6,16 +6,93 @@ describe "Utility" do
 
   describe "normalize_url" do
     it "add trailing slash if missing" do
-      U.normalize_url("sunlightlabs.com").should == "http://sunlightlabs.com/"
+      U.normalize_url("sunlightlabs.com").should ==
+        "http://sunlightlabs.com/"
     end
 
     it "lowercases" do
-      U.normalize_url("http://SunlightLabs.com/").should == "http://sunlightlabs.com/"
+      U.normalize_url("http://SunlightLabs.com/").should ==
+        "http://sunlightlabs.com/"
     end
 
     it "adds http if missing" do
-      U.normalize_url("sunlightlabs.com/").should == "http://sunlightlabs.com/"
+      U.normalize_url("sunlightlabs.com/").should ==
+        "http://sunlightlabs.com/"
     end
   end
   
+  describe "absolute_url" do
+    it "should work" do
+      U.absolute_url("http://sunlightlabs.com", "/contact").should ==
+        "http://sunlightlabs.com/contact"
+    end
+  end
+  
+  describe "single_line_clean" do
+    it "should clean up leading and trailing whitespace" do
+      U.single_line_clean("\t \ttext\t\t ").should == "text"
+    end
+
+    it "should clean up leading and trailing newlines" do
+      U.single_line_clean("\n\ntext\n\n").should == "text"
+    end
+
+    it "should clean up all newlines" do
+      U.single_line_clean("sunlight\nlabs").should == "sunlight labs"
+    end
+  end
+  
+  describe "multi_line_clean" do
+    it "should remove leading and trailing newlines" do
+      input = "\nline 1\nline 2\nline 3\n"
+      U.multi_line_clean(input).should == "line 1\nline 2\nline 3"
+    end
+  end
+  
+  describe "fetch" do
+    before do
+      @readable = Object.new
+      @readable.stub(:read).and_return("result")
+
+      @sleep_count = 0
+      U.stub(:sleep).and_return {
+        @sleep_count += 1
+      }
+    end
+
+    it "should work" do
+      U.stub(:open).and_return(@readable)
+      U.fetch("fake", :quiet => true).should == "result"
+    end
+    
+    it "bad fetches below retry limit are ok" do
+      @count = 0
+      U.stub(:open).and_return {
+        @count += 1
+        if @count <= 2
+          raise StandardError
+        else
+          @readable
+        end
+      }
+      U.fetch("fake", :max_attempts => 3, :quiet => true).should == "result"
+      @sleep_count.should == 2
+    end
+
+    it "bad fetches above retry limit give nil" do
+      @count = 0
+      U.stub(:open).and_return {
+        @count += 1
+        if @count <= 2
+          raise StandardError
+        else
+          @readable
+        end
+      }
+      U.fetch("fake", :max_attempts => 2, :quiet => true).should == nil
+      @sleep_count.should == 1
+    end
+    
+  end
+  
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 17
-  version: 0.1.17
+  - 18
+  version: 0.1.18
 platform: ruby
 authors: 
 - David James
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-05 00:00:00 -04:00
+date: 2010-05-06 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency