datacatalog-importer 0.1.16 → 0.1.17

data/VERSION

@@ -1 +1 @@
-0.1.16
+0.1.17

data/datacatalog-importer.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{datacatalog-importer}
-  s.version = "0.1.16"
+  s.version = "0.1.17"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["David James"]
@@ -32,9 +32,9 @@ Gem::Specification.new do |s|
      "lib/sort_yaml_hash.rb",
      "lib/tasks.rb",
      "lib/utility.rb",
-     "spec/datacatalog-importer_spec.rb",
      "spec/spec.opts",
-     "spec/spec_helper.rb"
+     "spec/spec_helper.rb",
+     "spec/utility_spec.rb"
   ]
   s.homepage = %q{http://github.com/djsun/datacatalog-importer}
   s.rdoc_options = ["--charset=UTF-8"]
@@ -42,8 +42,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.6}
   s.summary = %q{A framework to write National Data Catalog importers}
   s.test_files = [
-    "spec/datacatalog-importer_spec.rb",
-     "spec/spec_helper.rb"
+    "spec/spec_helper.rb",
+     "spec/utility_spec.rb"
   ]
 
   if s.respond_to? :specification_version then

data/lib/pusher.rb

@@ -87,7 +87,14 @@ module DataCatalog
         n = docs.length
         if n == 0
           puts "Creating Organization: #{name}"
-          DataCatalog::Organization.create(data)
+          begin
+            DataCatalog::Organization.create(data)
+          rescue DataCatalog::BadRequest => e
+            error("Cannot create Organization", {
+              :params => data,
+              :errors => e.errors,
+            })
+          end
         else
           if n > 1
             warning("Cannot find unique Source with url : #{url}", {
@@ -96,7 +103,14 @@ module DataCatalog
             })
           end
           puts "Updating Organization: #{name}"
-          DataCatalog::Organization.update(docs[0].id, data)
+          begin
+            DataCatalog::Organization.update(docs[0].id, data)
+          rescue DataCatalog::BadRequest => e
+            error("Cannot update Organization with id : #{docs[0].id}", {
+              :params => data,
+              :errors => e.errors,
+            })
+          end
         end
       end
       
@@ -161,10 +175,24 @@ module DataCatalog
         case n
         when 0
           puts "- Creating Download: #{data[:format]}"
-          DataCatalog::Download.create(data)
+          begin
+            DataCatalog::Download.create(data)
+          rescue DataCatalog::BadRequest => e
+            error("Cannot create Download", {
+              :params => data,
+              :errors => e.errors,
+            })
+          end
         when 1
           puts "- Updating Download: #{data[:format]}"
-          DataCatalog::Download.update(docs[0].id, data)
+          begin
+            DataCatalog::Download.update(docs[0].id, data)
+          rescue DataCatalog::BadRequest => e
+            error("Cannot update Download with id : #{docs[0].id}", {
+              :params => data,
+              :errors => e.errors,
+            })
+          end
         else
           error("Cannot find unique Download with source_id : #{source.id} and format : #{data[:format]}", {
             :error => "#{n} matches: " + docs.map { |x| x.id }.join(" ")
@@ -232,6 +260,8 @@ module DataCatalog
           :object => object,
         })
         nil
+      rescue DataCatalog::BadRequest => e
+        raise Error, "Could not upload Report to API: #{e.errors.inspect}"
       end
       
     end

data/lib/utility.rb

@@ -9,7 +9,15 @@ module DataCatalog
       # == URLs ==
 
       def self.absolute_url(page_url, url)
-        Utility.plain_string(URI.parse(page_url).merge(url).to_s)
+        plain_string(URI.parse(page_url).merge(url).to_s)
+      end
+      
+      def self.normalize_url(url)
+        uri = URI.parse(url).normalize
+        unless uri.scheme
+          uri = URI.parse("http://#{url}").normalize
+        end
+        uri.to_s
       end
 
       # == Cleaning ==
@@ -42,7 +50,7 @@ module DataCatalog
 
       def self.headers
         {
-          "UserAgent" => "National Data Catalog Importer/0.1.16",
+          "UserAgent" => "National Data Catalog Importer/0.1.17",
         }
       end
 
@@ -51,7 +59,9 @@ module DataCatalog
       # For background on rescuing net/http errors, see:
       # * http://jerith.livejournal.com/40063.html
       # * http://lindsaar.net/2007/12/9/rbuf_filltimeout-error
-      def self.fetch(uri, max_attempts=3)
+      def self.fetch(uri, options={})
+        max_attempts = options[:max_attempts] || 3
+        retry_delay  = options[:retry_delay] || 5
         attempts = 0
         loop do
           begin
@@ -63,6 +73,7 @@ module DataCatalog
             puts "  Attempt ##{attempts} failed."
             puts "  Error: #{e.inspect}"
             break if attempts >= max_attempts
+            sleep(retry_delay)
           end
         end
       end

data/spec/utility_spec.rb

@@ -0,0 +1,21 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+U = DataCatalog::ImporterFramework::Utility
+
+describe "Utility" do
+
+  describe "normalize_url" do
+    it "add trailing slash if missing" do
+      U.normalize_url("sunlightlabs.com").should == "http://sunlightlabs.com/"
+    end
+
+    it "lowercases" do
+      U.normalize_url("http://SunlightLabs.com/").should == "http://sunlightlabs.com/"
+    end
+
+    it "adds http if missing" do
+      U.normalize_url("sunlightlabs.com/").should == "http://sunlightlabs.com/"
+    end
+  end
+  
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 16
-  version: 0.1.16
+  - 17
+  version: 0.1.17
 platform: ruby
 authors: 
 - David James
@@ -84,9 +84,9 @@ files:
 - lib/sort_yaml_hash.rb
 - lib/tasks.rb
 - lib/utility.rb
-- spec/datacatalog-importer_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
+- spec/utility_spec.rb
 has_rdoc: true
 homepage: http://github.com/djsun/datacatalog-importer
 licenses: []
@@ -118,5 +118,5 @@ signing_key:
 specification_version: 3
 summary: A framework to write National Data Catalog importers
 test_files: 
-- spec/datacatalog-importer_spec.rb
 - spec/spec_helper.rb
+- spec/utility_spec.rb

data/spec/datacatalog-importer_spec.rb

@@ -1,7 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-
-describe "DatacatalogImporter" do
-  it "fails" do
-    fail "hey buddy, you should probably rename this file and start specing for real"
-  end
-end