coupler 0.0.3-java → 0.0.4-java

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/db/migrate/023_add_import_jobs.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  up do
+    alter_table(:jobs) do
+      add_column :import_id, Integer
+    end
+  end
+
+  down do
+    alter_table(:jobs) do
+      drop_column :import_id
+    end
+  end
+end

data/lib/coupler/import_buffer.rb

@@ -2,21 +2,23 @@ module Coupler
   # This class is used during resource transformation.  Its purpose
   # is for mass inserts into the local database for speed.
   class ImportBuffer
+    MAX_QUERY_SIZE = 1_048_576
+
     attr_writer :dataset
+
     def initialize(columns, dataset, &progress)
       @columns = columns
       @dataset = dataset
       @mutex = Mutex.new
       @progress = progress
       @pending = 0
-      @max_query_size = 1_048_576
     end
 
     def add(row)
       fragment = " " + @dataset.literal(row.is_a?(Hash) ? row.values_at(*@columns) : row) + ","
       @mutex.synchronize do
         init_query  if @query.nil?
-        if (@query.length + fragment.length) > @max_query_size
+        if (@query.length + fragment.length) > MAX_QUERY_SIZE
           flush(false)
           init_query
         end
@@ -41,7 +43,7 @@ module Coupler
 
     private
       def init_query
-        @query = String.alloc(@max_query_size)
+        @query = String.alloc(MAX_QUERY_SIZE)
         @query << @dataset.insert_sql(@columns, Sequel::LiteralString.new('VALUES'))
       end
   end

data/lib/coupler/models/import.rb

@@ -2,6 +2,7 @@ module Coupler
   module Models
     class Import < Sequel::Model
       include CommonModel
+      include Jobify
 
       # NOTE: yoinked from FasterCSV
       # A Regexp used to find and convert some common Date formats.
@@ -84,6 +85,11 @@ module Coupler
               next
             end
 
+            # convert values
+            row.each_with_index do |value, i|
+              row[i] = coerce(value, field_types[i])
+            end
+
             key = row[primary_key_index]
             num = key_frequencies[key] += 1
             row.push(num > 1 ? num : nil)
@@ -233,6 +239,20 @@ module Coupler
             end
           end
         end
+
+        def coerce(value, type)
+          return nil  if value.nil?
+
+          case type
+          when "integer"
+            chr = value[0]
+            return nil  if chr.nil?
+            ord = chr.ord
+            (ord.nil? || ord < 48 || ord > 57) ? nil : value.to_i
+          else
+            value
+          end
+        end
     end
   end
 end

data/test/integration/test_import.rb

@@ -75,4 +75,36 @@ class TestImport < Coupler::Test::IntegrationTest
       assert db.schema(:"import_#{import.id}").assoc(:id)[1][:primary_key]
     end
   end
+
+  test "handling bad integers" do
+    tempfile = Tempfile.new('coupler-import')
+    tempfile.write("id,foo,bar,baz\n1,2,3,4\n2,3,4,5\n3,4,5,6\n4,junk,,''\n5,,,\n")
+    tempfile.close
+
+    project = Project.create(:name => "foo")
+    import = Import.new({:data => file_upload(tempfile.path), :project => project })
+    import.field_types = %w{integer integer integer integer}
+    import.save!
+    import.import!
+
+    import.dataset do |ds|
+      row = ds[:id => 4]
+      assert_nil row[:foo]
+      assert_nil row[:bar]
+      assert_nil row[:baz]
+
+      row = ds[:id => 5]
+      assert_nil row[:foo]
+      assert_nil row[:bar]
+      assert_nil row[:baz]
+    end
+  end
+
+  test "handling bad dates" do
+    pend
+  end
+
+  test "handling variable row length" do
+    pend
+  end
 end

data/test/unit/models/test_import.rb

@@ -125,6 +125,10 @@ module Coupler
         assert_equal "id", import.primary_key_name
         assert import.has_headers
       end
+
+      test "jobified" do
+        assert Import.ancestors.include?(Jobify)
+      end
     end
   end
 end

data/test/unit/models/test_resource.rb

@@ -79,6 +79,10 @@ module Coupler
         assert_respond_to Resource.new, :scheduled_jobs
       end
 
+      test "jobified" do
+        assert Resource.ancestors.include?(Jobify)
+      end
+
       test "nested attributes for connection" do
         assert_respond_to Resource.new, :connection_attributes=
       end

data/test/unit/test_import_buffer.rb

@@ -0,0 +1,47 @@
+require 'helper'
+
+class TestImportBuffer < Coupler::Test::UnitTest
+  include Coupler
+
+  def setup
+    super
+    @database = stub('database', :run => nil)
+    @dataset = stub('dataset', {
+      :insert_sql => "INSERT INTO \"FOO\" (\"BAR\") VALUES",
+      :db => @database,
+      :first_source_alias => :foo
+    })
+  end
+
+  test "single insert" do
+    @dataset.expects(:insert_sql).with([:bar], Sequel::LiteralString.new("VALUES")).returns("INSERT INTO \"FOO\" (\"BAR\") VALUES")
+    @dataset.expects(:literal).with([123]).returns("(123)")
+    @database.expects(:run).with("INSERT INTO \"FOO\" (\"BAR\") VALUES (123)")
+    buffer = ImportBuffer.new([:bar], @dataset)
+    buffer.add({:bar => 123})
+    buffer.flush
+  end
+
+  test "multiple insert" do
+    @dataset.expects(:literal).with([123]).returns("(123)")
+    @dataset.expects(:literal).with([456]).returns("(456)")
+    @database.expects(:run).with("INSERT INTO \"FOO\" (\"BAR\") VALUES (123), (456)")
+    buffer = ImportBuffer.new([:bar], @dataset)
+    buffer.add({:bar => 123})
+    buffer.add({:bar => 456})
+    buffer.flush
+  end
+
+  test "max query size / auto-flush" do
+    size = ImportBuffer::MAX_QUERY_SIZE - 50
+    str = "x" * size
+
+    # bar is a string this time
+    @dataset.expects(:literal).twice.with([str]).returns("(#{str})")
+    @database.expects(:run).twice
+    buffer = ImportBuffer.new([:bar], @dataset)
+    buffer.add({:bar => str})
+    buffer.add({:bar => str})
+    buffer.flush
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: coupler
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.3
+  version: 0.0.4
 platform: java
 authors: 
   - Jeremy Stephens
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-13 00:00:00 -05:00
+date: 2011-05-16 00:00:00 -05:00
 default_executable: coupler
 dependencies: 
   - !ruby/object:Gem::Dependency 
@@ -303,6 +303,7 @@ files:
   - db/migrate/020_rename_import_columns.rb
   - db/migrate/021_add_fields_to_connections.rb
   - db/migrate/022_remove_database_name_from_resources.rb
+  - db/migrate/023_add_import_jobs.rb
   - features/connections.feature
   - features/matchers.feature
   - features/projects.feature
@@ -414,6 +415,7 @@ files:
   - test/unit/test_data_uploader.rb
   - test/unit/test_database.rb
   - test/unit/test_helpers.rb
+  - test/unit/test_import_buffer.rb
   - test/unit/test_logger.rb
   - test/unit/test_models.rb
   - test/unit/test_runner.rb