coupler 0.0.3-java → 0.0.4-java

Sign up to get free protection for your applications and to get access to all the features.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.3
1
+ 0.0.4
@@ -0,0 +1,13 @@
1
+ Sequel.migration do
2
+ up do
3
+ alter_table(:jobs) do
4
+ add_column :import_id, Integer
5
+ end
6
+ end
7
+
8
+ down do
9
+ alter_table(:jobs) do
10
+ drop_column :import_id
11
+ end
12
+ end
13
+ end
@@ -2,21 +2,23 @@ module Coupler
2
2
  # This class is used during resource transformation. Its purpose
3
3
  # is for mass inserts into the local database for speed.
4
4
  class ImportBuffer
5
+ MAX_QUERY_SIZE = 1_048_576
6
+
5
7
  attr_writer :dataset
8
+
6
9
  def initialize(columns, dataset, &progress)
7
10
  @columns = columns
8
11
  @dataset = dataset
9
12
  @mutex = Mutex.new
10
13
  @progress = progress
11
14
  @pending = 0
12
- @max_query_size = 1_048_576
13
15
  end
14
16
 
15
17
  def add(row)
16
18
  fragment = " " + @dataset.literal(row.is_a?(Hash) ? row.values_at(*@columns) : row) + ","
17
19
  @mutex.synchronize do
18
20
  init_query if @query.nil?
19
- if (@query.length + fragment.length) > @max_query_size
21
+ if (@query.length + fragment.length) > MAX_QUERY_SIZE
20
22
  flush(false)
21
23
  init_query
22
24
  end
@@ -41,7 +43,7 @@ module Coupler
41
43
 
42
44
  private
43
45
  def init_query
44
- @query = String.alloc(@max_query_size)
46
+ @query = String.alloc(MAX_QUERY_SIZE)
45
47
  @query << @dataset.insert_sql(@columns, Sequel::LiteralString.new('VALUES'))
46
48
  end
47
49
  end
@@ -2,6 +2,7 @@ module Coupler
2
2
  module Models
3
3
  class Import < Sequel::Model
4
4
  include CommonModel
5
+ include Jobify
5
6
 
6
7
  # NOTE: yoinked from FasterCSV
7
8
  # A Regexp used to find and convert some common Date formats.
@@ -84,6 +85,11 @@ module Coupler
84
85
  next
85
86
  end
86
87
 
88
+ # convert values
89
+ row.each_with_index do |value, i|
90
+ row[i] = coerce(value, field_types[i])
91
+ end
92
+
87
93
  key = row[primary_key_index]
88
94
  num = key_frequencies[key] += 1
89
95
  row.push(num > 1 ? num : nil)
@@ -233,6 +239,20 @@ module Coupler
233
239
  end
234
240
  end
235
241
  end
242
+
243
+ def coerce(value, type)
244
+ return nil if value.nil?
245
+
246
+ case type
247
+ when "integer"
248
+ chr = value[0]
249
+ return nil if chr.nil?
250
+ ord = chr.ord
251
+ (ord.nil? || ord < 48 || ord > 57) ? nil : value.to_i
252
+ else
253
+ value
254
+ end
255
+ end
236
256
  end
237
257
  end
238
258
  end
@@ -75,4 +75,36 @@ class TestImport < Coupler::Test::IntegrationTest
75
75
  assert db.schema(:"import_#{import.id}").assoc(:id)[1][:primary_key]
76
76
  end
77
77
  end
78
+
79
+ test "handling bad integers" do
80
+ tempfile = Tempfile.new('coupler-import')
81
+ tempfile.write("id,foo,bar,baz\n1,2,3,4\n2,3,4,5\n3,4,5,6\n4,junk,,''\n5,,,\n")
82
+ tempfile.close
83
+
84
+ project = Project.create(:name => "foo")
85
+ import = Import.new({:data => file_upload(tempfile.path), :project => project })
86
+ import.field_types = %w{integer integer integer integer}
87
+ import.save!
88
+ import.import!
89
+
90
+ import.dataset do |ds|
91
+ row = ds[:id => 4]
92
+ assert_nil row[:foo]
93
+ assert_nil row[:bar]
94
+ assert_nil row[:baz]
95
+
96
+ row = ds[:id => 5]
97
+ assert_nil row[:foo]
98
+ assert_nil row[:bar]
99
+ assert_nil row[:baz]
100
+ end
101
+ end
102
+
103
+ test "handling bad dates" do
104
+ pend
105
+ end
106
+
107
+ test "handling variable row length" do
108
+ pend
109
+ end
78
110
  end
@@ -125,6 +125,10 @@ module Coupler
125
125
  assert_equal "id", import.primary_key_name
126
126
  assert import.has_headers
127
127
  end
128
+
129
+ test "jobified" do
130
+ assert Import.ancestors.include?(Jobify)
131
+ end
128
132
  end
129
133
  end
130
134
  end
@@ -79,6 +79,10 @@ module Coupler
79
79
  assert_respond_to Resource.new, :scheduled_jobs
80
80
  end
81
81
 
82
+ test "jobified" do
83
+ assert Resource.ancestors.include?(Jobify)
84
+ end
85
+
82
86
  test "nested attributes for connection" do
83
87
  assert_respond_to Resource.new, :connection_attributes=
84
88
  end
@@ -0,0 +1,47 @@
1
+ require 'helper'
2
+
3
+ class TestImportBuffer < Coupler::Test::UnitTest
4
+ include Coupler
5
+
6
+ def setup
7
+ super
8
+ @database = stub('database', :run => nil)
9
+ @dataset = stub('dataset', {
10
+ :insert_sql => "INSERT INTO \"FOO\" (\"BAR\") VALUES",
11
+ :db => @database,
12
+ :first_source_alias => :foo
13
+ })
14
+ end
15
+
16
+ test "single insert" do
17
+ @dataset.expects(:insert_sql).with([:bar], Sequel::LiteralString.new("VALUES")).returns("INSERT INTO \"FOO\" (\"BAR\") VALUES")
18
+ @dataset.expects(:literal).with([123]).returns("(123)")
19
+ @database.expects(:run).with("INSERT INTO \"FOO\" (\"BAR\") VALUES (123)")
20
+ buffer = ImportBuffer.new([:bar], @dataset)
21
+ buffer.add({:bar => 123})
22
+ buffer.flush
23
+ end
24
+
25
+ test "multiple insert" do
26
+ @dataset.expects(:literal).with([123]).returns("(123)")
27
+ @dataset.expects(:literal).with([456]).returns("(456)")
28
+ @database.expects(:run).with("INSERT INTO \"FOO\" (\"BAR\") VALUES (123), (456)")
29
+ buffer = ImportBuffer.new([:bar], @dataset)
30
+ buffer.add({:bar => 123})
31
+ buffer.add({:bar => 456})
32
+ buffer.flush
33
+ end
34
+
35
+ test "max query size / auto-flush" do
36
+ size = ImportBuffer::MAX_QUERY_SIZE - 50
37
+ str = "x" * size
38
+
39
+ # bar is a string this time
40
+ @dataset.expects(:literal).twice.with([str]).returns("(#{str})")
41
+ @database.expects(:run).twice
42
+ buffer = ImportBuffer.new([:bar], @dataset)
43
+ buffer.add({:bar => str})
44
+ buffer.add({:bar => str})
45
+ buffer.flush
46
+ end
47
+ end
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: coupler
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.0.3
5
+ version: 0.0.4
6
6
  platform: java
7
7
  authors:
8
8
  - Jeremy Stephens
@@ -10,7 +10,7 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2011-05-13 00:00:00 -05:00
13
+ date: 2011-05-16 00:00:00 -05:00
14
14
  default_executable: coupler
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
@@ -303,6 +303,7 @@ files:
303
303
  - db/migrate/020_rename_import_columns.rb
304
304
  - db/migrate/021_add_fields_to_connections.rb
305
305
  - db/migrate/022_remove_database_name_from_resources.rb
306
+ - db/migrate/023_add_import_jobs.rb
306
307
  - features/connections.feature
307
308
  - features/matchers.feature
308
309
  - features/projects.feature
@@ -414,6 +415,7 @@ files:
414
415
  - test/unit/test_data_uploader.rb
415
416
  - test/unit/test_database.rb
416
417
  - test/unit/test_helpers.rb
418
+ - test/unit/test_import_buffer.rb
417
419
  - test/unit/test_logger.rb
418
420
  - test/unit/test_models.rb
419
421
  - test/unit/test_runner.rb