coupler 0.0.4-java → 0.0.6-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. data/Gemfile +7 -8
  2. data/Gemfile.lock +43 -24
  3. data/VERSION +1 -1
  4. data/coupler.gemspec +27 -31
  5. data/features/wizard.feature +2 -1
  6. data/lib/coupler.rb +2 -2
  7. data/lib/coupler/base.rb +4 -0
  8. data/lib/coupler/extensions/connections.rb +2 -12
  9. data/lib/coupler/extensions/jobs.rb +4 -2
  10. data/lib/coupler/extensions/projects.rb +1 -1
  11. data/lib/coupler/helpers.rb +9 -1
  12. data/lib/coupler/models.rb +8 -0
  13. data/lib/coupler/models/comparison.rb +5 -4
  14. data/lib/coupler/models/connection.rb +10 -1
  15. data/lib/coupler/models/import.rb +3 -7
  16. data/lib/coupler/models/transformer.rb +1 -1
  17. data/lib/coupler/runner.rb +1 -1
  18. data/lib/coupler/scheduler.rb +1 -1
  19. data/tasks/test.rake +8 -0
  20. data/test/functional/test_base.rb +17 -0
  21. data/test/functional/test_connections.rb +81 -0
  22. data/test/functional/test_imports.rb +76 -0
  23. data/test/{integration/extensions → functional}/test_jobs.rb +21 -12
  24. data/test/functional/test_matchers.rb +108 -0
  25. data/test/functional/test_projects.rb +67 -0
  26. data/test/functional/test_resources.rb +126 -0
  27. data/test/{integration/extensions → functional}/test_results.rb +20 -29
  28. data/test/functional/test_scenarios.rb +92 -0
  29. data/test/functional/test_transformations.rb +106 -0
  30. data/test/functional/test_transformers.rb +68 -0
  31. data/test/helper.rb +30 -20
  32. data/test/integration/test_transformation.rb +6 -1
  33. data/test/unit/models/test_common_model.rb +2 -2
  34. data/test/unit/models/test_comparison.rb +8 -8
  35. data/test/unit/models/test_connection.rb +2 -2
  36. data/test/unit/models/test_field.rb +2 -2
  37. data/test/unit/models/test_import.rb +9 -4
  38. data/test/unit/models/test_job.rb +2 -2
  39. data/test/unit/models/test_matcher.rb +2 -2
  40. data/test/unit/models/test_project.rb +2 -2
  41. data/test/unit/models/test_resource.rb +2 -2
  42. data/test/unit/models/test_result.rb +2 -2
  43. data/test/unit/models/test_scenario.rb +2 -2
  44. data/test/unit/models/test_transformation.rb +2 -2
  45. data/test/unit/models/test_transformer.rb +12 -2
  46. data/test/unit/test_base.rb +1 -14
  47. data/test/unit/test_data_uploader.rb +1 -1
  48. data/test/unit/test_database.rb +1 -1
  49. data/test/unit/test_helpers.rb +2 -2
  50. data/test/unit/test_import_buffer.rb +40 -38
  51. data/test/unit/test_logger.rb +1 -1
  52. data/test/unit/test_models.rb +1 -1
  53. data/test/unit/test_runner.rb +1 -1
  54. data/test/unit/test_scheduler.rb +1 -1
  55. data/webroot/public/css/style.css +7 -5
  56. data/webroot/public/js/jquery.dataTables.min.js +130 -574
  57. data/webroot/views/connections/new.erb +41 -3
  58. data/webroot/views/imports/new.erb +2 -0
  59. data/webroot/views/jobs/list.erb +25 -21
  60. data/webroot/views/projects/index.erb +23 -15
  61. data/webroot/views/resources/list.erb +1 -2
  62. data/webroot/views/resources/new.erb +2 -2
  63. data/webroot/views/resources/show.erb +5 -2
  64. data/webroot/views/scenarios/new.erb +1 -1
  65. data/webroot/views/transformations/new.erb +1 -1
  66. metadata +30 -44
  67. data/lib/coupler/config.rb +0 -128
  68. data/test/coupler/models/test_import.rb +0 -221
  69. data/test/factories.rb +0 -91
  70. data/test/integration/extensions/test_connections.rb +0 -80
  71. data/test/integration/extensions/test_imports.rb +0 -94
  72. data/test/integration/extensions/test_matchers.rb +0 -134
  73. data/test/integration/extensions/test_projects.rb +0 -82
  74. data/test/integration/extensions/test_resources.rb +0 -150
  75. data/test/integration/extensions/test_scenarios.rb +0 -88
  76. data/test/integration/extensions/test_transformations.rb +0 -113
  77. data/test/integration/extensions/test_transformers.rb +0 -80
  78. data/vendor/h2-1.3.154.jar +0 -0
@@ -1,221 +0,0 @@
1
- require File.dirname(__FILE__) + '/../../helper'
2
-
3
- module Coupler
4
- module Models
5
- class TestImport < Test::Unit::TestCase
6
- def test_sequel_model
7
- assert_equal ::Sequel::Model, Models::Import.superclass
8
- assert_equal :imports, Import.table_name
9
- end
10
-
11
- def test_many_to_one_project
12
- assert_respond_to Models::Import.new, :project
13
- end
14
-
15
- def test_gets_name_from_original_filename
16
- import = Import.new(:data => fixture_file_upload('people.csv'))
17
- assert_equal "People", import.name
18
- end
19
-
20
- def test_preview_with_headers
21
- import = Factory.build(:import)
22
- preview = import.preview
23
- assert_kind_of Array, preview
24
- assert_equal 50, preview.length
25
- assert_kind_of Array, preview[0]
26
- assert_not_equal %w{id first_name last_name age}, preview[0]
27
- end
28
-
29
- def test_discovers_field_names_and_types
30
- import = Factory.build(:import, :data => fixture_file_upload("people.csv"))
31
- expected_types = %w{integer string string integer}
32
- expected_names = %w{id first_name last_name age}
33
- assert_equal expected_names, import.field_names
34
- assert_equal expected_types, import.field_types
35
- assert_equal "id", import.primary_key_name
36
- assert import.has_headers
37
- end
38
-
39
- def test_discover_for_csv_with_no_headers
40
- tempfile = Tempfile.new('coupler-import')
41
- tempfile.write("foo,bar,1,2,3\njunk,blah,4,5,6")
42
- tempfile.close
43
- import = Factory.build(:import, :data => file_upload(tempfile.path))
44
- expected_types = %w{string string integer integer integer}
45
- assert_equal expected_types, import.field_types
46
- assert_nil import.field_names
47
- assert_nil import.primary_key_name
48
- assert !import.has_headers
49
- end
50
-
51
- def test_import!
52
- project = Factory(:project)
53
- import = Factory(:import, :data => fixture_file_upload("people.csv"), :project => project)
54
- now = Time.now
55
- Timecop.freeze(now) do
56
- assert import.import!
57
- assert_equal now, import.occurred_at
58
- end
59
-
60
- project.local_database do |db|
61
- name = :"import_#{import.id}"
62
- assert db.tables.include?(name)
63
- schema = db.schema(name)
64
- assert_equal [:integer, true], schema.assoc(:id)[1].values_at(:type, :primary_key)
65
- assert_equal :string, schema.assoc(:first_name)[1][:type]
66
- assert_equal :string, schema.assoc(:last_name)[1][:type]
67
- assert_equal :integer, schema.assoc(:age)[1][:type]
68
-
69
- ds = db[name]
70
- assert_equal 50, ds.count
71
- end
72
- end
73
-
74
- def test_requires_field_names
75
- import = Factory.build(:import, :data => fixture_file_upload('no-headers.csv'))
76
- assert_nil import.field_names
77
- assert !import.valid?
78
- end
79
-
80
- def test_requires_primary_key_name
81
- import = Factory.build(:import, :data => fixture_file_upload('no-headers.csv'))
82
- import.field_names = %w{id first_name last_name age}
83
- assert !import.valid?
84
- end
85
-
86
- def test_requires_valid_primary_key_name
87
- import = Factory.build(:import, :data => fixture_file_upload('no-headers.csv'))
88
- import.field_names = %w{id first_name last_name age}
89
- import.primary_key_name = "foo"
90
- assert !import.valid?
91
- end
92
-
93
- def test_flags_duplicate_primary_keys
94
- tempfile = Tempfile.new('coupler-import')
95
- tempfile.write("id,foo,bar\n1,abc,def\n2,ghi,jkl\n2,mno,pqr")
96
- tempfile.close
97
-
98
- project = Factory(:project)
99
- import = Factory(:import, :data => file_upload(tempfile.path), :project => project)
100
-
101
- now = Time.at(Time.now.to_i) # dumb usecs
102
- Timecop.freeze(now) do
103
- assert !import.import!
104
- assert import.has_duplicate_keys
105
- assert_equal now, import.occurred_at, "now: %d-%d; occurred_at: %d-%d" % [now.to_i, now.usec, import.occurred_at.to_i, import.occurred_at.usec]
106
- end
107
-
108
- project.local_database do |db|
109
- ds = db[:"import_#{import.id}"]
110
- assert ds.filter(:id => 2).select_map(:dup_key_count).all?
111
- end
112
- end
113
-
114
- def test_requires_unique_field_names
115
- tempfile = Tempfile.new('coupler-import')
116
- tempfile.write("id,foo,foo\n1,abc,def\n2,ghi,jkl\n3,mno,pqr")
117
- tempfile.close
118
-
119
- import = Factory.build(:import, :data => file_upload(tempfile.path))
120
- assert !import.valid?
121
- end
122
-
123
- def test_requires_unused_resource_name
124
- project = Factory(:project)
125
- resource = Factory(:resource, :name => "Foo", :project => project)
126
- import = Factory.build(:import, :data => fixture_file_upload('people.csv'), :name => "Foo", :project => project)
127
- assert !import.valid?
128
- end
129
-
130
- def test_dataset
131
- project = Factory(:project)
132
- import = Factory(:import, :project => project)
133
- import.import!
134
- project.local_database do |db|
135
- import.dataset do |ds|
136
- expected = db[:"import_#{import.id}"]
137
- assert_equal expected.first_source, ds.first_source
138
- assert_equal db.uri, ds.db.uri
139
- end
140
- end
141
- end
142
-
143
- def test_filenames_dont_conflict
144
- dir = make_tmpdir
145
- filename = File.join(dir, "people.csv")
146
- tempfile = File.open(filename, 'w')
147
- tempfile.puts("id,foo,bar\n")
148
- tempfile.puts("abc,def,ghi\n")
149
- tempfile.close
150
- import_1 = Factory(:import, :data => fixture_file_upload('people.csv'))
151
- import_2 = Factory(:import, :data => file_upload(filename))
152
- assert_not_equal import_1.data.store_path, import_2.data.store_path
153
- end
154
-
155
- def test_repair_duplicate_keys
156
- project = Factory(:project)
157
- import = Factory(:import, :data => fixture_file_upload('duplicate-keys.csv'), :project => project)
158
- import.import!
159
-
160
- import.repair_duplicate_keys!(nil)
161
- import.dataset do |ds|
162
- assert !ds.columns!.include?(:dup_key_count)
163
- assert_equal 1, ds.filter(:id => 2).count
164
- assert_equal 1, ds.filter(:id => 4).count
165
- end
166
- project.local_database do |db|
167
- assert db.schema(:"import_#{import.id}").assoc(:id)[1][:primary_key]
168
- end
169
- end
170
-
171
- def test_repair_duplicate_keys_with_deletions
172
- tempfile = Tempfile.new('coupler-import')
173
- tempfile.write("id,foo,bar\n1,2,3\n1,4,5\n1,6,7\n123,456,789\n")
174
- tempfile.close
175
-
176
- project = Factory(:project)
177
- #puts "Project: #{project.id}"
178
- import = Factory(:import, :data => file_upload(tempfile.path), :project => project)
179
- #puts "Import: #{import.id}"
180
- import.import!
181
-
182
- import.repair_duplicate_keys!({"1" => ["1"]})
183
- import.dataset do |ds|
184
- assert !ds.columns!.include?(:dup_key_count)
185
- assert_equal 1, ds.filter(:id => 1).count
186
- assert_equal 1, ds.filter(:id => 124).count
187
- assert_equal 0, ds.filter(:id => 125).count
188
- end
189
- project.local_database do |db|
190
- assert db.schema(:"import_#{import.id}").assoc(:id)[1][:primary_key]
191
- end
192
- end
193
-
194
- def test_discover_fields_for_csv_with_headers_and_varying_number_of_fields
195
- tempfile = Tempfile.new('coupler-import')
196
- tempfile.write("id,foo,bar\n1,2,3\n1,4,5\n1,6,7,\n123,456,789,,\n")
197
- tempfile.close
198
-
199
- import = Factory.build(:import, :data => file_upload(tempfile.path))
200
- expected_types = %w{integer integer integer string string}
201
- expected_names = %w{id foo bar}
202
- assert_equal expected_names, import.field_names
203
- assert_equal expected_types, import.field_types
204
- assert_equal "id", import.primary_key_name
205
- assert import.has_headers
206
- end
207
-
208
- def test_importing_bad_integers
209
- tempfile = Tempfile.new('coupler-import')
210
- tempfile.write(%{id,foo,bar\n1,2,3\n2,4,5\n3,6,7\n4,456,""\n})
211
- tempfile.close
212
-
213
- project = Factory(:project)
214
- import = Factory(:import, :data => file_upload(tempfile.path), :project => project)
215
- assert_nothing_raised do
216
- import.import!
217
- end
218
- end
219
- end
220
- end
221
- end
@@ -1,91 +0,0 @@
1
- require 'factory_girl'
2
-
3
- Factory.sequence(:connection_name) { |n| "Connection #{n}" }
4
- Factory.define :connection, :class => Coupler::Models::Connection do |c|
5
- c.name { Factory.next(:connection_name) }
6
- c.adapter "h2"
7
- end
8
-
9
- Factory.sequence(:resource_name) { |n| "Resource #{n}" }
10
- Factory.define :resource, :class => Coupler::Models::Resource do |r|
11
- r.name { Factory.next(:resource_name) }
12
- r.table_name "people"
13
- r.association :connection
14
- r.association :project
15
- end
16
-
17
- Factory.define :project, :class => Coupler::Models::Project do |d|
18
- d.sequence(:name) { |n| "Project #{n}" }
19
- end
20
-
21
- Factory.define :transformation, :class => Coupler::Models::Transformation do |t|
22
- t.association :transformer
23
- t.association :resource
24
- t.source_field do |record|
25
- record.resource.fields_dataset.first rescue nil
26
- end
27
- end
28
-
29
- Factory.define :scenario, :class => Coupler::Models::Scenario do |s|
30
- s.sequence(:name) { |n| "Scenario #{n}" }
31
- s.association :project
32
- s.resource_1 { |x| x.project ? Factory(:resource, :project => x.project) : nil }
33
- end
34
-
35
- Factory.define :matcher, :class => Coupler::Models::Matcher do |m|
36
- m.association :scenario
37
- m.comparisons_attributes do |record|
38
- resources = record.scenario.resources
39
- [{
40
- 'lhs_type' => 'field', 'raw_lhs_value' => resources[0].fields_dataset.order('id DESC').last.id, 'lhs_which' => 1,
41
- 'rhs_type' => 'field', 'raw_rhs_value' => resources[-1].fields_dataset.order('id DESC').last.id, 'rhs_which' => 2,
42
- 'operator' => 'equals'
43
- }]
44
- end
45
- end
46
-
47
- Factory.define :result, :class => Coupler::Models::Result do |r|
48
- r.association :scenario
49
- end
50
-
51
- Factory.define :resource_job, :class => Coupler::Models::Job do |j|
52
- j.name 'transform'
53
- j.status 'scheduled'
54
- j.association :resource
55
- end
56
-
57
- Factory.define :scenario_job, :class => Coupler::Models::Job do |j|
58
- j.name 'run_scenario'
59
- j.status 'scheduled'
60
- j.association :scenario
61
- end
62
-
63
- Factory.define :transformer, :class => Coupler::Models::Transformer do |t|
64
- t.sequence(:name) { |n| "Transformer #{n}" }
65
- t.code "value"
66
- t.allowed_types { |x| %w{string integer datetime} }
67
- t.result_type "same"
68
- end
69
-
70
- Factory.define :field, :class => Coupler::Models::Field do |f|
71
- f.sequence(:name) { |n| "field_#{n}" }
72
- f.add_attribute :type, "integer"
73
- f.db_type "int(11)"
74
- f.is_primary_key 0
75
- f.is_selected 1
76
- f.association :resource
77
- end
78
-
79
- Factory.define :comparison, :class => Coupler::Models::Comparison do |c|
80
- c.association :matcher
81
- c.lhs_type "integer"
82
- c.raw_lhs_value 1
83
- c.rhs_type "integer"
84
- c.raw_rhs_value 1
85
- c.operator "equals"
86
- end
87
-
88
- Factory.define :import, :class => Coupler::Models::Import do |i|
89
- i.data { File.open(File.join(File.dirname(__FILE__), 'fixtures', 'people.csv')) }
90
- i.association :project
91
- end
@@ -1,80 +0,0 @@
1
- require 'helper'
2
-
3
- module TestExtensions
4
- class TestConnections < Coupler::Test::IntegrationTest
5
- def setup
6
- super
7
- @connections = []
8
- @configs = {}
9
- each_adapter do |adapter, config|
10
- conn = new_connection(adapter, :name => "#{adapter} connection").save!
11
- @connections << conn
12
- @configs[adapter] = config
13
- end
14
- end
15
-
16
- test "index" do
17
- get "/connections"
18
- assert last_response.ok?
19
- end
20
-
21
- test "new" do
22
- get "/connections/new"
23
- assert last_response.ok?
24
-
25
- doc = Nokogiri::HTML(last_response.body)
26
- assert_equal 1, doc.css("form[action='/connections']").length
27
- assert_equal 1, doc.css("select[name='connection[adapter]']").length
28
- %w{name host port username password}.each do |name|
29
- assert_equal 1, doc.css("input[name='connection[#{name}]']").length
30
- end
31
- end
32
-
33
- test "successfully creating connection" do
34
- attributes = @configs['mysql'].merge(:name => 'foo', :adapter => 'mysql')
35
- post "/connections", { 'connection' => attributes }
36
- connection = Connection[:name => 'foo']
37
- assert connection
38
-
39
- assert last_response.redirect?, "Wasn't redirected"
40
- assert_equal "http://example.org/connections", last_response['location']
41
- end
42
-
43
- test "successfully creating connection with return to" do
44
- attributes = @configs['mysql'].merge(:name => 'foo', :adapter => 'mysql')
45
- post "/connections", { 'connection' => attributes }, { 'rack.session' => { :return_to => '/foo' } }
46
-
47
- assert last_response.redirect?, "Wasn't redirected"
48
- assert_equal "http://example.org/foo", last_response['location']
49
- end
50
-
51
- test "successfully creating connection with first use" do
52
- attributes = @configs['mysql'].merge(:name => 'foo', :adapter => 'mysql')
53
- post "/connections", { 'connection' => attributes }, { 'rack.session' => { :first_use => true } }
54
-
55
- assert last_response.redirect?, "Wasn't redirected"
56
- assert_equal "http://example.org/projects/new", last_response['location']
57
- end
58
-
59
- test "failing to create connection" do
60
- attributes = @configs['mysql'].merge(:name => nil, :adapter => 'mysql')
61
- post "/connections", { 'connection' => attributes }
62
- assert last_response.ok?
63
- assert_match /Name is not present/, last_response.body
64
- end
65
-
66
- test "show" do
67
- @connections.each do |conn|
68
- get "/connections/#{conn.id}"
69
- assert last_response.ok?
70
- end
71
- end
72
-
73
- test "destroy" do
74
- delete "/connections/#{@connections[0].id}"
75
- assert_nil Models::Connection[@connections[0].id]
76
- assert last_response.redirect?
77
- assert_equal "http://example.org/connections", last_response['location']
78
- end
79
- end
80
- end
@@ -1,94 +0,0 @@
1
- require 'helper'
2
-
3
- module TestExtensions
4
- class TestImports < Coupler::Test::IntegrationTest
5
- def setup
6
- super
7
- @project = Project.create(:name => 'foo')
8
- end
9
-
10
- test "upload saves file" do
11
- post "/projects/#{@project.id}/imports/upload", :data => fixture_file_upload("people.csv")
12
- assert last_response.ok?
13
- end
14
-
15
- test "upload with no headers" do
16
- post "/projects/#{@project.id}/imports/upload", :data => fixture_file_upload("no-headers.csv")
17
- assert last_response.ok?
18
- end
19
-
20
- test "create with no issues" do
21
- cached_import = Import.new(:project => @project, :data => fixture_file_upload('people.csv'))
22
-
23
- post("/projects/#{@project.id}/imports", {
24
- :import => {
25
- :name => cached_import.name,
26
- :data_cache => cached_import.data_cache,
27
- :primary_key_name => cached_import.primary_key_name,
28
- :field_names => cached_import.field_names,
29
- :field_types => cached_import.field_types,
30
- :has_headers => true
31
- }
32
- })
33
- assert last_response.redirect?, "Wasn't a redirect"
34
- assert_match %r{^http://example.org/projects/#{@project[:id]}/resources/\d+$}, last_response['location']
35
- end
36
-
37
- test "create with invalid import" do
38
- cached_import = Import.new(:project => @project, :data => fixture_file_upload('people.csv'))
39
-
40
- post("/projects/#{@project.id}/imports", {
41
- :import => {
42
- :name => cached_import.name,
43
- :data_cache => cached_import.data_cache,
44
- :primary_key_name => cached_import.primary_key_name,
45
- :field_names => nil,
46
- :field_types => cached_import.field_types,
47
- :has_headers => true
48
- }
49
- })
50
- assert last_response.ok?
51
- end
52
-
53
- test "create with non existant project" do
54
- post "/projects/8675309/imports"
55
- assert last_response.redirect?
56
- assert_equal "http://example.org/projects", last_response['location']
57
- follow_redirect!
58
- assert_match /The project you were looking for doesn't exist/, last_response.body
59
- end
60
-
61
- test "create with failed import" do
62
- cached_import = Import.new(:project => @project, :data => fixture_file_upload('duplicate-keys.csv'))
63
-
64
- post("/projects/#{@project.id}/imports", {
65
- :import => {
66
- :name => cached_import.name,
67
- :data_cache => cached_import.data_cache,
68
- :primary_key_name => cached_import.primary_key_name,
69
- :field_names => cached_import.field_names,
70
- :field_types => cached_import.field_types,
71
- :has_headers => true
72
- }
73
- })
74
- assert last_response.redirect?, "Wasn't redirected"
75
- assert_match %r{http://example.org/projects/#{@project.id}/imports/\d+/edit}, last_response['location']
76
- end
77
-
78
- test "edit import with duplicate keys" do
79
- import = Import.create(:project => @project, :data => fixture_file_upload('duplicate-keys.csv'))
80
- import.import!
81
- get "/projects/#{@project.id}/imports/#{import.id}/edit"
82
- assert last_response.ok?
83
- end
84
-
85
- test "update import with duplicate keys" do
86
- import = Import.create(:project => @project, :data => fixture_file_upload('duplicate-keys.csv'))
87
- import.import!
88
-
89
- put("/projects/#{@project[:id]}/imports/#{import.id}", { :delete => { "1" => ["1", "2"] }})
90
- assert last_response.redirect?
91
- assert_match %r{http://example.org/projects/#{@project[:id]}/resources/\d+}, last_response['location']
92
- end
93
- end
94
- end