masamune 0.17.12 → 0.17.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -25,7 +25,7 @@ describe Masamune::Helpers::Postgres do
|
|
25
25
|
let(:instance) { described_class.new(environment) }
|
26
26
|
|
27
27
|
describe '#database_exists' do
|
28
|
-
let(:mock_status) {
|
28
|
+
let(:mock_status) {}
|
29
29
|
|
30
30
|
before do
|
31
31
|
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT version();', fail_fast: false, retries: 0)).and_return(mock_status)
|
@@ -87,7 +87,7 @@ describe Masamune::Schema::Catalog do
|
|
87
87
|
end
|
88
88
|
end
|
89
89
|
|
90
|
-
it { expect { schema }.to raise_error ArgumentError,
|
90
|
+
it { expect { schema }.to raise_error ArgumentError, 'schema store arguments required' }
|
91
91
|
end
|
92
92
|
|
93
93
|
context 'when schema defines unknown store' do
|
@@ -203,7 +203,7 @@ describe Masamune::Schema::Catalog do
|
|
203
203
|
column 'name', type: :string, unique: true
|
204
204
|
column 'description', type: :string
|
205
205
|
|
206
|
-
row name: 'current_database()', attributes: {default: true}
|
206
|
+
row name: 'current_database()', attributes: { default: true }
|
207
207
|
end
|
208
208
|
end
|
209
209
|
end
|
@@ -237,10 +237,10 @@ describe Masamune::Schema::Catalog do
|
|
237
237
|
let(:fact_one) { postgres.fact_one_fact }
|
238
238
|
let(:fact_two) { postgres.fact_two_fact }
|
239
239
|
|
240
|
-
it { expect(fact_one.references).to include :dimension_one}
|
240
|
+
it { expect(fact_one.references).to include :dimension_one }
|
241
241
|
it { expect(fact_one.measures).to include :measure_one }
|
242
242
|
it { expect(fact_one.measures[:measure_one].aggregate).to eq(:sum) }
|
243
|
-
it { expect(fact_two.references).to include :dimension_one}
|
243
|
+
it { expect(fact_two.references).to include :dimension_one }
|
244
244
|
it { expect(fact_two.measures).to include :measure_two }
|
245
245
|
it { expect(fact_two.measures[:measure_two].aggregate).to eq(:average) }
|
246
246
|
end
|
@@ -373,8 +373,8 @@ describe Masamune::Schema::Catalog do
|
|
373
373
|
context 'when schema contains file with headers & format override' do
|
374
374
|
before do
|
375
375
|
instance.schema :postgres do
|
376
|
-
file 'override', headers: false, format: :tsv
|
377
|
-
file 'default'
|
376
|
+
file 'override', headers: false, format: :tsv
|
377
|
+
file 'default'
|
378
378
|
end
|
379
379
|
end
|
380
380
|
|
@@ -399,7 +399,7 @@ describe Masamune::Schema::Catalog do
|
|
399
399
|
end
|
400
400
|
|
401
401
|
it 'should raise an exception' do
|
402
|
-
expect { schema }.to raise_error
|
402
|
+
expect { schema }.to raise_error(/dimension user_account not defined/)
|
403
403
|
end
|
404
404
|
end
|
405
405
|
|
@@ -429,7 +429,7 @@ describe Masamune::Schema::Catalog do
|
|
429
429
|
{
|
430
430
|
'tenant_id' => row[:tenant_id],
|
431
431
|
'user_id' => row[:id],
|
432
|
-
'user_account_state.name' => row[:deleted_at] ? 'deleted' :
|
432
|
+
'user_account_state.name' => row[:deleted_at] ? 'deleted' : 'active',
|
433
433
|
'start_at' => row[:updated_at],
|
434
434
|
'delta' => 0
|
435
435
|
}
|
@@ -456,7 +456,7 @@ describe Masamune::Schema::Catalog do
|
|
456
456
|
end
|
457
457
|
|
458
458
|
it 'should raise an exception' do
|
459
|
-
expect { schema }.to raise_error
|
459
|
+
expect { schema }.to raise_error(/invalid map, from: is missing/)
|
460
460
|
end
|
461
461
|
end
|
462
462
|
|
@@ -472,14 +472,14 @@ describe Masamune::Schema::Catalog do
|
|
472
472
|
end
|
473
473
|
|
474
474
|
it 'should raise an exception' do
|
475
|
-
expect { schema }.to raise_error
|
475
|
+
expect { schema }.to raise_error(/invalid map, from: is missing/)
|
476
476
|
end
|
477
477
|
end
|
478
478
|
|
479
479
|
context 'when schema contains map missing the to: field' do
|
480
480
|
subject(:schema) do
|
481
481
|
instance.schema :postgres do
|
482
|
-
file 'users'
|
482
|
+
file 'users'
|
483
483
|
|
484
484
|
map from: postgres.users_file do
|
485
485
|
field 'tenant_id'
|
@@ -488,15 +488,15 @@ describe Masamune::Schema::Catalog do
|
|
488
488
|
end
|
489
489
|
|
490
490
|
it 'should raise an exception' do
|
491
|
-
expect { schema }.to raise_error
|
491
|
+
expect { schema }.to raise_error(/invalid map from: 'users', to: is missing/)
|
492
492
|
end
|
493
493
|
end
|
494
494
|
|
495
495
|
context 'when schema addressed with symbols' do
|
496
496
|
before do
|
497
497
|
instance.schema :postgres do
|
498
|
-
dimension 'user', type: :one
|
499
|
-
file 'users'
|
498
|
+
dimension 'user', type: :one
|
499
|
+
file 'users'
|
500
500
|
|
501
501
|
map from: postgres.files[:users], to: postgres.dimensions[:user] do
|
502
502
|
field 'tenant_id'
|
@@ -514,8 +514,8 @@ describe Masamune::Schema::Catalog do
|
|
514
514
|
context 'when schema addressed with strings' do
|
515
515
|
before do
|
516
516
|
instance.schema :postgres do
|
517
|
-
dimension 'user', type: :one
|
518
|
-
file 'users'
|
517
|
+
dimension 'user', type: :one
|
518
|
+
file 'users'
|
519
519
|
|
520
520
|
map from: postgres.files['users'], to: postgres.dimensions['user'] do
|
521
521
|
field 'tenant_id'
|
@@ -123,7 +123,7 @@ describe Masamune::Schema::Column do
|
|
123
123
|
end
|
124
124
|
|
125
125
|
context 'with index: ["id", "shared"]' do
|
126
|
-
subject(:column) { described_class.new(id: 'id', index:
|
126
|
+
subject(:column) { described_class.new(id: 'id', index: %w(id shared)) }
|
127
127
|
context '#index' do
|
128
128
|
subject { column.index }
|
129
129
|
it { is_expected.to include(:id) }
|
@@ -189,7 +189,7 @@ describe Masamune::Schema::Column do
|
|
189
189
|
let(:column) { described_class.new(id: 'string', type: :string) }
|
190
190
|
context 'with string value' do
|
191
191
|
let(:value) { 'value' }
|
192
|
-
it { is_expected.to eq(
|
192
|
+
it { is_expected.to eq("'value'") }
|
193
193
|
end
|
194
194
|
it_behaves_like 'with :null value'
|
195
195
|
end
|
@@ -213,7 +213,7 @@ describe Masamune::Schema::Column do
|
|
213
213
|
let(:column) { described_class.new(id: 'enum', type: :enum, values: %w(public private)) }
|
214
214
|
context 'with enum value' do
|
215
215
|
let(:value) { 'public' }
|
216
|
-
it { is_expected.to eq(
|
216
|
+
it { is_expected.to eq("'public'::ENUM_TYPE") }
|
217
217
|
end
|
218
218
|
|
219
219
|
it_behaves_like 'with :null value'
|
@@ -261,7 +261,7 @@ describe Masamune::Schema::Column do
|
|
261
261
|
end
|
262
262
|
|
263
263
|
context "when ''1''" do
|
264
|
-
let(:value) { %
|
264
|
+
let(:value) { %('1') }
|
265
265
|
it { is_expected.to eq(true) }
|
266
266
|
end
|
267
267
|
|
@@ -286,7 +286,7 @@ describe Masamune::Schema::Column do
|
|
286
286
|
end
|
287
287
|
|
288
288
|
context "when ''0''" do
|
289
|
-
let(:value) { %
|
289
|
+
let(:value) { %('0') }
|
290
290
|
it { is_expected.to eq(false) }
|
291
291
|
end
|
292
292
|
|
@@ -325,18 +325,18 @@ describe Masamune::Schema::Column do
|
|
325
325
|
end
|
326
326
|
|
327
327
|
context 'when Date' do
|
328
|
-
let(:value) { Date.civil(2015,01,01) }
|
328
|
+
let(:value) { Date.civil(2015, 01, 01) }
|
329
329
|
it { is_expected.to eq(value) }
|
330
330
|
end
|
331
331
|
|
332
332
|
context 'when YYYY-mm-dd' do
|
333
333
|
let(:value) { '2015-01-01' }
|
334
|
-
it { is_expected.to eq(Date.civil(2015,01,01)) }
|
334
|
+
it { is_expected.to eq(Date.civil(2015, 01, 01)) }
|
335
335
|
end
|
336
336
|
|
337
337
|
context 'when ISO8601' do
|
338
338
|
let(:value) { Date.parse('2015-01-01').iso8601 }
|
339
|
-
it { is_expected.to eq(Date.civil(2015,01,01)) }
|
339
|
+
it { is_expected.to eq(Date.civil(2015, 01, 01)) }
|
340
340
|
end
|
341
341
|
end
|
342
342
|
|
@@ -354,7 +354,7 @@ describe Masamune::Schema::Column do
|
|
354
354
|
end
|
355
355
|
|
356
356
|
context 'when String encoded Integer' do
|
357
|
-
let(:value) {
|
357
|
+
let(:value) { '1' }
|
358
358
|
it { is_expected.to eq(1) }
|
359
359
|
end
|
360
360
|
|
@@ -397,12 +397,12 @@ describe Masamune::Schema::Column do
|
|
397
397
|
end
|
398
398
|
|
399
399
|
context 'when Date' do
|
400
|
-
let(:value) { Date.civil(2015,01,01) }
|
400
|
+
let(:value) { Date.civil(2015, 01, 01) }
|
401
401
|
it { is_expected.to eq(value.to_time) }
|
402
402
|
end
|
403
403
|
|
404
404
|
context 'when DateTime' do
|
405
|
-
let(:value) { DateTime.civil(2015,01,01) }
|
405
|
+
let(:value) { DateTime.civil(2015, 01, 01) }
|
406
406
|
it { is_expected.to eq(value.to_time) }
|
407
407
|
end
|
408
408
|
|
@@ -417,7 +417,7 @@ describe Masamune::Schema::Column do
|
|
417
417
|
end
|
418
418
|
|
419
419
|
context 'when String encoded Integer' do
|
420
|
-
let(:value) {
|
420
|
+
let(:value) { Time.now.utc.to_i.to_s }
|
421
421
|
it { is_expected.to eq(Time.at(value.to_i)) }
|
422
422
|
end
|
423
423
|
|
@@ -428,7 +428,7 @@ describe Masamune::Schema::Column do
|
|
428
428
|
|
429
429
|
context 'when ISO8601' do
|
430
430
|
let(:value) { Date.parse('2015-01-01').to_time.iso8601 }
|
431
|
-
it { is_expected.to eq(Date.civil(2015,01,01).to_time) }
|
431
|
+
it { is_expected.to eq(Date.civil(2015, 01, 01).to_time) }
|
432
432
|
end
|
433
433
|
end
|
434
434
|
|
@@ -452,7 +452,7 @@ describe Masamune::Schema::Column do
|
|
452
452
|
|
453
453
|
context 'when array' do
|
454
454
|
let(:value) { '[1,2]' }
|
455
|
-
it { is_expected.to eq([1,2]) }
|
455
|
+
it { is_expected.to eq([1, 2]) }
|
456
456
|
end
|
457
457
|
end
|
458
458
|
|
@@ -476,7 +476,7 @@ describe Masamune::Schema::Column do
|
|
476
476
|
|
477
477
|
context 'when array' do
|
478
478
|
let(:value) { '{"k":"v"}' }
|
479
|
-
it { is_expected.to eq(
|
479
|
+
it { is_expected.to eq('k' => 'v') }
|
480
480
|
end
|
481
481
|
end
|
482
482
|
end
|
@@ -587,12 +587,12 @@ describe Masamune::Schema::Column do
|
|
587
587
|
end
|
588
588
|
|
589
589
|
context 'when array of integer' do
|
590
|
-
let(:value) { [1,2] }
|
590
|
+
let(:value) { [1, 2] }
|
591
591
|
it { is_expected.to eq('[1,2]') }
|
592
592
|
end
|
593
593
|
|
594
594
|
context 'when array of string' do
|
595
|
-
let(:value) {
|
595
|
+
let(:value) { %w(1 2) }
|
596
596
|
it { is_expected.to eq('[1,2]') }
|
597
597
|
end
|
598
598
|
end
|
@@ -616,12 +616,12 @@ describe Masamune::Schema::Column do
|
|
616
616
|
end
|
617
617
|
|
618
618
|
context 'when array of string' do
|
619
|
-
let(:value) {
|
619
|
+
let(:value) { %w(1 2) }
|
620
620
|
it { is_expected.to eq('["1","2"]') }
|
621
621
|
end
|
622
622
|
|
623
623
|
context 'when array of integer' do
|
624
|
-
let(:value) { [1,2] }
|
624
|
+
let(:value) { [1, 2] }
|
625
625
|
it { is_expected.to eq('["1","2"]') }
|
626
626
|
end
|
627
627
|
end
|
@@ -80,7 +80,7 @@ describe Masamune::Schema::Dimension do
|
|
80
80
|
]
|
81
81
|
end
|
82
82
|
|
83
|
-
it { expect { dimension }.to raise_error
|
83
|
+
it { expect { dimension }.to raise_error(/contains undefined columns/) }
|
84
84
|
end
|
85
85
|
|
86
86
|
context 'for type :four' do
|
@@ -93,7 +93,7 @@ describe Masamune::Schema::Dimension do
|
|
93
93
|
rows: [
|
94
94
|
Masamune::Schema::Row.new(values: {
|
95
95
|
name: 'active',
|
96
|
-
description: 'Active'
|
96
|
+
description: 'Active'
|
97
97
|
}, default: true)
|
98
98
|
]
|
99
99
|
end
|
@@ -158,9 +158,9 @@ describe Masamune::Schema::Fact do
|
|
158
158
|
|
159
159
|
it 'yields partition tables' do
|
160
160
|
expect { |b| fact.partition_tables(start_date, stop_date, &b) }.to yield_successive_args \
|
161
|
-
|
162
|
-
|
163
|
-
|
161
|
+
fact.partition_table(Date.civil(2015, 01, 01)),
|
162
|
+
fact.partition_table(Date.civil(2015, 02, 01)),
|
163
|
+
fact.partition_table(Date.civil(2015, 03, 01))
|
164
164
|
end
|
165
165
|
end
|
166
166
|
end
|
@@ -113,7 +113,7 @@ describe Masamune::Schema::Map do
|
|
113
113
|
file 'input'
|
114
114
|
file 'output'
|
115
115
|
|
116
|
-
map from: files.input
|
116
|
+
map from: files.input, to: files.output do |_row|
|
117
117
|
# Empty
|
118
118
|
end
|
119
119
|
end
|
@@ -178,12 +178,12 @@ describe Masamune::Schema::Map do
|
|
178
178
|
before do
|
179
179
|
expect(environment.logger).to receive(:warn).with(/missing required columns 'id'/).ordered
|
180
180
|
expect(environment.logger).to receive(:debug).with(
|
181
|
-
:
|
182
|
-
:
|
183
|
-
:
|
184
|
-
:
|
185
|
-
:
|
186
|
-
:
|
181
|
+
message: "missing required columns 'id'",
|
182
|
+
source: 'user_stage',
|
183
|
+
target: 'user_dimension_ledger',
|
184
|
+
file: input.path,
|
185
|
+
line: 3,
|
186
|
+
row: {
|
187
187
|
'id' => nil,
|
188
188
|
'tenant_id' => '50',
|
189
189
|
'junk_id' => 'X',
|
@@ -238,32 +238,32 @@ describe Masamune::Schema::Map do
|
|
238
238
|
before do
|
239
239
|
expect(environment.logger).to receive(:warn).with(/failed to process/).ordered
|
240
240
|
expect(environment.logger).to receive(:debug).with(
|
241
|
-
:
|
242
|
-
:
|
243
|
-
:
|
244
|
-
:
|
245
|
-
:
|
246
|
-
:
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
241
|
+
message: 'failed to process',
|
242
|
+
source: 'input_stage',
|
243
|
+
target: 'user_dimension_ledger',
|
244
|
+
file: input.path,
|
245
|
+
line: 2,
|
246
|
+
row: {
|
247
|
+
'id' => 1,
|
248
|
+
'tenant_id' => 42,
|
249
|
+
'admin' => false,
|
250
|
+
'preferences' => {},
|
251
|
+
'deleted_at' => nil
|
252
252
|
}
|
253
253
|
).ordered
|
254
254
|
expect(environment.logger).to receive(:warn).with("Could not coerce 'INVALID_JSON' into :json for column 'preferences'").ordered
|
255
255
|
expect(environment.logger).to receive(:debug).with(
|
256
|
-
:
|
257
|
-
:
|
258
|
-
:
|
259
|
-
:
|
260
|
-
:
|
261
|
-
:
|
262
|
-
:
|
263
|
-
:
|
264
|
-
:
|
265
|
-
:
|
266
|
-
:
|
256
|
+
message: "Could not coerce 'INVALID_JSON' into :json for column 'preferences'",
|
257
|
+
source: 'input_stage',
|
258
|
+
target: 'user_dimension_ledger',
|
259
|
+
file: input.path,
|
260
|
+
line: 4,
|
261
|
+
row: {
|
262
|
+
id: '3',
|
263
|
+
tenant_id: '50',
|
264
|
+
admin: '0',
|
265
|
+
preferences: 'INVALID_JSON',
|
266
|
+
deleted_at: nil
|
267
267
|
}
|
268
268
|
).ordered
|
269
269
|
end
|
@@ -698,7 +698,7 @@ describe Masamune::Schema::Map do
|
|
698
698
|
column 'id', type: :integer
|
699
699
|
end
|
700
700
|
|
701
|
-
map from: files.input, to: files.output, columns: [:id], fail_fast: true do |
|
701
|
+
map from: files.input, to: files.output, columns: [:id], fail_fast: true do |_row|
|
702
702
|
raise 'wha happen'
|
703
703
|
end
|
704
704
|
end
|
@@ -729,17 +729,17 @@ describe Masamune::Schema::Map do
|
|
729
729
|
before do
|
730
730
|
expect(environment.logger).to receive(:error).with(/wha happen/).ordered
|
731
731
|
expect(environment.logger).to receive(:debug).with(
|
732
|
-
:
|
733
|
-
:
|
734
|
-
:
|
735
|
-
:
|
736
|
-
:
|
737
|
-
:
|
732
|
+
message: 'wha happen',
|
733
|
+
source: 'input_stage',
|
734
|
+
target: 'output_stage',
|
735
|
+
file: input.path,
|
736
|
+
line: 0,
|
737
|
+
row: { 'id' => 1 }
|
738
738
|
).ordered
|
739
739
|
end
|
740
740
|
|
741
741
|
it 'raises exception' do
|
742
|
-
expect { subject }.to raise_error
|
742
|
+
expect { subject }.to raise_error(/wha happen/)
|
743
743
|
end
|
744
744
|
end
|
745
745
|
end
|
@@ -806,7 +806,7 @@ describe Masamune::Schema::Map do
|
|
806
806
|
io.write '{},{}'
|
807
807
|
io.rewind
|
808
808
|
end
|
809
|
-
it { is_expected.to eq(%
|
809
|
+
it { is_expected.to eq(%("{}","{}")) }
|
810
810
|
end
|
811
811
|
|
812
812
|
context 'with quoted empty json' do
|
@@ -814,7 +814,7 @@ describe Masamune::Schema::Map do
|
|
814
814
|
io.write '"{}","{}"'
|
815
815
|
io.rewind
|
816
816
|
end
|
817
|
-
it { is_expected.to eq(%
|
817
|
+
it { is_expected.to eq(%("{}","{}")) }
|
818
818
|
end
|
819
819
|
|
820
820
|
context 'with raw json' do
|
@@ -822,7 +822,7 @@ describe Masamune::Schema::Map do
|
|
822
822
|
io.write '{"enabled":true,"state":""}'
|
823
823
|
io.rewind
|
824
824
|
end
|
825
|
-
it { is_expected.to eq(%
|
825
|
+
it { is_expected.to eq(%("{""enabled"":true,""state"":""""}")) }
|
826
826
|
end
|
827
827
|
|
828
828
|
context 'with quoted json' do
|
@@ -830,7 +830,7 @@ describe Masamune::Schema::Map do
|
|
830
830
|
io.write '"{""enabled"":true,""state"":""""}"'
|
831
831
|
io.rewind
|
832
832
|
end
|
833
|
-
it { is_expected.to eq(%
|
833
|
+
it { is_expected.to eq(%("{""enabled"":true,""state"":""""}")) }
|
834
834
|
end
|
835
835
|
end
|
836
836
|
end
|