masamune 0.17.12 → 0.17.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -25,7 +25,7 @@ describe Masamune::Helpers::Postgres do
|
|
25
25
|
let(:instance) { described_class.new(environment) }
|
26
26
|
|
27
27
|
describe '#database_exists' do
|
28
|
-
let(:mock_status) {
|
28
|
+
let(:mock_status) {}
|
29
29
|
|
30
30
|
before do
|
31
31
|
expect(instance).to receive(:postgres).with(hash_including(exec: 'SELECT version();', fail_fast: false, retries: 0)).and_return(mock_status)
|
@@ -87,7 +87,7 @@ describe Masamune::Schema::Catalog do
|
|
87
87
|
end
|
88
88
|
end
|
89
89
|
|
90
|
-
it { expect { schema }.to raise_error ArgumentError,
|
90
|
+
it { expect { schema }.to raise_error ArgumentError, 'schema store arguments required' }
|
91
91
|
end
|
92
92
|
|
93
93
|
context 'when schema defines unknown store' do
|
@@ -203,7 +203,7 @@ describe Masamune::Schema::Catalog do
|
|
203
203
|
column 'name', type: :string, unique: true
|
204
204
|
column 'description', type: :string
|
205
205
|
|
206
|
-
row name: 'current_database()', attributes: {default: true}
|
206
|
+
row name: 'current_database()', attributes: { default: true }
|
207
207
|
end
|
208
208
|
end
|
209
209
|
end
|
@@ -237,10 +237,10 @@ describe Masamune::Schema::Catalog do
|
|
237
237
|
let(:fact_one) { postgres.fact_one_fact }
|
238
238
|
let(:fact_two) { postgres.fact_two_fact }
|
239
239
|
|
240
|
-
it { expect(fact_one.references).to include :dimension_one}
|
240
|
+
it { expect(fact_one.references).to include :dimension_one }
|
241
241
|
it { expect(fact_one.measures).to include :measure_one }
|
242
242
|
it { expect(fact_one.measures[:measure_one].aggregate).to eq(:sum) }
|
243
|
-
it { expect(fact_two.references).to include :dimension_one}
|
243
|
+
it { expect(fact_two.references).to include :dimension_one }
|
244
244
|
it { expect(fact_two.measures).to include :measure_two }
|
245
245
|
it { expect(fact_two.measures[:measure_two].aggregate).to eq(:average) }
|
246
246
|
end
|
@@ -373,8 +373,8 @@ describe Masamune::Schema::Catalog do
|
|
373
373
|
context 'when schema contains file with headers & format override' do
|
374
374
|
before do
|
375
375
|
instance.schema :postgres do
|
376
|
-
file 'override', headers: false, format: :tsv
|
377
|
-
file 'default'
|
376
|
+
file 'override', headers: false, format: :tsv
|
377
|
+
file 'default'
|
378
378
|
end
|
379
379
|
end
|
380
380
|
|
@@ -399,7 +399,7 @@ describe Masamune::Schema::Catalog do
|
|
399
399
|
end
|
400
400
|
|
401
401
|
it 'should raise an exception' do
|
402
|
-
expect { schema }.to raise_error
|
402
|
+
expect { schema }.to raise_error(/dimension user_account not defined/)
|
403
403
|
end
|
404
404
|
end
|
405
405
|
|
@@ -429,7 +429,7 @@ describe Masamune::Schema::Catalog do
|
|
429
429
|
{
|
430
430
|
'tenant_id' => row[:tenant_id],
|
431
431
|
'user_id' => row[:id],
|
432
|
-
'user_account_state.name' => row[:deleted_at] ? 'deleted' :
|
432
|
+
'user_account_state.name' => row[:deleted_at] ? 'deleted' : 'active',
|
433
433
|
'start_at' => row[:updated_at],
|
434
434
|
'delta' => 0
|
435
435
|
}
|
@@ -456,7 +456,7 @@ describe Masamune::Schema::Catalog do
|
|
456
456
|
end
|
457
457
|
|
458
458
|
it 'should raise an exception' do
|
459
|
-
expect { schema }.to raise_error
|
459
|
+
expect { schema }.to raise_error(/invalid map, from: is missing/)
|
460
460
|
end
|
461
461
|
end
|
462
462
|
|
@@ -472,14 +472,14 @@ describe Masamune::Schema::Catalog do
|
|
472
472
|
end
|
473
473
|
|
474
474
|
it 'should raise an exception' do
|
475
|
-
expect { schema }.to raise_error
|
475
|
+
expect { schema }.to raise_error(/invalid map, from: is missing/)
|
476
476
|
end
|
477
477
|
end
|
478
478
|
|
479
479
|
context 'when schema contains map missing the to: field' do
|
480
480
|
subject(:schema) do
|
481
481
|
instance.schema :postgres do
|
482
|
-
file 'users'
|
482
|
+
file 'users'
|
483
483
|
|
484
484
|
map from: postgres.users_file do
|
485
485
|
field 'tenant_id'
|
@@ -488,15 +488,15 @@ describe Masamune::Schema::Catalog do
|
|
488
488
|
end
|
489
489
|
|
490
490
|
it 'should raise an exception' do
|
491
|
-
expect { schema }.to raise_error
|
491
|
+
expect { schema }.to raise_error(/invalid map from: 'users', to: is missing/)
|
492
492
|
end
|
493
493
|
end
|
494
494
|
|
495
495
|
context 'when schema addressed with symbols' do
|
496
496
|
before do
|
497
497
|
instance.schema :postgres do
|
498
|
-
dimension 'user', type: :one
|
499
|
-
file 'users'
|
498
|
+
dimension 'user', type: :one
|
499
|
+
file 'users'
|
500
500
|
|
501
501
|
map from: postgres.files[:users], to: postgres.dimensions[:user] do
|
502
502
|
field 'tenant_id'
|
@@ -514,8 +514,8 @@ describe Masamune::Schema::Catalog do
|
|
514
514
|
context 'when schema addressed with strings' do
|
515
515
|
before do
|
516
516
|
instance.schema :postgres do
|
517
|
-
dimension 'user', type: :one
|
518
|
-
file 'users'
|
517
|
+
dimension 'user', type: :one
|
518
|
+
file 'users'
|
519
519
|
|
520
520
|
map from: postgres.files['users'], to: postgres.dimensions['user'] do
|
521
521
|
field 'tenant_id'
|
@@ -123,7 +123,7 @@ describe Masamune::Schema::Column do
|
|
123
123
|
end
|
124
124
|
|
125
125
|
context 'with index: ["id", "shared"]' do
|
126
|
-
subject(:column) { described_class.new(id: 'id', index:
|
126
|
+
subject(:column) { described_class.new(id: 'id', index: %w(id shared)) }
|
127
127
|
context '#index' do
|
128
128
|
subject { column.index }
|
129
129
|
it { is_expected.to include(:id) }
|
@@ -189,7 +189,7 @@ describe Masamune::Schema::Column do
|
|
189
189
|
let(:column) { described_class.new(id: 'string', type: :string) }
|
190
190
|
context 'with string value' do
|
191
191
|
let(:value) { 'value' }
|
192
|
-
it { is_expected.to eq(
|
192
|
+
it { is_expected.to eq("'value'") }
|
193
193
|
end
|
194
194
|
it_behaves_like 'with :null value'
|
195
195
|
end
|
@@ -213,7 +213,7 @@ describe Masamune::Schema::Column do
|
|
213
213
|
let(:column) { described_class.new(id: 'enum', type: :enum, values: %w(public private)) }
|
214
214
|
context 'with enum value' do
|
215
215
|
let(:value) { 'public' }
|
216
|
-
it { is_expected.to eq(
|
216
|
+
it { is_expected.to eq("'public'::ENUM_TYPE") }
|
217
217
|
end
|
218
218
|
|
219
219
|
it_behaves_like 'with :null value'
|
@@ -261,7 +261,7 @@ describe Masamune::Schema::Column do
|
|
261
261
|
end
|
262
262
|
|
263
263
|
context "when ''1''" do
|
264
|
-
let(:value) { %
|
264
|
+
let(:value) { %('1') }
|
265
265
|
it { is_expected.to eq(true) }
|
266
266
|
end
|
267
267
|
|
@@ -286,7 +286,7 @@ describe Masamune::Schema::Column do
|
|
286
286
|
end
|
287
287
|
|
288
288
|
context "when ''0''" do
|
289
|
-
let(:value) { %
|
289
|
+
let(:value) { %('0') }
|
290
290
|
it { is_expected.to eq(false) }
|
291
291
|
end
|
292
292
|
|
@@ -325,18 +325,18 @@ describe Masamune::Schema::Column do
|
|
325
325
|
end
|
326
326
|
|
327
327
|
context 'when Date' do
|
328
|
-
let(:value) { Date.civil(2015,01,01) }
|
328
|
+
let(:value) { Date.civil(2015, 01, 01) }
|
329
329
|
it { is_expected.to eq(value) }
|
330
330
|
end
|
331
331
|
|
332
332
|
context 'when YYYY-mm-dd' do
|
333
333
|
let(:value) { '2015-01-01' }
|
334
|
-
it { is_expected.to eq(Date.civil(2015,01,01)) }
|
334
|
+
it { is_expected.to eq(Date.civil(2015, 01, 01)) }
|
335
335
|
end
|
336
336
|
|
337
337
|
context 'when ISO8601' do
|
338
338
|
let(:value) { Date.parse('2015-01-01').iso8601 }
|
339
|
-
it { is_expected.to eq(Date.civil(2015,01,01)) }
|
339
|
+
it { is_expected.to eq(Date.civil(2015, 01, 01)) }
|
340
340
|
end
|
341
341
|
end
|
342
342
|
|
@@ -354,7 +354,7 @@ describe Masamune::Schema::Column do
|
|
354
354
|
end
|
355
355
|
|
356
356
|
context 'when String encoded Integer' do
|
357
|
-
let(:value) {
|
357
|
+
let(:value) { '1' }
|
358
358
|
it { is_expected.to eq(1) }
|
359
359
|
end
|
360
360
|
|
@@ -397,12 +397,12 @@ describe Masamune::Schema::Column do
|
|
397
397
|
end
|
398
398
|
|
399
399
|
context 'when Date' do
|
400
|
-
let(:value) { Date.civil(2015,01,01) }
|
400
|
+
let(:value) { Date.civil(2015, 01, 01) }
|
401
401
|
it { is_expected.to eq(value.to_time) }
|
402
402
|
end
|
403
403
|
|
404
404
|
context 'when DateTime' do
|
405
|
-
let(:value) { DateTime.civil(2015,01,01) }
|
405
|
+
let(:value) { DateTime.civil(2015, 01, 01) }
|
406
406
|
it { is_expected.to eq(value.to_time) }
|
407
407
|
end
|
408
408
|
|
@@ -417,7 +417,7 @@ describe Masamune::Schema::Column do
|
|
417
417
|
end
|
418
418
|
|
419
419
|
context 'when String encoded Integer' do
|
420
|
-
let(:value) {
|
420
|
+
let(:value) { Time.now.utc.to_i.to_s }
|
421
421
|
it { is_expected.to eq(Time.at(value.to_i)) }
|
422
422
|
end
|
423
423
|
|
@@ -428,7 +428,7 @@ describe Masamune::Schema::Column do
|
|
428
428
|
|
429
429
|
context 'when ISO8601' do
|
430
430
|
let(:value) { Date.parse('2015-01-01').to_time.iso8601 }
|
431
|
-
it { is_expected.to eq(Date.civil(2015,01,01).to_time) }
|
431
|
+
it { is_expected.to eq(Date.civil(2015, 01, 01).to_time) }
|
432
432
|
end
|
433
433
|
end
|
434
434
|
|
@@ -452,7 +452,7 @@ describe Masamune::Schema::Column do
|
|
452
452
|
|
453
453
|
context 'when array' do
|
454
454
|
let(:value) { '[1,2]' }
|
455
|
-
it { is_expected.to eq([1,2]) }
|
455
|
+
it { is_expected.to eq([1, 2]) }
|
456
456
|
end
|
457
457
|
end
|
458
458
|
|
@@ -476,7 +476,7 @@ describe Masamune::Schema::Column do
|
|
476
476
|
|
477
477
|
context 'when array' do
|
478
478
|
let(:value) { '{"k":"v"}' }
|
479
|
-
it { is_expected.to eq(
|
479
|
+
it { is_expected.to eq('k' => 'v') }
|
480
480
|
end
|
481
481
|
end
|
482
482
|
end
|
@@ -587,12 +587,12 @@ describe Masamune::Schema::Column do
|
|
587
587
|
end
|
588
588
|
|
589
589
|
context 'when array of integer' do
|
590
|
-
let(:value) { [1,2] }
|
590
|
+
let(:value) { [1, 2] }
|
591
591
|
it { is_expected.to eq('[1,2]') }
|
592
592
|
end
|
593
593
|
|
594
594
|
context 'when array of string' do
|
595
|
-
let(:value) {
|
595
|
+
let(:value) { %w(1 2) }
|
596
596
|
it { is_expected.to eq('[1,2]') }
|
597
597
|
end
|
598
598
|
end
|
@@ -616,12 +616,12 @@ describe Masamune::Schema::Column do
|
|
616
616
|
end
|
617
617
|
|
618
618
|
context 'when array of string' do
|
619
|
-
let(:value) {
|
619
|
+
let(:value) { %w(1 2) }
|
620
620
|
it { is_expected.to eq('["1","2"]') }
|
621
621
|
end
|
622
622
|
|
623
623
|
context 'when array of integer' do
|
624
|
-
let(:value) { [1,2] }
|
624
|
+
let(:value) { [1, 2] }
|
625
625
|
it { is_expected.to eq('["1","2"]') }
|
626
626
|
end
|
627
627
|
end
|
@@ -80,7 +80,7 @@ describe Masamune::Schema::Dimension do
|
|
80
80
|
]
|
81
81
|
end
|
82
82
|
|
83
|
-
it { expect { dimension }.to raise_error
|
83
|
+
it { expect { dimension }.to raise_error(/contains undefined columns/) }
|
84
84
|
end
|
85
85
|
|
86
86
|
context 'for type :four' do
|
@@ -93,7 +93,7 @@ describe Masamune::Schema::Dimension do
|
|
93
93
|
rows: [
|
94
94
|
Masamune::Schema::Row.new(values: {
|
95
95
|
name: 'active',
|
96
|
-
description: 'Active'
|
96
|
+
description: 'Active'
|
97
97
|
}, default: true)
|
98
98
|
]
|
99
99
|
end
|
@@ -158,9 +158,9 @@ describe Masamune::Schema::Fact do
|
|
158
158
|
|
159
159
|
it 'yields partition tables' do
|
160
160
|
expect { |b| fact.partition_tables(start_date, stop_date, &b) }.to yield_successive_args \
|
161
|
-
|
162
|
-
|
163
|
-
|
161
|
+
fact.partition_table(Date.civil(2015, 01, 01)),
|
162
|
+
fact.partition_table(Date.civil(2015, 02, 01)),
|
163
|
+
fact.partition_table(Date.civil(2015, 03, 01))
|
164
164
|
end
|
165
165
|
end
|
166
166
|
end
|
@@ -113,7 +113,7 @@ describe Masamune::Schema::Map do
|
|
113
113
|
file 'input'
|
114
114
|
file 'output'
|
115
115
|
|
116
|
-
map from: files.input
|
116
|
+
map from: files.input, to: files.output do |_row|
|
117
117
|
# Empty
|
118
118
|
end
|
119
119
|
end
|
@@ -178,12 +178,12 @@ describe Masamune::Schema::Map do
|
|
178
178
|
before do
|
179
179
|
expect(environment.logger).to receive(:warn).with(/missing required columns 'id'/).ordered
|
180
180
|
expect(environment.logger).to receive(:debug).with(
|
181
|
-
:
|
182
|
-
:
|
183
|
-
:
|
184
|
-
:
|
185
|
-
:
|
186
|
-
:
|
181
|
+
message: "missing required columns 'id'",
|
182
|
+
source: 'user_stage',
|
183
|
+
target: 'user_dimension_ledger',
|
184
|
+
file: input.path,
|
185
|
+
line: 3,
|
186
|
+
row: {
|
187
187
|
'id' => nil,
|
188
188
|
'tenant_id' => '50',
|
189
189
|
'junk_id' => 'X',
|
@@ -238,32 +238,32 @@ describe Masamune::Schema::Map do
|
|
238
238
|
before do
|
239
239
|
expect(environment.logger).to receive(:warn).with(/failed to process/).ordered
|
240
240
|
expect(environment.logger).to receive(:debug).with(
|
241
|
-
:
|
242
|
-
:
|
243
|
-
:
|
244
|
-
:
|
245
|
-
:
|
246
|
-
:
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
241
|
+
message: 'failed to process',
|
242
|
+
source: 'input_stage',
|
243
|
+
target: 'user_dimension_ledger',
|
244
|
+
file: input.path,
|
245
|
+
line: 2,
|
246
|
+
row: {
|
247
|
+
'id' => 1,
|
248
|
+
'tenant_id' => 42,
|
249
|
+
'admin' => false,
|
250
|
+
'preferences' => {},
|
251
|
+
'deleted_at' => nil
|
252
252
|
}
|
253
253
|
).ordered
|
254
254
|
expect(environment.logger).to receive(:warn).with("Could not coerce 'INVALID_JSON' into :json for column 'preferences'").ordered
|
255
255
|
expect(environment.logger).to receive(:debug).with(
|
256
|
-
:
|
257
|
-
:
|
258
|
-
:
|
259
|
-
:
|
260
|
-
:
|
261
|
-
:
|
262
|
-
:
|
263
|
-
:
|
264
|
-
:
|
265
|
-
:
|
266
|
-
:
|
256
|
+
message: "Could not coerce 'INVALID_JSON' into :json for column 'preferences'",
|
257
|
+
source: 'input_stage',
|
258
|
+
target: 'user_dimension_ledger',
|
259
|
+
file: input.path,
|
260
|
+
line: 4,
|
261
|
+
row: {
|
262
|
+
id: '3',
|
263
|
+
tenant_id: '50',
|
264
|
+
admin: '0',
|
265
|
+
preferences: 'INVALID_JSON',
|
266
|
+
deleted_at: nil
|
267
267
|
}
|
268
268
|
).ordered
|
269
269
|
end
|
@@ -698,7 +698,7 @@ describe Masamune::Schema::Map do
|
|
698
698
|
column 'id', type: :integer
|
699
699
|
end
|
700
700
|
|
701
|
-
map from: files.input, to: files.output, columns: [:id], fail_fast: true do |
|
701
|
+
map from: files.input, to: files.output, columns: [:id], fail_fast: true do |_row|
|
702
702
|
raise 'wha happen'
|
703
703
|
end
|
704
704
|
end
|
@@ -729,17 +729,17 @@ describe Masamune::Schema::Map do
|
|
729
729
|
before do
|
730
730
|
expect(environment.logger).to receive(:error).with(/wha happen/).ordered
|
731
731
|
expect(environment.logger).to receive(:debug).with(
|
732
|
-
:
|
733
|
-
:
|
734
|
-
:
|
735
|
-
:
|
736
|
-
:
|
737
|
-
:
|
732
|
+
message: 'wha happen',
|
733
|
+
source: 'input_stage',
|
734
|
+
target: 'output_stage',
|
735
|
+
file: input.path,
|
736
|
+
line: 0,
|
737
|
+
row: { 'id' => 1 }
|
738
738
|
).ordered
|
739
739
|
end
|
740
740
|
|
741
741
|
it 'raises exception' do
|
742
|
-
expect { subject }.to raise_error
|
742
|
+
expect { subject }.to raise_error(/wha happen/)
|
743
743
|
end
|
744
744
|
end
|
745
745
|
end
|
@@ -806,7 +806,7 @@ describe Masamune::Schema::Map do
|
|
806
806
|
io.write '{},{}'
|
807
807
|
io.rewind
|
808
808
|
end
|
809
|
-
it { is_expected.to eq(%
|
809
|
+
it { is_expected.to eq(%("{}","{}")) }
|
810
810
|
end
|
811
811
|
|
812
812
|
context 'with quoted empty json' do
|
@@ -814,7 +814,7 @@ describe Masamune::Schema::Map do
|
|
814
814
|
io.write '"{}","{}"'
|
815
815
|
io.rewind
|
816
816
|
end
|
817
|
-
it { is_expected.to eq(%
|
817
|
+
it { is_expected.to eq(%("{}","{}")) }
|
818
818
|
end
|
819
819
|
|
820
820
|
context 'with raw json' do
|
@@ -822,7 +822,7 @@ describe Masamune::Schema::Map do
|
|
822
822
|
io.write '{"enabled":true,"state":""}'
|
823
823
|
io.rewind
|
824
824
|
end
|
825
|
-
it { is_expected.to eq(%
|
825
|
+
it { is_expected.to eq(%("{""enabled"":true,""state"":""""}")) }
|
826
826
|
end
|
827
827
|
|
828
828
|
context 'with quoted json' do
|
@@ -830,7 +830,7 @@ describe Masamune::Schema::Map do
|
|
830
830
|
io.write '"{""enabled"":true,""state"":""""}"'
|
831
831
|
io.rewind
|
832
832
|
end
|
833
|
-
it { is_expected.to eq(%
|
833
|
+
it { is_expected.to eq(%("{""enabled"":true,""state"":""""}")) }
|
834
834
|
end
|
835
835
|
end
|
836
836
|
end
|