masamune 0.17.12 → 0.17.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -273,8 +273,10 @@ describe Masamune::DataPlan::Set do
|
|
273
273
|
end
|
274
274
|
|
275
275
|
describe '#with_grain' do
|
276
|
-
let(:paths)
|
277
|
-
|
276
|
+
let(:paths) do
|
277
|
+
['/table/y=2012/m=12/d=29', '/table/y=2012/m=12/d=30', '/table/y=2012/m=12/d=31',
|
278
|
+
'/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=02/d=01']
|
279
|
+
end
|
278
280
|
|
279
281
|
let(:instance) { Masamune::DataPlan::Set.new(target_rule, paths) }
|
280
282
|
|
@@ -346,7 +348,7 @@ describe Masamune::DataPlan::Set do
|
|
346
348
|
let(:enum) { ['/log/20130101.*.log', '/log/20130102.*.log'] }
|
347
349
|
let(:elem) { '/log/20130101.random_1.log' }
|
348
350
|
|
349
|
-
it { is_expected.to eq(false)
|
351
|
+
it { is_expected.to eq(false) }
|
350
352
|
end
|
351
353
|
end
|
352
354
|
|
@@ -371,7 +373,7 @@ describe Masamune::DataPlan::Set do
|
|
371
373
|
context 'when some incomplete' do
|
372
374
|
before do
|
373
375
|
(1..31).each do |day|
|
374
|
-
fs.touch!('/log/201401%02d.random_1.log'
|
376
|
+
fs.touch!(format('/log/201401%02d.random_1.log', day))
|
375
377
|
end
|
376
378
|
end
|
377
379
|
|
@@ -382,8 +384,8 @@ describe Masamune::DataPlan::Set do
|
|
382
384
|
context 'when none incomplete' do
|
383
385
|
before do
|
384
386
|
(1..31).each do |day|
|
385
|
-
fs.touch!('/log/201401%02d.random_1.log'
|
386
|
-
fs.touch!('/log/201402%02d.random_1.log'
|
387
|
+
fs.touch!(format('/log/201401%02d.random_1.log', day))
|
388
|
+
fs.touch!(format('/log/201402%02d.random_1.log', day))
|
387
389
|
end
|
388
390
|
end
|
389
391
|
|
@@ -26,15 +26,15 @@ describe Masamune::Environment do
|
|
26
26
|
|
27
27
|
describe '#with_exclusive_lock' do
|
28
28
|
context 'when run_dir not defined' do
|
29
|
-
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error
|
29
|
+
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error(/filesystem path :run_dir not defined/) }
|
30
30
|
end
|
31
31
|
|
32
32
|
context 'when lock can be acquired' do
|
33
33
|
before do
|
34
34
|
instance.filesystem.add_path(:run_dir, run_dir)
|
35
35
|
expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
|
36
|
-
expect(instance.logger).to receive(:debug).with(
|
37
|
-
expect(instance.logger).to receive(:debug).with(
|
36
|
+
expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock'")
|
37
|
+
expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock'")
|
38
38
|
end
|
39
39
|
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
|
40
40
|
end
|
@@ -44,8 +44,8 @@ describe Masamune::Environment do
|
|
44
44
|
instance.filesystem.add_path(:run_dir, run_dir)
|
45
45
|
instance.configuration.lock = 'long_running'
|
46
46
|
expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
|
47
|
-
expect(instance.logger).to receive(:debug).with(
|
48
|
-
expect(instance.logger).to receive(:debug).with(
|
47
|
+
expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock:long_running'")
|
48
|
+
expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock:long_running'")
|
49
49
|
end
|
50
50
|
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
|
51
51
|
end
|
@@ -129,22 +129,22 @@ shared_examples_for 'Filesystem' do
|
|
129
129
|
|
130
130
|
context 'with nil' do
|
131
131
|
let(:path) { nil }
|
132
|
-
it { expect { |
|
132
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
133
133
|
end
|
134
134
|
|
135
135
|
context 'with blank' do
|
136
136
|
let(:path) { ' ' }
|
137
|
-
it { expect { |
|
137
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
138
138
|
end
|
139
139
|
|
140
140
|
context 'with empty' do
|
141
141
|
let(:path) { '' }
|
142
|
-
it { expect { |
|
142
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
143
143
|
end
|
144
144
|
|
145
145
|
context 'with relative path' do
|
146
146
|
let(:path) { 'tmp' }
|
147
|
-
it { expect { |
|
147
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
148
148
|
end
|
149
149
|
|
150
150
|
context 'with local root' do
|
@@ -337,7 +337,7 @@ shared_examples_for 'Filesystem' do
|
|
337
337
|
|
338
338
|
describe '#touch!' do
|
339
339
|
subject do
|
340
|
-
File.
|
340
|
+
File.exist?(new_file) && File.exist?(other_new_file)
|
341
341
|
end
|
342
342
|
|
343
343
|
context 'local' do
|
@@ -372,8 +372,8 @@ shared_examples_for 'Filesystem' do
|
|
372
372
|
|
373
373
|
context 'hdfs missing file' do
|
374
374
|
before do
|
375
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
376
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
375
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
376
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
377
377
|
expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + new_file, safe: true).at_most(:once).and_return(mock_failure)
|
378
378
|
end
|
379
379
|
subject { instance.exists?('file://' + new_file) }
|
@@ -395,9 +395,9 @@ shared_examples_for 'Filesystem' do
|
|
395
395
|
|
396
396
|
context 'hdfs existing file' do
|
397
397
|
before do
|
398
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
399
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
400
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
398
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
399
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
400
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
401
401
|
expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + old_file, safe: true).at_most(:once).and_return(mock_success)
|
402
402
|
end
|
403
403
|
subject { instance.exists?('file://' + old_file) }
|
@@ -406,12 +406,12 @@ shared_examples_for 'Filesystem' do
|
|
406
406
|
|
407
407
|
context 's3 existing file' do
|
408
408
|
before do
|
409
|
-
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once)
|
410
|
-
and_yield(
|
411
|
-
and_yield(
|
412
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once)
|
413
|
-
and_yield(
|
414
|
-
and_yield(
|
409
|
+
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once)
|
410
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
411
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
412
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once)
|
413
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
414
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
415
415
|
end
|
416
416
|
|
417
417
|
subject { instance.exists?('s3://bucket/00') }
|
@@ -421,12 +421,12 @@ shared_examples_for 'Filesystem' do
|
|
421
421
|
|
422
422
|
context 's3 missing file' do
|
423
423
|
before do
|
424
|
-
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once)
|
425
|
-
and_yield(
|
426
|
-
and_yield(
|
427
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once)
|
428
|
-
and_yield(
|
429
|
-
and_yield(
|
424
|
+
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once)
|
425
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
426
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
427
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once)
|
428
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
429
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
430
430
|
end
|
431
431
|
|
432
432
|
subject { instance.exists?('s3://bucket/0') }
|
@@ -444,14 +444,14 @@ shared_examples_for 'Filesystem' do
|
|
444
444
|
|
445
445
|
context 'hdfs missing file' do
|
446
446
|
before do
|
447
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
448
|
-
and_yield('')
|
449
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
450
|
-
and_yield('')
|
451
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once)
|
452
|
-
and_yield('')
|
453
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once)
|
454
|
-
and_yield('')
|
447
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
448
|
+
.and_yield('')
|
449
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
450
|
+
.and_yield('')
|
451
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once)
|
452
|
+
.and_yield('')
|
453
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once)
|
454
|
+
.and_yield('')
|
455
455
|
end
|
456
456
|
let(:result) { instance.stat('file://' + new_file) }
|
457
457
|
it { is_expected.to be_nil }
|
@@ -460,8 +460,8 @@ shared_examples_for 'Filesystem' do
|
|
460
460
|
context 's3 missing file' do
|
461
461
|
before do
|
462
462
|
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/', safe: true).at_most(:once)
|
463
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
464
|
-
and_yield('')
|
463
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
464
|
+
.and_yield('')
|
465
465
|
end
|
466
466
|
let(:result) { instance.stat('s3://bucket/file.txt') }
|
467
467
|
it { is_expected.to be_nil }
|
@@ -494,16 +494,16 @@ shared_examples_for 'Filesystem' do
|
|
494
494
|
|
495
495
|
context 'local existing file (recursive)' do
|
496
496
|
let(:result) { instance.stat(File.join(tmp_dir, '*')) }
|
497
|
-
it { expect { result }.to raise_error
|
497
|
+
it { expect { result }.to raise_error(/cannot contain wildcard/) }
|
498
498
|
end
|
499
499
|
|
500
500
|
context 'hdfs existing file' do
|
501
501
|
before do
|
502
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
503
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
504
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
505
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once)
|
506
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
502
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
503
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
504
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
505
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once)
|
506
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
507
507
|
end
|
508
508
|
|
509
509
|
let(:result) { instance.stat('file://' + old_file) }
|
@@ -527,8 +527,8 @@ shared_examples_for 'Filesystem' do
|
|
527
527
|
|
528
528
|
context 's3 existing file' do
|
529
529
|
before do
|
530
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
531
|
-
and_yield(
|
530
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
531
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/file.txt')
|
532
532
|
end
|
533
533
|
let(:result) { instance.stat('s3://bucket/file.txt') }
|
534
534
|
|
@@ -551,8 +551,8 @@ shared_examples_for 'Filesystem' do
|
|
551
551
|
|
552
552
|
context 's3 existing directory' do
|
553
553
|
before do
|
554
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true)
|
555
|
-
and_yield(
|
554
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true)
|
555
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
|
556
556
|
end
|
557
557
|
|
558
558
|
let(:result) { instance.stat('s3://bucket/dir') }
|
@@ -577,7 +577,7 @@ shared_examples_for 'Filesystem' do
|
|
577
577
|
|
578
578
|
describe '#mkdir!' do
|
579
579
|
subject do
|
580
|
-
Dir.
|
580
|
+
Dir.exist?(new_dir) && Dir.exist?(other_new_dir)
|
581
581
|
end
|
582
582
|
|
583
583
|
context 'local directory' do
|
@@ -588,7 +588,7 @@ shared_examples_for 'Filesystem' do
|
|
588
588
|
end
|
589
589
|
|
590
590
|
context 'local existing directory' do
|
591
|
-
subject { Dir.
|
591
|
+
subject { Dir.exist?(old_dir) }
|
592
592
|
before do
|
593
593
|
expect(FileUtils).to receive(:mkdir_p).never
|
594
594
|
instance.mkdir!(old_dir)
|
@@ -660,12 +660,12 @@ shared_examples_for 'Filesystem' do
|
|
660
660
|
|
661
661
|
context 'hdfs no matches' do
|
662
662
|
before do
|
663
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
664
|
-
and_yield('')
|
665
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
666
|
-
and_yield('')
|
667
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once)
|
668
|
-
and_yield('')
|
663
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
664
|
+
.and_yield('')
|
665
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
666
|
+
.and_yield('')
|
667
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once)
|
668
|
+
.and_yield('')
|
669
669
|
end
|
670
670
|
let(:pattern) { File.join(new_dir, '*') }
|
671
671
|
it { expect(subject.count).to eq(0) }
|
@@ -674,12 +674,12 @@ shared_examples_for 'Filesystem' do
|
|
674
674
|
|
675
675
|
context 'hdfs one matches' do
|
676
676
|
before do
|
677
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
678
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
679
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
680
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
681
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once)
|
682
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
677
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
678
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
679
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
680
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
681
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once)
|
682
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
683
683
|
end
|
684
684
|
let(:pattern) { File.join(File.dirname(old_file), '*') }
|
685
685
|
it { expect(subject.count).to eq(1) }
|
@@ -690,8 +690,8 @@ shared_examples_for 'Filesystem' do
|
|
690
690
|
let(:pattern) { 's3://bucket/dir/*.txt' }
|
691
691
|
|
692
692
|
before do
|
693
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive',
|
694
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive',
|
693
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir', safe: true).at_most(:once)
|
694
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir/*', safe: true).at_most(:once)
|
695
695
|
end
|
696
696
|
|
697
697
|
it { expect(subject.count).to eq(0) }
|
@@ -701,9 +701,9 @@ shared_examples_for 'Filesystem' do
|
|
701
701
|
let(:pattern) { 's3://bucket/dir/0' }
|
702
702
|
|
703
703
|
before do
|
704
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
705
|
-
and_yield(
|
706
|
-
and_yield(
|
704
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
705
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/01.txt')
|
706
|
+
.and_yield('2013-05-24 18:53 2912 s3://bucket/dir/02.txt')
|
707
707
|
end
|
708
708
|
|
709
709
|
it { expect(subject.count).to eq(0) }
|
@@ -713,9 +713,9 @@ shared_examples_for 'Filesystem' do
|
|
713
713
|
let(:pattern) { 's3://bucket/dir/*.txt' }
|
714
714
|
|
715
715
|
before do
|
716
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
717
|
-
and_yield(
|
718
|
-
and_yield(
|
716
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
717
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
|
718
|
+
.and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
|
719
719
|
end
|
720
720
|
|
721
721
|
it { is_expected.to include 's3://bucket/dir/file.txt' }
|
@@ -726,10 +726,10 @@ shared_examples_for 'Filesystem' do
|
|
726
726
|
let(:pattern) { 's3://bucket/dir/*' }
|
727
727
|
|
728
728
|
before do
|
729
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
730
|
-
and_yield(
|
731
|
-
and_yield(
|
732
|
-
and_yield(
|
729
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
730
|
+
.and_yield(' DIR s3://bucket/dir/file_$folder$')
|
731
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
|
732
|
+
.and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
|
733
733
|
end
|
734
734
|
|
735
735
|
it { is_expected.to include 's3://bucket/dir/file.txt' }
|
@@ -754,7 +754,7 @@ shared_examples_for 'Filesystem' do
|
|
754
754
|
let(:result_file) { File.join(new_dir, File.basename(old_file)) }
|
755
755
|
|
756
756
|
subject do
|
757
|
-
File.
|
757
|
+
File.exist?(result_file)
|
758
758
|
end
|
759
759
|
|
760
760
|
context 'local file to local file' do
|
@@ -829,7 +829,7 @@ shared_examples_for 'Filesystem' do
|
|
829
829
|
let(:result_file) { File.join(new_dir, File.basename(old_file)) }
|
830
830
|
|
831
831
|
subject do
|
832
|
-
File.
|
832
|
+
File.exist?(result_file)
|
833
833
|
end
|
834
834
|
|
835
835
|
context 'local file to local dir' do
|
@@ -910,7 +910,7 @@ shared_examples_for 'Filesystem' do
|
|
910
910
|
|
911
911
|
describe '#copy_dir' do
|
912
912
|
subject do
|
913
|
-
File.
|
913
|
+
File.exist?(File.join(new_dir, File.basename(old_dir), File.basename(old_file)))
|
914
914
|
end
|
915
915
|
|
916
916
|
context 'local dir to local dir' do
|
@@ -983,7 +983,7 @@ shared_examples_for 'Filesystem' do
|
|
983
983
|
|
984
984
|
describe '#remove_file' do
|
985
985
|
subject do
|
986
|
-
File.
|
986
|
+
File.exist?(old_file)
|
987
987
|
end
|
988
988
|
|
989
989
|
context 'local false' do
|
@@ -991,7 +991,7 @@ shared_examples_for 'Filesystem' do
|
|
991
991
|
instance.remove_file(old_file)
|
992
992
|
end
|
993
993
|
|
994
|
-
it { is_expected.to eq(false)}
|
994
|
+
it { is_expected.to eq(false) }
|
995
995
|
end
|
996
996
|
|
997
997
|
context 'hdfs file' do
|
@@ -1011,7 +1011,7 @@ shared_examples_for 'Filesystem' do
|
|
1011
1011
|
|
1012
1012
|
describe '#remove_dir' do
|
1013
1013
|
subject do
|
1014
|
-
File.
|
1014
|
+
File.exist?(old_dir)
|
1015
1015
|
end
|
1016
1016
|
|
1017
1017
|
context 'local dir' do
|
@@ -1020,7 +1020,7 @@ shared_examples_for 'Filesystem' do
|
|
1020
1020
|
instance.remove_dir(old_dir)
|
1021
1021
|
end
|
1022
1022
|
|
1023
|
-
it { is_expected.to eq(false)}
|
1023
|
+
it { is_expected.to eq(false) }
|
1024
1024
|
end
|
1025
1025
|
|
1026
1026
|
context 'local root dir' do
|
@@ -1028,7 +1028,7 @@ shared_examples_for 'Filesystem' do
|
|
1028
1028
|
expect(filesystem).to receive(:root_path?).once.and_return(true)
|
1029
1029
|
end
|
1030
1030
|
|
1031
|
-
it { expect { instance.remove_dir(old_dir) }.to raise_error
|
1031
|
+
it { expect { instance.remove_dir(old_dir) }.to raise_error(/root path/) }
|
1032
1032
|
end
|
1033
1033
|
|
1034
1034
|
context 'hdfs dir' do
|
@@ -1044,7 +1044,7 @@ shared_examples_for 'Filesystem' do
|
|
1044
1044
|
expect(filesystem).to receive(:root_path?).once.and_return(true)
|
1045
1045
|
end
|
1046
1046
|
|
1047
|
-
it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error
|
1047
|
+
it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error(/root path/) }
|
1048
1048
|
end
|
1049
1049
|
|
1050
1050
|
context 's3 dir' do
|
@@ -1060,17 +1060,17 @@ shared_examples_for 'Filesystem' do
|
|
1060
1060
|
expect(filesystem).to receive(:s3cmd).never
|
1061
1061
|
end
|
1062
1062
|
|
1063
|
-
it { expect { instance.remove_dir('s3://bucket/') }.to raise_error
|
1063
|
+
it { expect { instance.remove_dir('s3://bucket/') }.to raise_error(/root path/) }
|
1064
1064
|
end
|
1065
1065
|
end
|
1066
1066
|
|
1067
1067
|
describe '#move_file_to_file' do
|
1068
1068
|
subject(:removes_old_file) do
|
1069
|
-
!File.
|
1069
|
+
!File.exist?(old_file)
|
1070
1070
|
end
|
1071
1071
|
|
1072
1072
|
subject(:creates_new_file) do
|
1073
|
-
File.
|
1073
|
+
File.exist?(new_file)
|
1074
1074
|
end
|
1075
1075
|
|
1076
1076
|
context 'local file to local file' do
|
@@ -1095,7 +1095,7 @@ shared_examples_for 'Filesystem' do
|
|
1095
1095
|
context 'local file to hdfs file' do
|
1096
1096
|
it do
|
1097
1097
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + old_dir).once
|
1098
|
-
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' +
|
1098
|
+
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_file)
|
1099
1099
|
instance.move_file_to_file(old_file, 'file://' + new_file)
|
1100
1100
|
end
|
1101
1101
|
end
|
@@ -1154,11 +1154,11 @@ shared_examples_for 'Filesystem' do
|
|
1154
1154
|
end
|
1155
1155
|
|
1156
1156
|
subject(:removes_old_file) do
|
1157
|
-
!File.
|
1157
|
+
!File.exist?(old_file)
|
1158
1158
|
end
|
1159
1159
|
|
1160
1160
|
subject(:creates_new_file) do
|
1161
|
-
File.
|
1161
|
+
File.exist?(File.join(new_dir, File.basename(old_file)))
|
1162
1162
|
end
|
1163
1163
|
|
1164
1164
|
context 'local file to local dir' do
|
@@ -1183,7 +1183,7 @@ shared_examples_for 'Filesystem' do
|
|
1183
1183
|
context 'local file to hdfs dir' do
|
1184
1184
|
it do
|
1185
1185
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir).once
|
1186
|
-
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' +
|
1186
|
+
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_dir)
|
1187
1187
|
instance.move_file_to_dir(old_file, 'file://' + new_dir)
|
1188
1188
|
end
|
1189
1189
|
end
|
@@ -1238,11 +1238,11 @@ shared_examples_for 'Filesystem' do
|
|
1238
1238
|
|
1239
1239
|
describe '#move_dir' do
|
1240
1240
|
subject(:removes_old_dir) do
|
1241
|
-
!File.
|
1241
|
+
!File.exist?(old_dir)
|
1242
1242
|
end
|
1243
1243
|
|
1244
1244
|
subject(:creates_new_dir) do
|
1245
|
-
File.
|
1245
|
+
File.exist?(new_dir)
|
1246
1246
|
end
|
1247
1247
|
|
1248
1248
|
context 'local dir to local dir' do
|
@@ -1266,7 +1266,7 @@ shared_examples_for 'Filesystem' do
|
|
1266
1266
|
context 'local dir to hdfs dir' do
|
1267
1267
|
it do
|
1268
1268
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + File.dirname(new_dir)).once
|
1269
|
-
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' +
|
1269
|
+
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_dir, 'file://' + new_dir)
|
1270
1270
|
instance.move_dir(old_dir, 'file://' + new_dir)
|
1271
1271
|
end
|
1272
1272
|
end
|