masamune 0.17.12 → 0.17.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -273,8 +273,10 @@ describe Masamune::DataPlan::Set do
|
|
273
273
|
end
|
274
274
|
|
275
275
|
describe '#with_grain' do
|
276
|
-
let(:paths)
|
277
|
-
|
276
|
+
let(:paths) do
|
277
|
+
['/table/y=2012/m=12/d=29', '/table/y=2012/m=12/d=30', '/table/y=2012/m=12/d=31',
|
278
|
+
'/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=02/d=01']
|
279
|
+
end
|
278
280
|
|
279
281
|
let(:instance) { Masamune::DataPlan::Set.new(target_rule, paths) }
|
280
282
|
|
@@ -346,7 +348,7 @@ describe Masamune::DataPlan::Set do
|
|
346
348
|
let(:enum) { ['/log/20130101.*.log', '/log/20130102.*.log'] }
|
347
349
|
let(:elem) { '/log/20130101.random_1.log' }
|
348
350
|
|
349
|
-
it { is_expected.to eq(false)
|
351
|
+
it { is_expected.to eq(false) }
|
350
352
|
end
|
351
353
|
end
|
352
354
|
|
@@ -371,7 +373,7 @@ describe Masamune::DataPlan::Set do
|
|
371
373
|
context 'when some incomplete' do
|
372
374
|
before do
|
373
375
|
(1..31).each do |day|
|
374
|
-
fs.touch!('/log/201401%02d.random_1.log'
|
376
|
+
fs.touch!(format('/log/201401%02d.random_1.log', day))
|
375
377
|
end
|
376
378
|
end
|
377
379
|
|
@@ -382,8 +384,8 @@ describe Masamune::DataPlan::Set do
|
|
382
384
|
context 'when none incomplete' do
|
383
385
|
before do
|
384
386
|
(1..31).each do |day|
|
385
|
-
fs.touch!('/log/201401%02d.random_1.log'
|
386
|
-
fs.touch!('/log/201402%02d.random_1.log'
|
387
|
+
fs.touch!(format('/log/201401%02d.random_1.log', day))
|
388
|
+
fs.touch!(format('/log/201402%02d.random_1.log', day))
|
387
389
|
end
|
388
390
|
end
|
389
391
|
|
@@ -26,15 +26,15 @@ describe Masamune::Environment do
|
|
26
26
|
|
27
27
|
describe '#with_exclusive_lock' do
|
28
28
|
context 'when run_dir not defined' do
|
29
|
-
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error
|
29
|
+
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error(/filesystem path :run_dir not defined/) }
|
30
30
|
end
|
31
31
|
|
32
32
|
context 'when lock can be acquired' do
|
33
33
|
before do
|
34
34
|
instance.filesystem.add_path(:run_dir, run_dir)
|
35
35
|
expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
|
36
|
-
expect(instance.logger).to receive(:debug).with(
|
37
|
-
expect(instance.logger).to receive(:debug).with(
|
36
|
+
expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock'")
|
37
|
+
expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock'")
|
38
38
|
end
|
39
39
|
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
|
40
40
|
end
|
@@ -44,8 +44,8 @@ describe Masamune::Environment do
|
|
44
44
|
instance.filesystem.add_path(:run_dir, run_dir)
|
45
45
|
instance.configuration.lock = 'long_running'
|
46
46
|
expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
|
47
|
-
expect(instance.logger).to receive(:debug).with(
|
48
|
-
expect(instance.logger).to receive(:debug).with(
|
47
|
+
expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock:long_running'")
|
48
|
+
expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock:long_running'")
|
49
49
|
end
|
50
50
|
it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
|
51
51
|
end
|
@@ -129,22 +129,22 @@ shared_examples_for 'Filesystem' do
|
|
129
129
|
|
130
130
|
context 'with nil' do
|
131
131
|
let(:path) { nil }
|
132
|
-
it { expect { |
|
132
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
133
133
|
end
|
134
134
|
|
135
135
|
context 'with blank' do
|
136
136
|
let(:path) { ' ' }
|
137
|
-
it { expect { |
|
137
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
138
138
|
end
|
139
139
|
|
140
140
|
context 'with empty' do
|
141
141
|
let(:path) { '' }
|
142
|
-
it { expect { |
|
142
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
143
143
|
end
|
144
144
|
|
145
145
|
context 'with relative path' do
|
146
146
|
let(:path) { 'tmp' }
|
147
|
-
it { expect { |
|
147
|
+
it { expect { |_b| subject }.to raise_error ArgumentError }
|
148
148
|
end
|
149
149
|
|
150
150
|
context 'with local root' do
|
@@ -337,7 +337,7 @@ shared_examples_for 'Filesystem' do
|
|
337
337
|
|
338
338
|
describe '#touch!' do
|
339
339
|
subject do
|
340
|
-
File.
|
340
|
+
File.exist?(new_file) && File.exist?(other_new_file)
|
341
341
|
end
|
342
342
|
|
343
343
|
context 'local' do
|
@@ -372,8 +372,8 @@ shared_examples_for 'Filesystem' do
|
|
372
372
|
|
373
373
|
context 'hdfs missing file' do
|
374
374
|
before do
|
375
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
376
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
375
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
376
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
377
377
|
expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + new_file, safe: true).at_most(:once).and_return(mock_failure)
|
378
378
|
end
|
379
379
|
subject { instance.exists?('file://' + new_file) }
|
@@ -395,9 +395,9 @@ shared_examples_for 'Filesystem' do
|
|
395
395
|
|
396
396
|
context 'hdfs existing file' do
|
397
397
|
before do
|
398
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
399
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
400
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
398
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
399
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
400
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
401
401
|
expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + old_file, safe: true).at_most(:once).and_return(mock_success)
|
402
402
|
end
|
403
403
|
subject { instance.exists?('file://' + old_file) }
|
@@ -406,12 +406,12 @@ shared_examples_for 'Filesystem' do
|
|
406
406
|
|
407
407
|
context 's3 existing file' do
|
408
408
|
before do
|
409
|
-
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once)
|
410
|
-
and_yield(
|
411
|
-
and_yield(
|
412
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once)
|
413
|
-
and_yield(
|
414
|
-
and_yield(
|
409
|
+
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once)
|
410
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
411
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
412
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once)
|
413
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
414
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
415
415
|
end
|
416
416
|
|
417
417
|
subject { instance.exists?('s3://bucket/00') }
|
@@ -421,12 +421,12 @@ shared_examples_for 'Filesystem' do
|
|
421
421
|
|
422
422
|
context 's3 missing file' do
|
423
423
|
before do
|
424
|
-
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once)
|
425
|
-
and_yield(
|
426
|
-
and_yield(
|
427
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once)
|
428
|
-
and_yield(
|
429
|
-
and_yield(
|
424
|
+
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once)
|
425
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
426
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
427
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once)
|
428
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/00')
|
429
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/01')
|
430
430
|
end
|
431
431
|
|
432
432
|
subject { instance.exists?('s3://bucket/0') }
|
@@ -444,14 +444,14 @@ shared_examples_for 'Filesystem' do
|
|
444
444
|
|
445
445
|
context 'hdfs missing file' do
|
446
446
|
before do
|
447
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
448
|
-
and_yield('')
|
449
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
450
|
-
and_yield('')
|
451
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once)
|
452
|
-
and_yield('')
|
453
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once)
|
454
|
-
and_yield('')
|
447
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
448
|
+
.and_yield('')
|
449
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
450
|
+
.and_yield('')
|
451
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once)
|
452
|
+
.and_yield('')
|
453
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once)
|
454
|
+
.and_yield('')
|
455
455
|
end
|
456
456
|
let(:result) { instance.stat('file://' + new_file) }
|
457
457
|
it { is_expected.to be_nil }
|
@@ -460,8 +460,8 @@ shared_examples_for 'Filesystem' do
|
|
460
460
|
context 's3 missing file' do
|
461
461
|
before do
|
462
462
|
expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/', safe: true).at_most(:once)
|
463
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
464
|
-
and_yield('')
|
463
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
464
|
+
.and_yield('')
|
465
465
|
end
|
466
466
|
let(:result) { instance.stat('s3://bucket/file.txt') }
|
467
467
|
it { is_expected.to be_nil }
|
@@ -494,16 +494,16 @@ shared_examples_for 'Filesystem' do
|
|
494
494
|
|
495
495
|
context 'local existing file (recursive)' do
|
496
496
|
let(:result) { instance.stat(File.join(tmp_dir, '*')) }
|
497
|
-
it { expect { result }.to raise_error
|
497
|
+
it { expect { result }.to raise_error(/cannot contain wildcard/) }
|
498
498
|
end
|
499
499
|
|
500
500
|
context 'hdfs existing file' do
|
501
501
|
before do
|
502
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
503
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
504
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
505
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once)
|
506
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
502
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
|
503
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
|
504
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
505
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once)
|
506
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
507
507
|
end
|
508
508
|
|
509
509
|
let(:result) { instance.stat('file://' + old_file) }
|
@@ -527,8 +527,8 @@ shared_examples_for 'Filesystem' do
|
|
527
527
|
|
528
528
|
context 's3 existing file' do
|
529
529
|
before do
|
530
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
531
|
-
and_yield(
|
530
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
|
531
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/file.txt')
|
532
532
|
end
|
533
533
|
let(:result) { instance.stat('s3://bucket/file.txt') }
|
534
534
|
|
@@ -551,8 +551,8 @@ shared_examples_for 'Filesystem' do
|
|
551
551
|
|
552
552
|
context 's3 existing directory' do
|
553
553
|
before do
|
554
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true)
|
555
|
-
and_yield(
|
554
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true)
|
555
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
|
556
556
|
end
|
557
557
|
|
558
558
|
let(:result) { instance.stat('s3://bucket/dir') }
|
@@ -577,7 +577,7 @@ shared_examples_for 'Filesystem' do
|
|
577
577
|
|
578
578
|
describe '#mkdir!' do
|
579
579
|
subject do
|
580
|
-
Dir.
|
580
|
+
Dir.exist?(new_dir) && Dir.exist?(other_new_dir)
|
581
581
|
end
|
582
582
|
|
583
583
|
context 'local directory' do
|
@@ -588,7 +588,7 @@ shared_examples_for 'Filesystem' do
|
|
588
588
|
end
|
589
589
|
|
590
590
|
context 'local existing directory' do
|
591
|
-
subject { Dir.
|
591
|
+
subject { Dir.exist?(old_dir) }
|
592
592
|
before do
|
593
593
|
expect(FileUtils).to receive(:mkdir_p).never
|
594
594
|
instance.mkdir!(old_dir)
|
@@ -660,12 +660,12 @@ shared_examples_for 'Filesystem' do
|
|
660
660
|
|
661
661
|
context 'hdfs no matches' do
|
662
662
|
before do
|
663
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
664
|
-
and_yield('')
|
665
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
666
|
-
and_yield('')
|
667
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once)
|
668
|
-
and_yield('')
|
663
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
664
|
+
.and_yield('')
|
665
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once)
|
666
|
+
.and_yield('')
|
667
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once)
|
668
|
+
.and_yield('')
|
669
669
|
end
|
670
670
|
let(:pattern) { File.join(new_dir, '*') }
|
671
671
|
it { expect(subject.count).to eq(0) }
|
@@ -674,12 +674,12 @@ shared_examples_for 'Filesystem' do
|
|
674
674
|
|
675
675
|
context 'hdfs one matches' do
|
676
676
|
before do
|
677
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
678
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
679
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
680
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
681
|
-
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once)
|
682
|
-
and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
677
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
678
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
679
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
|
680
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
681
|
+
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once)
|
682
|
+
.and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
|
683
683
|
end
|
684
684
|
let(:pattern) { File.join(File.dirname(old_file), '*') }
|
685
685
|
it { expect(subject.count).to eq(1) }
|
@@ -690,8 +690,8 @@ shared_examples_for 'Filesystem' do
|
|
690
690
|
let(:pattern) { 's3://bucket/dir/*.txt' }
|
691
691
|
|
692
692
|
before do
|
693
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive',
|
694
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive',
|
693
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir', safe: true).at_most(:once)
|
694
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir/*', safe: true).at_most(:once)
|
695
695
|
end
|
696
696
|
|
697
697
|
it { expect(subject.count).to eq(0) }
|
@@ -701,9 +701,9 @@ shared_examples_for 'Filesystem' do
|
|
701
701
|
let(:pattern) { 's3://bucket/dir/0' }
|
702
702
|
|
703
703
|
before do
|
704
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
705
|
-
and_yield(
|
706
|
-
and_yield(
|
704
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
705
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/01.txt')
|
706
|
+
.and_yield('2013-05-24 18:53 2912 s3://bucket/dir/02.txt')
|
707
707
|
end
|
708
708
|
|
709
709
|
it { expect(subject.count).to eq(0) }
|
@@ -713,9 +713,9 @@ shared_examples_for 'Filesystem' do
|
|
713
713
|
let(:pattern) { 's3://bucket/dir/*.txt' }
|
714
714
|
|
715
715
|
before do
|
716
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
717
|
-
and_yield(
|
718
|
-
and_yield(
|
716
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
717
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
|
718
|
+
.and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
|
719
719
|
end
|
720
720
|
|
721
721
|
it { is_expected.to include 's3://bucket/dir/file.txt' }
|
@@ -726,10 +726,10 @@ shared_examples_for 'Filesystem' do
|
|
726
726
|
let(:pattern) { 's3://bucket/dir/*' }
|
727
727
|
|
728
728
|
before do
|
729
|
-
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
730
|
-
and_yield(
|
731
|
-
and_yield(
|
732
|
-
and_yield(
|
729
|
+
expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
|
730
|
+
.and_yield(' DIR s3://bucket/dir/file_$folder$')
|
731
|
+
.and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
|
732
|
+
.and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
|
733
733
|
end
|
734
734
|
|
735
735
|
it { is_expected.to include 's3://bucket/dir/file.txt' }
|
@@ -754,7 +754,7 @@ shared_examples_for 'Filesystem' do
|
|
754
754
|
let(:result_file) { File.join(new_dir, File.basename(old_file)) }
|
755
755
|
|
756
756
|
subject do
|
757
|
-
File.
|
757
|
+
File.exist?(result_file)
|
758
758
|
end
|
759
759
|
|
760
760
|
context 'local file to local file' do
|
@@ -829,7 +829,7 @@ shared_examples_for 'Filesystem' do
|
|
829
829
|
let(:result_file) { File.join(new_dir, File.basename(old_file)) }
|
830
830
|
|
831
831
|
subject do
|
832
|
-
File.
|
832
|
+
File.exist?(result_file)
|
833
833
|
end
|
834
834
|
|
835
835
|
context 'local file to local dir' do
|
@@ -910,7 +910,7 @@ shared_examples_for 'Filesystem' do
|
|
910
910
|
|
911
911
|
describe '#copy_dir' do
|
912
912
|
subject do
|
913
|
-
File.
|
913
|
+
File.exist?(File.join(new_dir, File.basename(old_dir), File.basename(old_file)))
|
914
914
|
end
|
915
915
|
|
916
916
|
context 'local dir to local dir' do
|
@@ -983,7 +983,7 @@ shared_examples_for 'Filesystem' do
|
|
983
983
|
|
984
984
|
describe '#remove_file' do
|
985
985
|
subject do
|
986
|
-
File.
|
986
|
+
File.exist?(old_file)
|
987
987
|
end
|
988
988
|
|
989
989
|
context 'local false' do
|
@@ -991,7 +991,7 @@ shared_examples_for 'Filesystem' do
|
|
991
991
|
instance.remove_file(old_file)
|
992
992
|
end
|
993
993
|
|
994
|
-
it { is_expected.to eq(false)}
|
994
|
+
it { is_expected.to eq(false) }
|
995
995
|
end
|
996
996
|
|
997
997
|
context 'hdfs file' do
|
@@ -1011,7 +1011,7 @@ shared_examples_for 'Filesystem' do
|
|
1011
1011
|
|
1012
1012
|
describe '#remove_dir' do
|
1013
1013
|
subject do
|
1014
|
-
File.
|
1014
|
+
File.exist?(old_dir)
|
1015
1015
|
end
|
1016
1016
|
|
1017
1017
|
context 'local dir' do
|
@@ -1020,7 +1020,7 @@ shared_examples_for 'Filesystem' do
|
|
1020
1020
|
instance.remove_dir(old_dir)
|
1021
1021
|
end
|
1022
1022
|
|
1023
|
-
it { is_expected.to eq(false)}
|
1023
|
+
it { is_expected.to eq(false) }
|
1024
1024
|
end
|
1025
1025
|
|
1026
1026
|
context 'local root dir' do
|
@@ -1028,7 +1028,7 @@ shared_examples_for 'Filesystem' do
|
|
1028
1028
|
expect(filesystem).to receive(:root_path?).once.and_return(true)
|
1029
1029
|
end
|
1030
1030
|
|
1031
|
-
it { expect { instance.remove_dir(old_dir) }.to raise_error
|
1031
|
+
it { expect { instance.remove_dir(old_dir) }.to raise_error(/root path/) }
|
1032
1032
|
end
|
1033
1033
|
|
1034
1034
|
context 'hdfs dir' do
|
@@ -1044,7 +1044,7 @@ shared_examples_for 'Filesystem' do
|
|
1044
1044
|
expect(filesystem).to receive(:root_path?).once.and_return(true)
|
1045
1045
|
end
|
1046
1046
|
|
1047
|
-
it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error
|
1047
|
+
it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error(/root path/) }
|
1048
1048
|
end
|
1049
1049
|
|
1050
1050
|
context 's3 dir' do
|
@@ -1060,17 +1060,17 @@ shared_examples_for 'Filesystem' do
|
|
1060
1060
|
expect(filesystem).to receive(:s3cmd).never
|
1061
1061
|
end
|
1062
1062
|
|
1063
|
-
it { expect { instance.remove_dir('s3://bucket/') }.to raise_error
|
1063
|
+
it { expect { instance.remove_dir('s3://bucket/') }.to raise_error(/root path/) }
|
1064
1064
|
end
|
1065
1065
|
end
|
1066
1066
|
|
1067
1067
|
describe '#move_file_to_file' do
|
1068
1068
|
subject(:removes_old_file) do
|
1069
|
-
!File.
|
1069
|
+
!File.exist?(old_file)
|
1070
1070
|
end
|
1071
1071
|
|
1072
1072
|
subject(:creates_new_file) do
|
1073
|
-
File.
|
1073
|
+
File.exist?(new_file)
|
1074
1074
|
end
|
1075
1075
|
|
1076
1076
|
context 'local file to local file' do
|
@@ -1095,7 +1095,7 @@ shared_examples_for 'Filesystem' do
|
|
1095
1095
|
context 'local file to hdfs file' do
|
1096
1096
|
it do
|
1097
1097
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + old_dir).once
|
1098
|
-
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' +
|
1098
|
+
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_file)
|
1099
1099
|
instance.move_file_to_file(old_file, 'file://' + new_file)
|
1100
1100
|
end
|
1101
1101
|
end
|
@@ -1154,11 +1154,11 @@ shared_examples_for 'Filesystem' do
|
|
1154
1154
|
end
|
1155
1155
|
|
1156
1156
|
subject(:removes_old_file) do
|
1157
|
-
!File.
|
1157
|
+
!File.exist?(old_file)
|
1158
1158
|
end
|
1159
1159
|
|
1160
1160
|
subject(:creates_new_file) do
|
1161
|
-
File.
|
1161
|
+
File.exist?(File.join(new_dir, File.basename(old_file)))
|
1162
1162
|
end
|
1163
1163
|
|
1164
1164
|
context 'local file to local dir' do
|
@@ -1183,7 +1183,7 @@ shared_examples_for 'Filesystem' do
|
|
1183
1183
|
context 'local file to hdfs dir' do
|
1184
1184
|
it do
|
1185
1185
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir).once
|
1186
|
-
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' +
|
1186
|
+
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_dir)
|
1187
1187
|
instance.move_file_to_dir(old_file, 'file://' + new_dir)
|
1188
1188
|
end
|
1189
1189
|
end
|
@@ -1238,11 +1238,11 @@ shared_examples_for 'Filesystem' do
|
|
1238
1238
|
|
1239
1239
|
describe '#move_dir' do
|
1240
1240
|
subject(:removes_old_dir) do
|
1241
|
-
!File.
|
1241
|
+
!File.exist?(old_dir)
|
1242
1242
|
end
|
1243
1243
|
|
1244
1244
|
subject(:creates_new_dir) do
|
1245
|
-
File.
|
1245
|
+
File.exist?(new_dir)
|
1246
1246
|
end
|
1247
1247
|
|
1248
1248
|
context 'local dir to local dir' do
|
@@ -1266,7 +1266,7 @@ shared_examples_for 'Filesystem' do
|
|
1266
1266
|
context 'local dir to hdfs dir' do
|
1267
1267
|
it do
|
1268
1268
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + File.dirname(new_dir)).once
|
1269
|
-
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' +
|
1269
|
+
expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_dir, 'file://' + new_dir)
|
1270
1270
|
instance.move_dir(old_dir, 'file://' + new_dir)
|
1271
1271
|
end
|
1272
1272
|
end
|