masamune 0.17.12 → 0.17.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +3 -3
  3. data/bin/masamune-aws-emr +1 -1
  4. data/bin/masamune-dump +1 -1
  5. data/bin/masamune-hive +1 -1
  6. data/bin/masamune-psql +1 -1
  7. data/bin/masamune-shell +1 -1
  8. data/lib/masamune.rb +1 -1
  9. data/lib/masamune/actions/aws_emr.rb +2 -2
  10. data/lib/masamune/actions/data_flow.rb +10 -10
  11. data/lib/masamune/actions/date_parse.rb +4 -6
  12. data/lib/masamune/actions/execute.rb +3 -3
  13. data/lib/masamune/actions/filesystem.rb +1 -1
  14. data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
  15. data/lib/masamune/actions/hive.rb +5 -5
  16. data/lib/masamune/actions/invoke_parallel.rb +5 -5
  17. data/lib/masamune/actions/postgres.rb +6 -6
  18. data/lib/masamune/actions/postgres_admin.rb +1 -1
  19. data/lib/masamune/actions/s3cmd.rb +3 -3
  20. data/lib/masamune/actions/transform.rb +4 -4
  21. data/lib/masamune/after_initialize_callbacks.rb +4 -4
  22. data/lib/masamune/cached_filesystem.rb +7 -11
  23. data/lib/masamune/commands/aws_emr.rb +12 -12
  24. data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
  25. data/lib/masamune/commands/hadoop_streaming.rb +16 -16
  26. data/lib/masamune/commands/hive.rb +31 -35
  27. data/lib/masamune/commands/postgres.rb +30 -32
  28. data/lib/masamune/commands/postgres_admin.rb +17 -16
  29. data/lib/masamune/commands/postgres_common.rb +1 -1
  30. data/lib/masamune/commands/retry_with_backoff.rb +19 -19
  31. data/lib/masamune/commands/s3cmd.rb +7 -7
  32. data/lib/masamune/commands/shell.rb +25 -38
  33. data/lib/masamune/configuration.rb +8 -9
  34. data/lib/masamune/data_plan/builder.rb +5 -5
  35. data/lib/masamune/data_plan/elem.rb +7 -7
  36. data/lib/masamune/data_plan/engine.rb +12 -15
  37. data/lib/masamune/data_plan/rule.rb +14 -12
  38. data/lib/masamune/data_plan/set.rb +13 -13
  39. data/lib/masamune/environment.rb +3 -3
  40. data/lib/masamune/filesystem.rb +29 -28
  41. data/lib/masamune/helpers/postgres.rb +8 -5
  42. data/lib/masamune/io.rb +2 -2
  43. data/lib/masamune/last_element.rb +3 -3
  44. data/lib/masamune/method_logger.rb +1 -1
  45. data/lib/masamune/schema/catalog.rb +9 -12
  46. data/lib/masamune/schema/column.rb +12 -10
  47. data/lib/masamune/schema/dimension.rb +5 -4
  48. data/lib/masamune/schema/fact.rb +2 -2
  49. data/lib/masamune/schema/map.rb +9 -13
  50. data/lib/masamune/schema/row.rb +3 -2
  51. data/lib/masamune/schema/store.rb +13 -14
  52. data/lib/masamune/schema/table.rb +24 -28
  53. data/lib/masamune/schema/table_reference.rb +7 -7
  54. data/lib/masamune/spec_helper.rb +1 -1
  55. data/lib/masamune/string_format.rb +1 -1
  56. data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
  57. data/lib/masamune/tasks/dump_thor.rb +5 -5
  58. data/lib/masamune/tasks/hive_thor.rb +13 -13
  59. data/lib/masamune/tasks/postgres_thor.rb +8 -8
  60. data/lib/masamune/tasks/shell_thor.rb +3 -3
  61. data/lib/masamune/template.rb +4 -4
  62. data/lib/masamune/thor.rb +26 -25
  63. data/lib/masamune/transform/common/denormalize_table.rb +6 -6
  64. data/lib/masamune/transform/define_table.rb +1 -1
  65. data/lib/masamune/transform/hive/define_table.rb +0 -2
  66. data/lib/masamune/transform/insert_reference_values.rb +1 -1
  67. data/lib/masamune/transform/operator.rb +3 -2
  68. data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
  69. data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
  70. data/lib/masamune/transform/postgres/define_table.rb +2 -2
  71. data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
  72. data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
  73. data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
  74. data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
  75. data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
  76. data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
  77. data/lib/masamune/version.rb +1 -1
  78. data/spec/masamune/actions/aws_emr_spec.rb +8 -8
  79. data/spec/masamune/actions/execute_spec.rb +6 -6
  80. data/spec/masamune/actions/hive_spec.rb +20 -15
  81. data/spec/masamune/actions/postgres_spec.rb +17 -10
  82. data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
  83. data/spec/masamune/commands/aws_emr_spec.rb +10 -10
  84. data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
  85. data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
  86. data/spec/masamune/commands/hive_spec.rb +11 -11
  87. data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
  88. data/spec/masamune/commands/postgres_spec.rb +15 -15
  89. data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
  90. data/spec/masamune/commands/s3cmd_spec.rb +3 -3
  91. data/spec/masamune/commands/shell_spec.rb +16 -16
  92. data/spec/masamune/data_plan/builder_spec.rb +22 -19
  93. data/spec/masamune/data_plan/elem_spec.rb +20 -20
  94. data/spec/masamune/data_plan/engine_spec.rb +41 -33
  95. data/spec/masamune/data_plan/rule_spec.rb +19 -19
  96. data/spec/masamune/data_plan/set_spec.rb +8 -6
  97. data/spec/masamune/environment_spec.rb +5 -5
  98. data/spec/masamune/filesystem_spec.rb +87 -87
  99. data/spec/masamune/helpers/postgres_spec.rb +1 -1
  100. data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
  101. data/spec/masamune/schema/catalog_spec.rb +16 -16
  102. data/spec/masamune/schema/column_spec.rb +19 -19
  103. data/spec/masamune/schema/dimension_spec.rb +2 -2
  104. data/spec/masamune/schema/fact_spec.rb +3 -3
  105. data/spec/masamune/schema/map_spec.rb +41 -41
  106. data/spec/masamune/schema/table_spec.rb +16 -16
  107. data/spec/masamune/string_format_spec.rb +7 -7
  108. data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
  109. data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
  110. data/spec/masamune/template_spec.rb +3 -2
  111. data/spec/masamune/thor_spec.rb +8 -8
  112. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
  113. data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
  114. data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
  115. data/spec/masamune/transform/define_table.table_spec.rb +6 -6
  116. data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
  117. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
  118. data/spec/masamune/transform/load_dimension_spec.rb +1 -1
  119. data/spec/masamune/transform/load_fact_spec.rb +1 -1
  120. data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
  121. data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
  122. data/spec/masamune/transform/stage_fact_spec.rb +4 -4
  123. data/spec/spec_helper.rb +1 -1
  124. data/spec/support/masamune/example_group.rb +2 -2
  125. data/spec/support/masamune/job_example_group.rb +3 -3
  126. data/spec/support/masamune/job_fixture.rb +4 -4
  127. data/spec/support/masamune/mock_command.rb +10 -10
  128. data/spec/support/masamune/mock_delegate.rb +2 -2
  129. data/spec/support/masamune/mock_filesystem.rb +3 -3
  130. data/spec/support/masamune/shared_example_group.rb +14 -16
  131. data/spec/support/masamune/step_example_group.rb +4 -4
  132. data/spec/support/masamune/step_fixture.rb +1 -1
  133. data/spec/support/masamune/task_example_group.rb +1 -1
  134. data/spec/support/masamune/thor_mute.rb +3 -2
  135. data/spec/support/rspec/example/action_example_group.rb +1 -1
  136. data/spec/support/rspec/example/task_example_group.rb +7 -4
  137. data/spec/support/rspec/example/transform_example_group.rb +1 -1
  138. data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
  139. metadata +16 -2
@@ -273,8 +273,10 @@ describe Masamune::DataPlan::Set do
273
273
  end
274
274
 
275
275
  describe '#with_grain' do
276
- let(:paths) { ['/table/y=2012/m=12/d=29', '/table/y=2012/m=12/d=30', '/table/y=2012/m=12/d=31',
277
- '/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=02/d=01', ] }
276
+ let(:paths) do
277
+ ['/table/y=2012/m=12/d=29', '/table/y=2012/m=12/d=30', '/table/y=2012/m=12/d=31',
278
+ '/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=02/d=01']
279
+ end
278
280
 
279
281
  let(:instance) { Masamune::DataPlan::Set.new(target_rule, paths) }
280
282
 
@@ -346,7 +348,7 @@ describe Masamune::DataPlan::Set do
346
348
  let(:enum) { ['/log/20130101.*.log', '/log/20130102.*.log'] }
347
349
  let(:elem) { '/log/20130101.random_1.log' }
348
350
 
349
- it { is_expected.to eq(false) }
351
+ it { is_expected.to eq(false) }
350
352
  end
351
353
  end
352
354
 
@@ -371,7 +373,7 @@ describe Masamune::DataPlan::Set do
371
373
  context 'when some incomplete' do
372
374
  before do
373
375
  (1..31).each do |day|
374
- fs.touch!('/log/201401%02d.random_1.log' % day)
376
+ fs.touch!(format('/log/201401%02d.random_1.log', day))
375
377
  end
376
378
  end
377
379
 
@@ -382,8 +384,8 @@ describe Masamune::DataPlan::Set do
382
384
  context 'when none incomplete' do
383
385
  before do
384
386
  (1..31).each do |day|
385
- fs.touch!('/log/201401%02d.random_1.log' % day)
386
- fs.touch!('/log/201402%02d.random_1.log' % day)
387
+ fs.touch!(format('/log/201401%02d.random_1.log', day))
388
+ fs.touch!(format('/log/201402%02d.random_1.log', day))
387
389
  end
388
390
  end
389
391
 
@@ -26,15 +26,15 @@ describe Masamune::Environment do
26
26
 
27
27
  describe '#with_exclusive_lock' do
28
28
  context 'when run_dir not defined' do
29
- it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error /filesystem path :run_dir not defined/ }
29
+ it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error(/filesystem path :run_dir not defined/) }
30
30
  end
31
31
 
32
32
  context 'when lock can be acquired' do
33
33
  before do
34
34
  instance.filesystem.add_path(:run_dir, run_dir)
35
35
  expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
36
- expect(instance.logger).to receive(:debug).with(%q{acquiring lock 'some_lock'})
37
- expect(instance.logger).to receive(:debug).with(%q{releasing lock 'some_lock'})
36
+ expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock'")
37
+ expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock'")
38
38
  end
39
39
  it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
40
40
  end
@@ -44,8 +44,8 @@ describe Masamune::Environment do
44
44
  instance.filesystem.add_path(:run_dir, run_dir)
45
45
  instance.configuration.lock = 'long_running'
46
46
  expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
47
- expect(instance.logger).to receive(:debug).with(%q{acquiring lock 'some_lock:long_running'})
48
- expect(instance.logger).to receive(:debug).with(%q{releasing lock 'some_lock:long_running'})
47
+ expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock:long_running'")
48
+ expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock:long_running'")
49
49
  end
50
50
  it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
51
51
  end
@@ -129,22 +129,22 @@ shared_examples_for 'Filesystem' do
129
129
 
130
130
  context 'with nil' do
131
131
  let(:path) { nil }
132
- it { expect { |b| subject }.to raise_error ArgumentError }
132
+ it { expect { |_b| subject }.to raise_error ArgumentError }
133
133
  end
134
134
 
135
135
  context 'with blank' do
136
136
  let(:path) { ' ' }
137
- it { expect { |b| subject }.to raise_error ArgumentError }
137
+ it { expect { |_b| subject }.to raise_error ArgumentError }
138
138
  end
139
139
 
140
140
  context 'with empty' do
141
141
  let(:path) { '' }
142
- it { expect { |b| subject }.to raise_error ArgumentError }
142
+ it { expect { |_b| subject }.to raise_error ArgumentError }
143
143
  end
144
144
 
145
145
  context 'with relative path' do
146
146
  let(:path) { 'tmp' }
147
- it { expect { |b| subject }.to raise_error ArgumentError }
147
+ it { expect { |_b| subject }.to raise_error ArgumentError }
148
148
  end
149
149
 
150
150
  context 'with local root' do
@@ -337,7 +337,7 @@ shared_examples_for 'Filesystem' do
337
337
 
338
338
  describe '#touch!' do
339
339
  subject do
340
- File.exists?(new_file) && File.exists?(other_new_file)
340
+ File.exist?(new_file) && File.exist?(other_new_file)
341
341
  end
342
342
 
343
343
  context 'local' do
@@ -372,8 +372,8 @@ shared_examples_for 'Filesystem' do
372
372
 
373
373
  context 'hdfs missing file' do
374
374
  before do
375
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
376
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
375
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
376
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
377
377
  expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + new_file, safe: true).at_most(:once).and_return(mock_failure)
378
378
  end
379
379
  subject { instance.exists?('file://' + new_file) }
@@ -395,9 +395,9 @@ shared_examples_for 'Filesystem' do
395
395
 
396
396
  context 'hdfs existing file' do
397
397
  before do
398
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
399
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}").
400
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
398
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
399
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
400
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
401
401
  expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + old_file, safe: true).at_most(:once).and_return(mock_success)
402
402
  end
403
403
  subject { instance.exists?('file://' + old_file) }
@@ -406,12 +406,12 @@ shared_examples_for 'Filesystem' do
406
406
 
407
407
  context 's3 existing file' do
408
408
  before do
409
- expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once).
410
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
411
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
412
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once).
413
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
414
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
409
+ expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once)
410
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
411
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
412
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once)
413
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
414
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
415
415
  end
416
416
 
417
417
  subject { instance.exists?('s3://bucket/00') }
@@ -421,12 +421,12 @@ shared_examples_for 'Filesystem' do
421
421
 
422
422
  context 's3 missing file' do
423
423
  before do
424
- expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once).
425
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
426
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
427
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once).
428
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
429
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
424
+ expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once)
425
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
426
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
427
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once)
428
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
429
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
430
430
  end
431
431
 
432
432
  subject { instance.exists?('s3://bucket/0') }
@@ -444,14 +444,14 @@ shared_examples_for 'Filesystem' do
444
444
 
445
445
  context 'hdfs missing file' do
446
446
  before do
447
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
448
- and_yield('')
449
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once).
450
- and_yield('')
451
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once).
452
- and_yield('')
453
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once).
454
- and_yield('')
447
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
448
+ .and_yield('')
449
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
450
+ .and_yield('')
451
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once)
452
+ .and_yield('')
453
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once)
454
+ .and_yield('')
455
455
  end
456
456
  let(:result) { instance.stat('file://' + new_file) }
457
457
  it { is_expected.to be_nil }
@@ -460,8 +460,8 @@ shared_examples_for 'Filesystem' do
460
460
  context 's3 missing file' do
461
461
  before do
462
462
  expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/', safe: true).at_most(:once)
463
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true).
464
- and_yield('')
463
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
464
+ .and_yield('')
465
465
  end
466
466
  let(:result) { instance.stat('s3://bucket/file.txt') }
467
467
  it { is_expected.to be_nil }
@@ -494,16 +494,16 @@ shared_examples_for 'Filesystem' do
494
494
 
495
495
  context 'local existing file (recursive)' do
496
496
  let(:result) { instance.stat(File.join(tmp_dir, '*')) }
497
- it { expect { result }.to raise_error /cannot contain wildcard/ }
497
+ it { expect { result }.to raise_error(/cannot contain wildcard/) }
498
498
  end
499
499
 
500
500
  context 'hdfs existing file' do
501
501
  before do
502
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
503
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}").
504
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
505
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once).
506
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
502
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
503
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
504
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
505
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once)
506
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
507
507
  end
508
508
 
509
509
  let(:result) { instance.stat('file://' + old_file) }
@@ -527,8 +527,8 @@ shared_examples_for 'Filesystem' do
527
527
 
528
528
  context 's3 existing file' do
529
529
  before do
530
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true).
531
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/file.txt))
530
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
531
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/file.txt')
532
532
  end
533
533
  let(:result) { instance.stat('s3://bucket/file.txt') }
534
534
 
@@ -551,8 +551,8 @@ shared_examples_for 'Filesystem' do
551
551
 
552
552
  context 's3 existing directory' do
553
553
  before do
554
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true).
555
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/file.txt))
554
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true)
555
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
556
556
  end
557
557
 
558
558
  let(:result) { instance.stat('s3://bucket/dir') }
@@ -577,7 +577,7 @@ shared_examples_for 'Filesystem' do
577
577
 
578
578
  describe '#mkdir!' do
579
579
  subject do
580
- Dir.exists?(new_dir) && Dir.exists?(other_new_dir)
580
+ Dir.exist?(new_dir) && Dir.exist?(other_new_dir)
581
581
  end
582
582
 
583
583
  context 'local directory' do
@@ -588,7 +588,7 @@ shared_examples_for 'Filesystem' do
588
588
  end
589
589
 
590
590
  context 'local existing directory' do
591
- subject { Dir.exists?(old_dir) }
591
+ subject { Dir.exist?(old_dir) }
592
592
  before do
593
593
  expect(FileUtils).to receive(:mkdir_p).never
594
594
  instance.mkdir!(old_dir)
@@ -660,12 +660,12 @@ shared_examples_for 'Filesystem' do
660
660
 
661
661
  context 'hdfs no matches' do
662
662
  before do
663
- expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once).
664
- and_yield('')
665
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once).
666
- and_yield('')
667
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once).
668
- and_yield('')
663
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once)
664
+ .and_yield('')
665
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once)
666
+ .and_yield('')
667
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once)
668
+ .and_yield('')
669
669
  end
670
670
  let(:pattern) { File.join(new_dir, '*') }
671
671
  it { expect(subject.count).to eq(0) }
@@ -674,12 +674,12 @@ shared_examples_for 'Filesystem' do
674
674
 
675
675
  context 'hdfs one matches' do
676
676
  before do
677
- expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once).
678
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
679
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once).
680
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
681
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
682
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
677
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once)
678
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
679
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
680
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
681
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once)
682
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
683
683
  end
684
684
  let(:pattern) { File.join(File.dirname(old_file), '*') }
685
685
  it { expect(subject.count).to eq(1) }
@@ -690,8 +690,8 @@ shared_examples_for 'Filesystem' do
690
690
  let(:pattern) { 's3://bucket/dir/*.txt' }
691
691
 
692
692
  before do
693
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', "s3://bucket/dir", safe: true).at_most(:once)
694
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', "s3://bucket/dir/*", safe: true).at_most(:once)
693
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir', safe: true).at_most(:once)
694
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir/*', safe: true).at_most(:once)
695
695
  end
696
696
 
697
697
  it { expect(subject.count).to eq(0) }
@@ -701,9 +701,9 @@ shared_examples_for 'Filesystem' do
701
701
  let(:pattern) { 's3://bucket/dir/0' }
702
702
 
703
703
  before do
704
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true).
705
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/01.txt)).
706
- and_yield(%q(2013-05-24 18:53 2912 s3://bucket/dir/02.txt))
704
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
705
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/01.txt')
706
+ .and_yield('2013-05-24 18:53 2912 s3://bucket/dir/02.txt')
707
707
  end
708
708
 
709
709
  it { expect(subject.count).to eq(0) }
@@ -713,9 +713,9 @@ shared_examples_for 'Filesystem' do
713
713
  let(:pattern) { 's3://bucket/dir/*.txt' }
714
714
 
715
715
  before do
716
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true).
717
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/file.txt)).
718
- and_yield(%q(2013-05-24 18:53 2912 s3://bucket/dir/file.csv))
716
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
717
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
718
+ .and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
719
719
  end
720
720
 
721
721
  it { is_expected.to include 's3://bucket/dir/file.txt' }
@@ -726,10 +726,10 @@ shared_examples_for 'Filesystem' do
726
726
  let(:pattern) { 's3://bucket/dir/*' }
727
727
 
728
728
  before do
729
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true).
730
- and_yield(%q( DIR s3://bucket/dir/file_$folder$)).
731
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/file.txt)).
732
- and_yield(%q(2013-05-24 18:53 2912 s3://bucket/dir/file.csv))
729
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
730
+ .and_yield(' DIR s3://bucket/dir/file_$folder$')
731
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
732
+ .and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
733
733
  end
734
734
 
735
735
  it { is_expected.to include 's3://bucket/dir/file.txt' }
@@ -754,7 +754,7 @@ shared_examples_for 'Filesystem' do
754
754
  let(:result_file) { File.join(new_dir, File.basename(old_file)) }
755
755
 
756
756
  subject do
757
- File.exists?(result_file)
757
+ File.exist?(result_file)
758
758
  end
759
759
 
760
760
  context 'local file to local file' do
@@ -829,7 +829,7 @@ shared_examples_for 'Filesystem' do
829
829
  let(:result_file) { File.join(new_dir, File.basename(old_file)) }
830
830
 
831
831
  subject do
832
- File.exists?(result_file)
832
+ File.exist?(result_file)
833
833
  end
834
834
 
835
835
  context 'local file to local dir' do
@@ -910,7 +910,7 @@ shared_examples_for 'Filesystem' do
910
910
 
911
911
  describe '#copy_dir' do
912
912
  subject do
913
- File.exists?(File.join(new_dir, File.basename(old_dir), File.basename(old_file)))
913
+ File.exist?(File.join(new_dir, File.basename(old_dir), File.basename(old_file)))
914
914
  end
915
915
 
916
916
  context 'local dir to local dir' do
@@ -983,7 +983,7 @@ shared_examples_for 'Filesystem' do
983
983
 
984
984
  describe '#remove_file' do
985
985
  subject do
986
- File.exists?(old_file)
986
+ File.exist?(old_file)
987
987
  end
988
988
 
989
989
  context 'local false' do
@@ -991,7 +991,7 @@ shared_examples_for 'Filesystem' do
991
991
  instance.remove_file(old_file)
992
992
  end
993
993
 
994
- it { is_expected.to eq(false)}
994
+ it { is_expected.to eq(false) }
995
995
  end
996
996
 
997
997
  context 'hdfs file' do
@@ -1011,7 +1011,7 @@ shared_examples_for 'Filesystem' do
1011
1011
 
1012
1012
  describe '#remove_dir' do
1013
1013
  subject do
1014
- File.exists?(old_dir)
1014
+ File.exist?(old_dir)
1015
1015
  end
1016
1016
 
1017
1017
  context 'local dir' do
@@ -1020,7 +1020,7 @@ shared_examples_for 'Filesystem' do
1020
1020
  instance.remove_dir(old_dir)
1021
1021
  end
1022
1022
 
1023
- it { is_expected.to eq(false)}
1023
+ it { is_expected.to eq(false) }
1024
1024
  end
1025
1025
 
1026
1026
  context 'local root dir' do
@@ -1028,7 +1028,7 @@ shared_examples_for 'Filesystem' do
1028
1028
  expect(filesystem).to receive(:root_path?).once.and_return(true)
1029
1029
  end
1030
1030
 
1031
- it { expect { instance.remove_dir(old_dir) }.to raise_error /root path/ }
1031
+ it { expect { instance.remove_dir(old_dir) }.to raise_error(/root path/) }
1032
1032
  end
1033
1033
 
1034
1034
  context 'hdfs dir' do
@@ -1044,7 +1044,7 @@ shared_examples_for 'Filesystem' do
1044
1044
  expect(filesystem).to receive(:root_path?).once.and_return(true)
1045
1045
  end
1046
1046
 
1047
- it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error /root path/ }
1047
+ it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error(/root path/) }
1048
1048
  end
1049
1049
 
1050
1050
  context 's3 dir' do
@@ -1060,17 +1060,17 @@ shared_examples_for 'Filesystem' do
1060
1060
  expect(filesystem).to receive(:s3cmd).never
1061
1061
  end
1062
1062
 
1063
- it { expect { instance.remove_dir('s3://bucket/') }.to raise_error /root path/ }
1063
+ it { expect { instance.remove_dir('s3://bucket/') }.to raise_error(/root path/) }
1064
1064
  end
1065
1065
  end
1066
1066
 
1067
1067
  describe '#move_file_to_file' do
1068
1068
  subject(:removes_old_file) do
1069
- !File.exists?(old_file)
1069
+ !File.exist?(old_file)
1070
1070
  end
1071
1071
 
1072
1072
  subject(:creates_new_file) do
1073
- File.exists?(new_file)
1073
+ File.exist?(new_file)
1074
1074
  end
1075
1075
 
1076
1076
  context 'local file to local file' do
@@ -1095,7 +1095,7 @@ shared_examples_for 'Filesystem' do
1095
1095
  context 'local file to hdfs file' do
1096
1096
  it do
1097
1097
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + old_dir).once
1098
- expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_file)
1098
+ expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_file)
1099
1099
  instance.move_file_to_file(old_file, 'file://' + new_file)
1100
1100
  end
1101
1101
  end
@@ -1154,11 +1154,11 @@ shared_examples_for 'Filesystem' do
1154
1154
  end
1155
1155
 
1156
1156
  subject(:removes_old_file) do
1157
- !File.exists?(old_file)
1157
+ !File.exist?(old_file)
1158
1158
  end
1159
1159
 
1160
1160
  subject(:creates_new_file) do
1161
- File.exists?(File.join(new_dir, File.basename(old_file)))
1161
+ File.exist?(File.join(new_dir, File.basename(old_file)))
1162
1162
  end
1163
1163
 
1164
1164
  context 'local file to local dir' do
@@ -1183,7 +1183,7 @@ shared_examples_for 'Filesystem' do
1183
1183
  context 'local file to hdfs dir' do
1184
1184
  it do
1185
1185
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir).once
1186
- expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_dir)
1186
+ expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_dir)
1187
1187
  instance.move_file_to_dir(old_file, 'file://' + new_dir)
1188
1188
  end
1189
1189
  end
@@ -1238,11 +1238,11 @@ shared_examples_for 'Filesystem' do
1238
1238
 
1239
1239
  describe '#move_dir' do
1240
1240
  subject(:removes_old_dir) do
1241
- !File.exists?(old_dir)
1241
+ !File.exist?(old_dir)
1242
1242
  end
1243
1243
 
1244
1244
  subject(:creates_new_dir) do
1245
- File.exists?(new_dir)
1245
+ File.exist?(new_dir)
1246
1246
  end
1247
1247
 
1248
1248
  context 'local dir to local dir' do
@@ -1266,7 +1266,7 @@ shared_examples_for 'Filesystem' do
1266
1266
  context 'local dir to hdfs dir' do
1267
1267
  it do
1268
1268
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + File.dirname(new_dir)).once
1269
- expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_dir, 'file://' + new_dir)
1269
+ expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_dir, 'file://' + new_dir)
1270
1270
  instance.move_dir(old_dir, 'file://' + new_dir)
1271
1271
  end
1272
1272
  end