masamune 0.17.12 → 0.17.13

Sign up to get free protection for your applications and to get access to all the features.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +3 -3
  3. data/bin/masamune-aws-emr +1 -1
  4. data/bin/masamune-dump +1 -1
  5. data/bin/masamune-hive +1 -1
  6. data/bin/masamune-psql +1 -1
  7. data/bin/masamune-shell +1 -1
  8. data/lib/masamune.rb +1 -1
  9. data/lib/masamune/actions/aws_emr.rb +2 -2
  10. data/lib/masamune/actions/data_flow.rb +10 -10
  11. data/lib/masamune/actions/date_parse.rb +4 -6
  12. data/lib/masamune/actions/execute.rb +3 -3
  13. data/lib/masamune/actions/filesystem.rb +1 -1
  14. data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
  15. data/lib/masamune/actions/hive.rb +5 -5
  16. data/lib/masamune/actions/invoke_parallel.rb +5 -5
  17. data/lib/masamune/actions/postgres.rb +6 -6
  18. data/lib/masamune/actions/postgres_admin.rb +1 -1
  19. data/lib/masamune/actions/s3cmd.rb +3 -3
  20. data/lib/masamune/actions/transform.rb +4 -4
  21. data/lib/masamune/after_initialize_callbacks.rb +4 -4
  22. data/lib/masamune/cached_filesystem.rb +7 -11
  23. data/lib/masamune/commands/aws_emr.rb +12 -12
  24. data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
  25. data/lib/masamune/commands/hadoop_streaming.rb +16 -16
  26. data/lib/masamune/commands/hive.rb +31 -35
  27. data/lib/masamune/commands/postgres.rb +30 -32
  28. data/lib/masamune/commands/postgres_admin.rb +17 -16
  29. data/lib/masamune/commands/postgres_common.rb +1 -1
  30. data/lib/masamune/commands/retry_with_backoff.rb +19 -19
  31. data/lib/masamune/commands/s3cmd.rb +7 -7
  32. data/lib/masamune/commands/shell.rb +25 -38
  33. data/lib/masamune/configuration.rb +8 -9
  34. data/lib/masamune/data_plan/builder.rb +5 -5
  35. data/lib/masamune/data_plan/elem.rb +7 -7
  36. data/lib/masamune/data_plan/engine.rb +12 -15
  37. data/lib/masamune/data_plan/rule.rb +14 -12
  38. data/lib/masamune/data_plan/set.rb +13 -13
  39. data/lib/masamune/environment.rb +3 -3
  40. data/lib/masamune/filesystem.rb +29 -28
  41. data/lib/masamune/helpers/postgres.rb +8 -5
  42. data/lib/masamune/io.rb +2 -2
  43. data/lib/masamune/last_element.rb +3 -3
  44. data/lib/masamune/method_logger.rb +1 -1
  45. data/lib/masamune/schema/catalog.rb +9 -12
  46. data/lib/masamune/schema/column.rb +12 -10
  47. data/lib/masamune/schema/dimension.rb +5 -4
  48. data/lib/masamune/schema/fact.rb +2 -2
  49. data/lib/masamune/schema/map.rb +9 -13
  50. data/lib/masamune/schema/row.rb +3 -2
  51. data/lib/masamune/schema/store.rb +13 -14
  52. data/lib/masamune/schema/table.rb +24 -28
  53. data/lib/masamune/schema/table_reference.rb +7 -7
  54. data/lib/masamune/spec_helper.rb +1 -1
  55. data/lib/masamune/string_format.rb +1 -1
  56. data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
  57. data/lib/masamune/tasks/dump_thor.rb +5 -5
  58. data/lib/masamune/tasks/hive_thor.rb +13 -13
  59. data/lib/masamune/tasks/postgres_thor.rb +8 -8
  60. data/lib/masamune/tasks/shell_thor.rb +3 -3
  61. data/lib/masamune/template.rb +4 -4
  62. data/lib/masamune/thor.rb +26 -25
  63. data/lib/masamune/transform/common/denormalize_table.rb +6 -6
  64. data/lib/masamune/transform/define_table.rb +1 -1
  65. data/lib/masamune/transform/hive/define_table.rb +0 -2
  66. data/lib/masamune/transform/insert_reference_values.rb +1 -1
  67. data/lib/masamune/transform/operator.rb +3 -2
  68. data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
  69. data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
  70. data/lib/masamune/transform/postgres/define_table.rb +2 -2
  71. data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
  72. data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
  73. data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
  74. data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
  75. data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
  76. data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
  77. data/lib/masamune/version.rb +1 -1
  78. data/spec/masamune/actions/aws_emr_spec.rb +8 -8
  79. data/spec/masamune/actions/execute_spec.rb +6 -6
  80. data/spec/masamune/actions/hive_spec.rb +20 -15
  81. data/spec/masamune/actions/postgres_spec.rb +17 -10
  82. data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
  83. data/spec/masamune/commands/aws_emr_spec.rb +10 -10
  84. data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
  85. data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
  86. data/spec/masamune/commands/hive_spec.rb +11 -11
  87. data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
  88. data/spec/masamune/commands/postgres_spec.rb +15 -15
  89. data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
  90. data/spec/masamune/commands/s3cmd_spec.rb +3 -3
  91. data/spec/masamune/commands/shell_spec.rb +16 -16
  92. data/spec/masamune/data_plan/builder_spec.rb +22 -19
  93. data/spec/masamune/data_plan/elem_spec.rb +20 -20
  94. data/spec/masamune/data_plan/engine_spec.rb +41 -33
  95. data/spec/masamune/data_plan/rule_spec.rb +19 -19
  96. data/spec/masamune/data_plan/set_spec.rb +8 -6
  97. data/spec/masamune/environment_spec.rb +5 -5
  98. data/spec/masamune/filesystem_spec.rb +87 -87
  99. data/spec/masamune/helpers/postgres_spec.rb +1 -1
  100. data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
  101. data/spec/masamune/schema/catalog_spec.rb +16 -16
  102. data/spec/masamune/schema/column_spec.rb +19 -19
  103. data/spec/masamune/schema/dimension_spec.rb +2 -2
  104. data/spec/masamune/schema/fact_spec.rb +3 -3
  105. data/spec/masamune/schema/map_spec.rb +41 -41
  106. data/spec/masamune/schema/table_spec.rb +16 -16
  107. data/spec/masamune/string_format_spec.rb +7 -7
  108. data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
  109. data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
  110. data/spec/masamune/template_spec.rb +3 -2
  111. data/spec/masamune/thor_spec.rb +8 -8
  112. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
  113. data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
  114. data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
  115. data/spec/masamune/transform/define_table.table_spec.rb +6 -6
  116. data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
  117. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
  118. data/spec/masamune/transform/load_dimension_spec.rb +1 -1
  119. data/spec/masamune/transform/load_fact_spec.rb +1 -1
  120. data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
  121. data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
  122. data/spec/masamune/transform/stage_fact_spec.rb +4 -4
  123. data/spec/spec_helper.rb +1 -1
  124. data/spec/support/masamune/example_group.rb +2 -2
  125. data/spec/support/masamune/job_example_group.rb +3 -3
  126. data/spec/support/masamune/job_fixture.rb +4 -4
  127. data/spec/support/masamune/mock_command.rb +10 -10
  128. data/spec/support/masamune/mock_delegate.rb +2 -2
  129. data/spec/support/masamune/mock_filesystem.rb +3 -3
  130. data/spec/support/masamune/shared_example_group.rb +14 -16
  131. data/spec/support/masamune/step_example_group.rb +4 -4
  132. data/spec/support/masamune/step_fixture.rb +1 -1
  133. data/spec/support/masamune/task_example_group.rb +1 -1
  134. data/spec/support/masamune/thor_mute.rb +3 -2
  135. data/spec/support/rspec/example/action_example_group.rb +1 -1
  136. data/spec/support/rspec/example/task_example_group.rb +7 -4
  137. data/spec/support/rspec/example/transform_example_group.rb +1 -1
  138. data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
  139. metadata +16 -2
@@ -273,8 +273,10 @@ describe Masamune::DataPlan::Set do
273
273
  end
274
274
 
275
275
  describe '#with_grain' do
276
- let(:paths) { ['/table/y=2012/m=12/d=29', '/table/y=2012/m=12/d=30', '/table/y=2012/m=12/d=31',
277
- '/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=02/d=01', ] }
276
+ let(:paths) do
277
+ ['/table/y=2012/m=12/d=29', '/table/y=2012/m=12/d=30', '/table/y=2012/m=12/d=31',
278
+ '/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=02/d=01']
279
+ end
278
280
 
279
281
  let(:instance) { Masamune::DataPlan::Set.new(target_rule, paths) }
280
282
 
@@ -346,7 +348,7 @@ describe Masamune::DataPlan::Set do
346
348
  let(:enum) { ['/log/20130101.*.log', '/log/20130102.*.log'] }
347
349
  let(:elem) { '/log/20130101.random_1.log' }
348
350
 
349
- it { is_expected.to eq(false) }
351
+ it { is_expected.to eq(false) }
350
352
  end
351
353
  end
352
354
 
@@ -371,7 +373,7 @@ describe Masamune::DataPlan::Set do
371
373
  context 'when some incomplete' do
372
374
  before do
373
375
  (1..31).each do |day|
374
- fs.touch!('/log/201401%02d.random_1.log' % day)
376
+ fs.touch!(format('/log/201401%02d.random_1.log', day))
375
377
  end
376
378
  end
377
379
 
@@ -382,8 +384,8 @@ describe Masamune::DataPlan::Set do
382
384
  context 'when none incomplete' do
383
385
  before do
384
386
  (1..31).each do |day|
385
- fs.touch!('/log/201401%02d.random_1.log' % day)
386
- fs.touch!('/log/201402%02d.random_1.log' % day)
387
+ fs.touch!(format('/log/201401%02d.random_1.log', day))
388
+ fs.touch!(format('/log/201402%02d.random_1.log', day))
387
389
  end
388
390
  end
389
391
 
@@ -26,15 +26,15 @@ describe Masamune::Environment do
26
26
 
27
27
  describe '#with_exclusive_lock' do
28
28
  context 'when run_dir not defined' do
29
- it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error /filesystem path :run_dir not defined/ }
29
+ it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to raise_error(/filesystem path :run_dir not defined/) }
30
30
  end
31
31
 
32
32
  context 'when lock can be acquired' do
33
33
  before do
34
34
  instance.filesystem.add_path(:run_dir, run_dir)
35
35
  expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
36
- expect(instance.logger).to receive(:debug).with(%q{acquiring lock 'some_lock'})
37
- expect(instance.logger).to receive(:debug).with(%q{releasing lock 'some_lock'})
36
+ expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock'")
37
+ expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock'")
38
38
  end
39
39
  it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
40
40
  end
@@ -44,8 +44,8 @@ describe Masamune::Environment do
44
44
  instance.filesystem.add_path(:run_dir, run_dir)
45
45
  instance.configuration.lock = 'long_running'
46
46
  expect_any_instance_of(File).to receive(:flock).twice.and_return(0)
47
- expect(instance.logger).to receive(:debug).with(%q{acquiring lock 'some_lock:long_running'})
48
- expect(instance.logger).to receive(:debug).with(%q{releasing lock 'some_lock:long_running'})
47
+ expect(instance.logger).to receive(:debug).with("acquiring lock 'some_lock:long_running'")
48
+ expect(instance.logger).to receive(:debug).with("releasing lock 'some_lock:long_running'")
49
49
  end
50
50
  it { expect { |b| instance.with_exclusive_lock('some_lock', &b) }.to yield_control }
51
51
  end
@@ -129,22 +129,22 @@ shared_examples_for 'Filesystem' do
129
129
 
130
130
  context 'with nil' do
131
131
  let(:path) { nil }
132
- it { expect { |b| subject }.to raise_error ArgumentError }
132
+ it { expect { |_b| subject }.to raise_error ArgumentError }
133
133
  end
134
134
 
135
135
  context 'with blank' do
136
136
  let(:path) { ' ' }
137
- it { expect { |b| subject }.to raise_error ArgumentError }
137
+ it { expect { |_b| subject }.to raise_error ArgumentError }
138
138
  end
139
139
 
140
140
  context 'with empty' do
141
141
  let(:path) { '' }
142
- it { expect { |b| subject }.to raise_error ArgumentError }
142
+ it { expect { |_b| subject }.to raise_error ArgumentError }
143
143
  end
144
144
 
145
145
  context 'with relative path' do
146
146
  let(:path) { 'tmp' }
147
- it { expect { |b| subject }.to raise_error ArgumentError }
147
+ it { expect { |_b| subject }.to raise_error ArgumentError }
148
148
  end
149
149
 
150
150
  context 'with local root' do
@@ -337,7 +337,7 @@ shared_examples_for 'Filesystem' do
337
337
 
338
338
  describe '#touch!' do
339
339
  subject do
340
- File.exists?(new_file) && File.exists?(other_new_file)
340
+ File.exist?(new_file) && File.exist?(other_new_file)
341
341
  end
342
342
 
343
343
  context 'local' do
@@ -372,8 +372,8 @@ shared_examples_for 'Filesystem' do
372
372
 
373
373
  context 'hdfs missing file' do
374
374
  before do
375
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
376
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
375
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
376
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
377
377
  expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + new_file, safe: true).at_most(:once).and_return(mock_failure)
378
378
  end
379
379
  subject { instance.exists?('file://' + new_file) }
@@ -395,9 +395,9 @@ shared_examples_for 'Filesystem' do
395
395
 
396
396
  context 'hdfs existing file' do
397
397
  before do
398
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
399
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}").
400
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
398
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
399
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
400
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
401
401
  expect(filesystem).to receive(:hadoop_fs).with('-test', '-e', 'file://' + old_file, safe: true).at_most(:once).and_return(mock_success)
402
402
  end
403
403
  subject { instance.exists?('file://' + old_file) }
@@ -406,12 +406,12 @@ shared_examples_for 'Filesystem' do
406
406
 
407
407
  context 's3 existing file' do
408
408
  before do
409
- expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once).
410
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
411
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
412
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once).
413
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
414
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
409
+ expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/00', safe: true).at_most(:once)
410
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
411
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
412
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/00', safe: true).at_most(:once)
413
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
414
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
415
415
  end
416
416
 
417
417
  subject { instance.exists?('s3://bucket/00') }
@@ -421,12 +421,12 @@ shared_examples_for 'Filesystem' do
421
421
 
422
422
  context 's3 missing file' do
423
423
  before do
424
- expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once).
425
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
426
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
427
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once).
428
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/00)).
429
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/01))
424
+ expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/0', safe: true).at_most(:once)
425
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
426
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
427
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/0', safe: true).at_most(:once)
428
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/00')
429
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/01')
430
430
  end
431
431
 
432
432
  subject { instance.exists?('s3://bucket/0') }
@@ -444,14 +444,14 @@ shared_examples_for 'Filesystem' do
444
444
 
445
445
  context 'hdfs missing file' do
446
446
  before do
447
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
448
- and_yield('')
449
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once).
450
- and_yield('')
451
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once).
452
- and_yield('')
453
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once).
454
- and_yield('')
447
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
448
+ .and_yield('')
449
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
450
+ .and_yield('')
451
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file + '/*', safe: true).at_most(:once)
452
+ .and_yield('')
453
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_file, safe: true).at_most(:once)
454
+ .and_yield('')
455
455
  end
456
456
  let(:result) { instance.stat('file://' + new_file) }
457
457
  it { is_expected.to be_nil }
@@ -460,8 +460,8 @@ shared_examples_for 'Filesystem' do
460
460
  context 's3 missing file' do
461
461
  before do
462
462
  expect(filesystem).to receive(:s3cmd).with('ls', 's3://bucket/', safe: true).at_most(:once)
463
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true).
464
- and_yield('')
463
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
464
+ .and_yield('')
465
465
  end
466
466
  let(:result) { instance.stat('s3://bucket/file.txt') }
467
467
  it { is_expected.to be_nil }
@@ -494,16 +494,16 @@ shared_examples_for 'Filesystem' do
494
494
 
495
495
  context 'local existing file (recursive)' do
496
496
  let(:result) { instance.stat(File.join(tmp_dir, '*')) }
497
- it { expect { result }.to raise_error /cannot contain wildcard/ }
497
+ it { expect { result }.to raise_error(/cannot contain wildcard/) }
498
498
  end
499
499
 
500
500
  context 'hdfs existing file' do
501
501
  before do
502
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once).
503
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}").
504
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
505
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once).
506
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
502
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.join(old_dir, '/*'), safe: true).at_most(:once)
503
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_dir}")
504
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
505
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_file, safe: true).at_most(:once)
506
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
507
507
  end
508
508
 
509
509
  let(:result) { instance.stat('file://' + old_file) }
@@ -527,8 +527,8 @@ shared_examples_for 'Filesystem' do
527
527
 
528
528
  context 's3 existing file' do
529
529
  before do
530
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true).
531
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/file.txt))
530
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|file.txt]}, safe: true)
531
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/file.txt')
532
532
  end
533
533
  let(:result) { instance.stat('s3://bucket/file.txt') }
534
534
 
@@ -551,8 +551,8 @@ shared_examples_for 'Filesystem' do
551
551
 
552
552
  context 's3 existing directory' do
553
553
  before do
554
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true).
555
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/file.txt))
554
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir]}, safe: true)
555
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
556
556
  end
557
557
 
558
558
  let(:result) { instance.stat('s3://bucket/dir') }
@@ -577,7 +577,7 @@ shared_examples_for 'Filesystem' do
577
577
 
578
578
  describe '#mkdir!' do
579
579
  subject do
580
- Dir.exists?(new_dir) && Dir.exists?(other_new_dir)
580
+ Dir.exist?(new_dir) && Dir.exist?(other_new_dir)
581
581
  end
582
582
 
583
583
  context 'local directory' do
@@ -588,7 +588,7 @@ shared_examples_for 'Filesystem' do
588
588
  end
589
589
 
590
590
  context 'local existing directory' do
591
- subject { Dir.exists?(old_dir) }
591
+ subject { Dir.exist?(old_dir) }
592
592
  before do
593
593
  expect(FileUtils).to receive(:mkdir_p).never
594
594
  instance.mkdir!(old_dir)
@@ -660,12 +660,12 @@ shared_examples_for 'Filesystem' do
660
660
 
661
661
  context 'hdfs no matches' do
662
662
  before do
663
- expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once).
664
- and_yield('')
665
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once).
666
- and_yield('')
667
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once).
668
- and_yield('')
663
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + new_dir + '/*', safe: true).at_most(:once)
664
+ .and_yield('')
665
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + new_dir + '/*', safe: true).at_most(:once)
666
+ .and_yield('')
667
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(new_dir) + '/*', safe: true).at_most(:once)
668
+ .and_yield('')
669
669
  end
670
670
  let(:pattern) { File.join(new_dir, '*') }
671
671
  it { expect(subject.count).to eq(0) }
@@ -674,12 +674,12 @@ shared_examples_for 'Filesystem' do
674
674
 
675
675
  context 'hdfs one matches' do
676
676
  before do
677
- expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once).
678
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
679
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once).
680
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
681
- expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
682
- and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
677
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', 'file://' + old_dir + '/*', safe: true).at_most(:once)
678
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
679
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + old_dir + '/*', safe: true).at_most(:once)
680
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
681
+ expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once)
682
+ .and_yield("drwxrwxrwt - root wheel 68 2015-02-24 12:09 #{old_file}")
683
683
  end
684
684
  let(:pattern) { File.join(File.dirname(old_file), '*') }
685
685
  it { expect(subject.count).to eq(1) }
@@ -690,8 +690,8 @@ shared_examples_for 'Filesystem' do
690
690
  let(:pattern) { 's3://bucket/dir/*.txt' }
691
691
 
692
692
  before do
693
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', "s3://bucket/dir", safe: true).at_most(:once)
694
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', "s3://bucket/dir/*", safe: true).at_most(:once)
693
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir', safe: true).at_most(:once)
694
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', 's3://bucket/dir/*', safe: true).at_most(:once)
695
695
  end
696
696
 
697
697
  it { expect(subject.count).to eq(0) }
@@ -701,9 +701,9 @@ shared_examples_for 'Filesystem' do
701
701
  let(:pattern) { 's3://bucket/dir/0' }
702
702
 
703
703
  before do
704
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true).
705
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/01.txt)).
706
- and_yield(%q(2013-05-24 18:53 2912 s3://bucket/dir/02.txt))
704
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
705
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/01.txt')
706
+ .and_yield('2013-05-24 18:53 2912 s3://bucket/dir/02.txt')
707
707
  end
708
708
 
709
709
  it { expect(subject.count).to eq(0) }
@@ -713,9 +713,9 @@ shared_examples_for 'Filesystem' do
713
713
  let(:pattern) { 's3://bucket/dir/*.txt' }
714
714
 
715
715
  before do
716
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true).
717
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/file.txt)).
718
- and_yield(%q(2013-05-24 18:53 2912 s3://bucket/dir/file.csv))
716
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
717
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
718
+ .and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
719
719
  end
720
720
 
721
721
  it { is_expected.to include 's3://bucket/dir/file.txt' }
@@ -726,10 +726,10 @@ shared_examples_for 'Filesystem' do
726
726
  let(:pattern) { 's3://bucket/dir/*' }
727
727
 
728
728
  before do
729
- expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true).
730
- and_yield(%q( DIR s3://bucket/dir/file_$folder$)).
731
- and_yield(%q(2013-05-24 18:52 2912 s3://bucket/dir/file.txt)).
732
- and_yield(%q(2013-05-24 18:53 2912 s3://bucket/dir/file.csv))
729
+ expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', %r{s3://bucket/[\*|dir/*]}, safe: true)
730
+ .and_yield(' DIR s3://bucket/dir/file_$folder$')
731
+ .and_yield('2013-05-24 18:52 2912 s3://bucket/dir/file.txt')
732
+ .and_yield('2013-05-24 18:53 2912 s3://bucket/dir/file.csv')
733
733
  end
734
734
 
735
735
  it { is_expected.to include 's3://bucket/dir/file.txt' }
@@ -754,7 +754,7 @@ shared_examples_for 'Filesystem' do
754
754
  let(:result_file) { File.join(new_dir, File.basename(old_file)) }
755
755
 
756
756
  subject do
757
- File.exists?(result_file)
757
+ File.exist?(result_file)
758
758
  end
759
759
 
760
760
  context 'local file to local file' do
@@ -829,7 +829,7 @@ shared_examples_for 'Filesystem' do
829
829
  let(:result_file) { File.join(new_dir, File.basename(old_file)) }
830
830
 
831
831
  subject do
832
- File.exists?(result_file)
832
+ File.exist?(result_file)
833
833
  end
834
834
 
835
835
  context 'local file to local dir' do
@@ -910,7 +910,7 @@ shared_examples_for 'Filesystem' do
910
910
 
911
911
  describe '#copy_dir' do
912
912
  subject do
913
- File.exists?(File.join(new_dir, File.basename(old_dir), File.basename(old_file)))
913
+ File.exist?(File.join(new_dir, File.basename(old_dir), File.basename(old_file)))
914
914
  end
915
915
 
916
916
  context 'local dir to local dir' do
@@ -983,7 +983,7 @@ shared_examples_for 'Filesystem' do
983
983
 
984
984
  describe '#remove_file' do
985
985
  subject do
986
- File.exists?(old_file)
986
+ File.exist?(old_file)
987
987
  end
988
988
 
989
989
  context 'local false' do
@@ -991,7 +991,7 @@ shared_examples_for 'Filesystem' do
991
991
  instance.remove_file(old_file)
992
992
  end
993
993
 
994
- it { is_expected.to eq(false)}
994
+ it { is_expected.to eq(false) }
995
995
  end
996
996
 
997
997
  context 'hdfs file' do
@@ -1011,7 +1011,7 @@ shared_examples_for 'Filesystem' do
1011
1011
 
1012
1012
  describe '#remove_dir' do
1013
1013
  subject do
1014
- File.exists?(old_dir)
1014
+ File.exist?(old_dir)
1015
1015
  end
1016
1016
 
1017
1017
  context 'local dir' do
@@ -1020,7 +1020,7 @@ shared_examples_for 'Filesystem' do
1020
1020
  instance.remove_dir(old_dir)
1021
1021
  end
1022
1022
 
1023
- it { is_expected.to eq(false)}
1023
+ it { is_expected.to eq(false) }
1024
1024
  end
1025
1025
 
1026
1026
  context 'local root dir' do
@@ -1028,7 +1028,7 @@ shared_examples_for 'Filesystem' do
1028
1028
  expect(filesystem).to receive(:root_path?).once.and_return(true)
1029
1029
  end
1030
1030
 
1031
- it { expect { instance.remove_dir(old_dir) }.to raise_error /root path/ }
1031
+ it { expect { instance.remove_dir(old_dir) }.to raise_error(/root path/) }
1032
1032
  end
1033
1033
 
1034
1034
  context 'hdfs dir' do
@@ -1044,7 +1044,7 @@ shared_examples_for 'Filesystem' do
1044
1044
  expect(filesystem).to receive(:root_path?).once.and_return(true)
1045
1045
  end
1046
1046
 
1047
- it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error /root path/ }
1047
+ it { expect { instance.remove_dir('file://' + old_dir) }.to raise_error(/root path/) }
1048
1048
  end
1049
1049
 
1050
1050
  context 's3 dir' do
@@ -1060,17 +1060,17 @@ shared_examples_for 'Filesystem' do
1060
1060
  expect(filesystem).to receive(:s3cmd).never
1061
1061
  end
1062
1062
 
1063
- it { expect { instance.remove_dir('s3://bucket/') }.to raise_error /root path/ }
1063
+ it { expect { instance.remove_dir('s3://bucket/') }.to raise_error(/root path/) }
1064
1064
  end
1065
1065
  end
1066
1066
 
1067
1067
  describe '#move_file_to_file' do
1068
1068
  subject(:removes_old_file) do
1069
- !File.exists?(old_file)
1069
+ !File.exist?(old_file)
1070
1070
  end
1071
1071
 
1072
1072
  subject(:creates_new_file) do
1073
- File.exists?(new_file)
1073
+ File.exist?(new_file)
1074
1074
  end
1075
1075
 
1076
1076
  context 'local file to local file' do
@@ -1095,7 +1095,7 @@ shared_examples_for 'Filesystem' do
1095
1095
  context 'local file to hdfs file' do
1096
1096
  it do
1097
1097
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + old_dir).once
1098
- expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_file)
1098
+ expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_file)
1099
1099
  instance.move_file_to_file(old_file, 'file://' + new_file)
1100
1100
  end
1101
1101
  end
@@ -1154,11 +1154,11 @@ shared_examples_for 'Filesystem' do
1154
1154
  end
1155
1155
 
1156
1156
  subject(:removes_old_file) do
1157
- !File.exists?(old_file)
1157
+ !File.exist?(old_file)
1158
1158
  end
1159
1159
 
1160
1160
  subject(:creates_new_file) do
1161
- File.exists?(File.join(new_dir, File.basename(old_file)))
1161
+ File.exist?(File.join(new_dir, File.basename(old_file)))
1162
1162
  end
1163
1163
 
1164
1164
  context 'local file to local dir' do
@@ -1183,7 +1183,7 @@ shared_examples_for 'Filesystem' do
1183
1183
  context 'local file to hdfs dir' do
1184
1184
  it do
1185
1185
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir).once
1186
- expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_dir)
1186
+ expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_file, 'file://' + new_dir)
1187
1187
  instance.move_file_to_dir(old_file, 'file://' + new_dir)
1188
1188
  end
1189
1189
  end
@@ -1238,11 +1238,11 @@ shared_examples_for 'Filesystem' do
1238
1238
 
1239
1239
  describe '#move_dir' do
1240
1240
  subject(:removes_old_dir) do
1241
- !File.exists?(old_dir)
1241
+ !File.exist?(old_dir)
1242
1242
  end
1243
1243
 
1244
1244
  subject(:creates_new_dir) do
1245
- File.exists?(new_dir)
1245
+ File.exist?(new_dir)
1246
1246
  end
1247
1247
 
1248
1248
  context 'local dir to local dir' do
@@ -1266,7 +1266,7 @@ shared_examples_for 'Filesystem' do
1266
1266
  context 'local dir to hdfs dir' do
1267
1267
  it do
1268
1268
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + File.dirname(new_dir)).once
1269
- expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_dir, 'file://' + new_dir)
1269
+ expect(filesystem).to receive(:hadoop_fs).with('-moveFromLocal', 'file://' + old_dir, 'file://' + new_dir)
1270
1270
  instance.move_dir(old_dir, 'file://' + new_dir)
1271
1271
  end
1272
1272
  end