masamune 0.17.12 → 0.17.13

Sign up to get free protection for your applications and to get access to all the features.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +3 -3
  3. data/bin/masamune-aws-emr +1 -1
  4. data/bin/masamune-dump +1 -1
  5. data/bin/masamune-hive +1 -1
  6. data/bin/masamune-psql +1 -1
  7. data/bin/masamune-shell +1 -1
  8. data/lib/masamune.rb +1 -1
  9. data/lib/masamune/actions/aws_emr.rb +2 -2
  10. data/lib/masamune/actions/data_flow.rb +10 -10
  11. data/lib/masamune/actions/date_parse.rb +4 -6
  12. data/lib/masamune/actions/execute.rb +3 -3
  13. data/lib/masamune/actions/filesystem.rb +1 -1
  14. data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
  15. data/lib/masamune/actions/hive.rb +5 -5
  16. data/lib/masamune/actions/invoke_parallel.rb +5 -5
  17. data/lib/masamune/actions/postgres.rb +6 -6
  18. data/lib/masamune/actions/postgres_admin.rb +1 -1
  19. data/lib/masamune/actions/s3cmd.rb +3 -3
  20. data/lib/masamune/actions/transform.rb +4 -4
  21. data/lib/masamune/after_initialize_callbacks.rb +4 -4
  22. data/lib/masamune/cached_filesystem.rb +7 -11
  23. data/lib/masamune/commands/aws_emr.rb +12 -12
  24. data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
  25. data/lib/masamune/commands/hadoop_streaming.rb +16 -16
  26. data/lib/masamune/commands/hive.rb +31 -35
  27. data/lib/masamune/commands/postgres.rb +30 -32
  28. data/lib/masamune/commands/postgres_admin.rb +17 -16
  29. data/lib/masamune/commands/postgres_common.rb +1 -1
  30. data/lib/masamune/commands/retry_with_backoff.rb +19 -19
  31. data/lib/masamune/commands/s3cmd.rb +7 -7
  32. data/lib/masamune/commands/shell.rb +25 -38
  33. data/lib/masamune/configuration.rb +8 -9
  34. data/lib/masamune/data_plan/builder.rb +5 -5
  35. data/lib/masamune/data_plan/elem.rb +7 -7
  36. data/lib/masamune/data_plan/engine.rb +12 -15
  37. data/lib/masamune/data_plan/rule.rb +14 -12
  38. data/lib/masamune/data_plan/set.rb +13 -13
  39. data/lib/masamune/environment.rb +3 -3
  40. data/lib/masamune/filesystem.rb +29 -28
  41. data/lib/masamune/helpers/postgres.rb +8 -5
  42. data/lib/masamune/io.rb +2 -2
  43. data/lib/masamune/last_element.rb +3 -3
  44. data/lib/masamune/method_logger.rb +1 -1
  45. data/lib/masamune/schema/catalog.rb +9 -12
  46. data/lib/masamune/schema/column.rb +12 -10
  47. data/lib/masamune/schema/dimension.rb +5 -4
  48. data/lib/masamune/schema/fact.rb +2 -2
  49. data/lib/masamune/schema/map.rb +9 -13
  50. data/lib/masamune/schema/row.rb +3 -2
  51. data/lib/masamune/schema/store.rb +13 -14
  52. data/lib/masamune/schema/table.rb +24 -28
  53. data/lib/masamune/schema/table_reference.rb +7 -7
  54. data/lib/masamune/spec_helper.rb +1 -1
  55. data/lib/masamune/string_format.rb +1 -1
  56. data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
  57. data/lib/masamune/tasks/dump_thor.rb +5 -5
  58. data/lib/masamune/tasks/hive_thor.rb +13 -13
  59. data/lib/masamune/tasks/postgres_thor.rb +8 -8
  60. data/lib/masamune/tasks/shell_thor.rb +3 -3
  61. data/lib/masamune/template.rb +4 -4
  62. data/lib/masamune/thor.rb +26 -25
  63. data/lib/masamune/transform/common/denormalize_table.rb +6 -6
  64. data/lib/masamune/transform/define_table.rb +1 -1
  65. data/lib/masamune/transform/hive/define_table.rb +0 -2
  66. data/lib/masamune/transform/insert_reference_values.rb +1 -1
  67. data/lib/masamune/transform/operator.rb +3 -2
  68. data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
  69. data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
  70. data/lib/masamune/transform/postgres/define_table.rb +2 -2
  71. data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
  72. data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
  73. data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
  74. data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
  75. data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
  76. data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
  77. data/lib/masamune/version.rb +1 -1
  78. data/spec/masamune/actions/aws_emr_spec.rb +8 -8
  79. data/spec/masamune/actions/execute_spec.rb +6 -6
  80. data/spec/masamune/actions/hive_spec.rb +20 -15
  81. data/spec/masamune/actions/postgres_spec.rb +17 -10
  82. data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
  83. data/spec/masamune/commands/aws_emr_spec.rb +10 -10
  84. data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
  85. data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
  86. data/spec/masamune/commands/hive_spec.rb +11 -11
  87. data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
  88. data/spec/masamune/commands/postgres_spec.rb +15 -15
  89. data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
  90. data/spec/masamune/commands/s3cmd_spec.rb +3 -3
  91. data/spec/masamune/commands/shell_spec.rb +16 -16
  92. data/spec/masamune/data_plan/builder_spec.rb +22 -19
  93. data/spec/masamune/data_plan/elem_spec.rb +20 -20
  94. data/spec/masamune/data_plan/engine_spec.rb +41 -33
  95. data/spec/masamune/data_plan/rule_spec.rb +19 -19
  96. data/spec/masamune/data_plan/set_spec.rb +8 -6
  97. data/spec/masamune/environment_spec.rb +5 -5
  98. data/spec/masamune/filesystem_spec.rb +87 -87
  99. data/spec/masamune/helpers/postgres_spec.rb +1 -1
  100. data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
  101. data/spec/masamune/schema/catalog_spec.rb +16 -16
  102. data/spec/masamune/schema/column_spec.rb +19 -19
  103. data/spec/masamune/schema/dimension_spec.rb +2 -2
  104. data/spec/masamune/schema/fact_spec.rb +3 -3
  105. data/spec/masamune/schema/map_spec.rb +41 -41
  106. data/spec/masamune/schema/table_spec.rb +16 -16
  107. data/spec/masamune/string_format_spec.rb +7 -7
  108. data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
  109. data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
  110. data/spec/masamune/template_spec.rb +3 -2
  111. data/spec/masamune/thor_spec.rb +8 -8
  112. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
  113. data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
  114. data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
  115. data/spec/masamune/transform/define_table.table_spec.rb +6 -6
  116. data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
  117. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
  118. data/spec/masamune/transform/load_dimension_spec.rb +1 -1
  119. data/spec/masamune/transform/load_fact_spec.rb +1 -1
  120. data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
  121. data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
  122. data/spec/masamune/transform/stage_fact_spec.rb +4 -4
  123. data/spec/spec_helper.rb +1 -1
  124. data/spec/support/masamune/example_group.rb +2 -2
  125. data/spec/support/masamune/job_example_group.rb +3 -3
  126. data/spec/support/masamune/job_fixture.rb +4 -4
  127. data/spec/support/masamune/mock_command.rb +10 -10
  128. data/spec/support/masamune/mock_delegate.rb +2 -2
  129. data/spec/support/masamune/mock_filesystem.rb +3 -3
  130. data/spec/support/masamune/shared_example_group.rb +14 -16
  131. data/spec/support/masamune/step_example_group.rb +4 -4
  132. data/spec/support/masamune/step_fixture.rb +1 -1
  133. data/spec/support/masamune/task_example_group.rb +1 -1
  134. data/spec/support/masamune/thor_mute.rb +3 -2
  135. data/spec/support/rspec/example/action_example_group.rb +1 -1
  136. data/spec/support/rspec/example/task_example_group.rb +7 -4
  137. data/spec/support/rspec/example/transform_example_group.rb +1 -1
  138. data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
  139. metadata +16 -2
@@ -27,27 +27,28 @@ describe Masamune::DataPlan::Builder do
27
27
  end
28
28
 
29
29
  context 'with multiple namespaces' do
30
- let(:namespaces) { ['a', 'b'] }
31
- let(:commands) { ['load', 'store'] }
30
+ let(:namespaces) { %w(a b) }
31
+ let(:commands) { %w(load store) }
32
32
  let(:sources) { [{ path: 'log/%Y%m%d.*.log' }, { path: 'table/y=%Y/m=%m/d=%d' }] }
33
33
  let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d' }, { path: 'daily/%Y-%m-%d' }] }
34
34
 
35
35
  before do
36
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', { path: 'table/y=%Y/m=%m/d=%d'})
37
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', { path: 'log/%Y%m%d.*.log' })
36
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', path: 'table/y=%Y/m=%m/d=%d')
37
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', path: 'log/%Y%m%d.*.log')
38
38
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:load', an_instance_of(Proc))
39
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', { path: 'daily/%Y-%m-%d' })
40
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', { path: 'table/y=%Y/m=%m/d=%d' })
39
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', path: 'daily/%Y-%m-%d')
40
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', path: 'table/y=%Y/m=%m/d=%d')
41
41
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('b:store', an_instance_of(Proc))
42
42
  subject
43
43
  end
44
44
 
45
- it 'should build a Masamune::DataPlan::Engine instance' do; end
45
+ it 'should build a Masamune::DataPlan::Engine instance' do
46
+ end
46
47
  end
47
48
 
48
49
  context 'with :for option' do
49
- let(:namespaces) { ['a', 'a', 'a'] }
50
- let(:commands) { ['missing_before', 'override', 'missing_after'] }
50
+ let(:namespaces) { %w(a a a) }
51
+ let(:commands) { %w(missing_before override missing_after) }
51
52
  let(:sources) { [{ path: 'log/%Y%m%d.*.log', for: 'override' }] }
52
53
  let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d', for: 'override' }] }
53
54
 
@@ -55,8 +56,8 @@ describe Masamune::DataPlan::Builder do
55
56
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
56
57
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
57
58
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
58
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', { path: 'table/y=%Y/m=%m/d=%d' })
59
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', { path: 'log/%Y%m%d.*.log' })
59
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
60
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
60
61
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
61
62
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_after', anything)
62
63
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_after', anything)
@@ -64,26 +65,28 @@ describe Masamune::DataPlan::Builder do
64
65
  subject
65
66
  end
66
67
 
67
- it 'should build a Masamune::DataPlan::Engine instance' do; end
68
+ it 'should build a Masamune::DataPlan::Engine instance' do
69
+ end
68
70
  end
69
71
 
70
72
  context 'with :skip option' do
71
- let(:namespaces) { ['a', 'a', 'a'] }
72
- let(:commands) { ['missing_before', 'override'] }
73
- let(:sources) { [{ skip: true}, { path: 'log/%Y%m%d.*.log' }] }
74
- let(:targets) { [{ skip: true}, { path: 'table/y=%Y/m=%m/d=%d' }] }
73
+ let(:namespaces) { %w(a a a) }
74
+ let(:commands) { %w(missing_before override) }
75
+ let(:sources) { [{ skip: true }, { path: 'log/%Y%m%d.*.log' }] }
76
+ let(:targets) { [{ skip: true }, { path: 'table/y=%Y/m=%m/d=%d' }] }
75
77
 
76
78
  before do
77
79
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
78
80
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
79
81
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
80
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', { path: 'table/y=%Y/m=%m/d=%d' })
81
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', { path: 'log/%Y%m%d.*.log' })
82
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
83
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
82
84
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
83
85
  subject
84
86
  end
85
87
 
86
- it 'should build a Masamune::DataPlan::Engine instance' do; end
88
+ it 'should build a Masamune::DataPlan::Engine instance' do
89
+ end
87
90
  end
88
91
  end
89
92
  end
@@ -32,14 +32,14 @@ describe Masamune::DataPlan::Elem do
32
32
 
33
33
  let(:name) { 'primary' }
34
34
  let(:type) { :target }
35
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
36
- let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/log/%Y%m%d.*.log'}) }
35
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
36
+ let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/log/%Y%m%d.*.log') }
37
37
 
38
- let(:start_time) { DateTime.civil(2013,07,19,11,07) }
39
- let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
38
+ let(:start_time) { DateTime.civil(2013, 07, 19, 11, 07) }
39
+ let(:other_start_time) { DateTime.civil(2013, 07, 20, 0, 0) }
40
40
 
41
- let(:options) { {tz: 'EST'} }
42
- let(:other_options) { {tz: 'PST'} }
41
+ let(:options) { { tz: 'EST' } }
42
+ let(:other_options) { { tz: 'PST' } }
43
43
 
44
44
  let(:instance) { described_class.new(rule, start_time, options) }
45
45
 
@@ -80,8 +80,8 @@ describe Masamune::DataPlan::Elem do
80
80
  end
81
81
 
82
82
  describe '#last_modified_at' do
83
- let(:early) { Time.parse("2014-05-01 00:00:00 +0000") }
84
- let(:later) { Time.parse("2014-06-01 00:00:00 +0000") }
83
+ let(:early) { Time.parse('2014-05-01 00:00:00 +0000') }
84
+ let(:later) { Time.parse('2014-06-01 00:00:00 +0000') }
85
85
 
86
86
  subject do
87
87
  instance.last_modified_at.utc
@@ -89,8 +89,8 @@ describe Masamune::DataPlan::Elem do
89
89
 
90
90
  context 'with missing mtime' do
91
91
  before do
92
- expect(rule.engine.filesystem).to receive(:stat).with(instance.path).
93
- and_return(nil)
92
+ expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
93
+ .and_return(nil)
94
94
  end
95
95
 
96
96
  it { is_expected.to eq(Masamune::DataPlan::Elem::MISSING_MODIFIED_AT) }
@@ -98,8 +98,8 @@ describe Masamune::DataPlan::Elem do
98
98
 
99
99
  context 'with single mtime' do
100
100
  before do
101
- expect(rule.engine.filesystem).to receive(:stat).with(instance.path).
102
- and_return(OpenStruct.new(mtime: early))
101
+ expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
102
+ .and_return(OpenStruct.new(mtime: early))
103
103
  end
104
104
 
105
105
  it { is_expected.to eq(early) }
@@ -110,7 +110,7 @@ describe Masamune::DataPlan::Elem do
110
110
  subject { instance.explode.map(&:path) }
111
111
 
112
112
  context 'with free path and existing files' do
113
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
113
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
114
114
  before do
115
115
  engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
116
116
  end
@@ -118,12 +118,12 @@ describe Masamune::DataPlan::Elem do
118
118
  end
119
119
 
120
120
  context 'with free path and missing files' do
121
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
121
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
122
122
  it { is_expected.to be_empty }
123
123
  end
124
124
 
125
125
  context 'with bound path and existing files' do
126
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
126
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
127
127
  before do
128
128
  engine.filesystem.touch!('/report/file')
129
129
  end
@@ -131,12 +131,12 @@ describe Masamune::DataPlan::Elem do
131
131
  end
132
132
 
133
133
  context 'with bound path and missing files' do
134
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
134
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
135
135
  it { is_expected.to be_empty }
136
136
  end
137
137
 
138
138
  context 'with free table and existing table' do
139
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
139
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
140
140
  before do
141
141
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
142
142
  end
@@ -144,7 +144,7 @@ describe Masamune::DataPlan::Elem do
144
144
  end
145
145
 
146
146
  context 'with free table and missing table' do
147
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
147
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
148
148
  before do
149
149
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
150
150
  end
@@ -152,7 +152,7 @@ describe Masamune::DataPlan::Elem do
152
152
  end
153
153
 
154
154
  context 'with bound table and existing table' do
155
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
155
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
156
156
  before do
157
157
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
158
158
  end
@@ -160,7 +160,7 @@ describe Masamune::DataPlan::Elem do
160
160
  end
161
161
 
162
162
  context 'with bound table and missing table' do
163
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
163
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
164
164
  before do
165
165
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
166
166
  end
@@ -31,7 +31,7 @@ describe Masamune::DataPlan::Engine do
31
31
  end
32
32
 
33
33
  let(:command) do
34
- Proc.new do |engine, rule|
34
+ proc do |engine, rule|
35
35
  missing_targets = []
36
36
  engine.targets(rule).missing.each do |target|
37
37
  missing_targets << target.path if target.sources.existing.any?
@@ -43,7 +43,7 @@ describe Masamune::DataPlan::Engine do
43
43
  before do
44
44
  engine.add_target_rule('non_primary', path: '/table/y=%Y/m=%m/d=%d', primary: false)
45
45
  engine.add_source_rule('non_primary', path: '/log/%Y%m%d.*.log', primary: false)
46
- engine.add_command_rule('non_primary', ->(*_) { fail } )
46
+ engine.add_command_rule('non_primary', ->(*_) { raise })
47
47
  engine.add_target_rule('primary', path: '/table/y=%Y/m=%m/d=%d')
48
48
  engine.add_source_rule('primary', path: '/log/%Y%m%d.*.log')
49
49
  engine.add_command_rule('primary', command)
@@ -61,8 +61,8 @@ describe Masamune::DataPlan::Engine do
61
61
  end
62
62
 
63
63
  describe '#targets_for_date_range' do
64
- let(:start) { Date.civil(2013,01,01) }
65
- let(:stop) { Date.civil(2013,01,03) }
64
+ let(:start) { Date.civil(2013, 01, 01) }
65
+ let(:stop) { Date.civil(2013, 01, 03) }
66
66
 
67
67
  subject { engine.targets_for_date_range(rule, start, stop).map(&:path) }
68
68
 
@@ -104,26 +104,26 @@ describe Masamune::DataPlan::Engine do
104
104
  let(:rule) { 'primary' }
105
105
  let(:source) { '/log/20130101.random.log' }
106
106
 
107
- it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
108
- it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
107
+ it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
108
+ it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
109
109
  it { expect(targets.first.path).to eq('/table/y=2013/m=01/d=01') }
110
110
  end
111
111
 
112
112
  context 'derived_daily' do
113
113
  let(:rule) { 'derived_daily' }
114
- let(:source) { '/table/y=2013/m=01/d=01' }
114
+ let(:source) { '/table/y=2013/m=01/d=01' }
115
115
 
116
- it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
117
- it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
116
+ it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
117
+ it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
118
118
  it { expect(targets.first.path).to eq('/daily/2013-01-01') }
119
119
  end
120
120
 
121
121
  context 'derived_monthly' do
122
122
  let(:rule) { 'derived_monthly' }
123
- let(:source) { '/table/y=2013/m=01/d=01' }
123
+ let(:source) { '/table/y=2013/m=01/d=01' }
124
124
 
125
- it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
126
- it { expect(targets.first.stop_time).to eq(Date.civil(2013,02,01)) }
125
+ it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
126
+ it { expect(targets.first.stop_time).to eq(Date.civil(2013, 02, 01)) }
127
127
  it { expect(targets.first.path).to eq('/monthly/2013-01') }
128
128
  end
129
129
  end
@@ -167,15 +167,15 @@ describe Masamune::DataPlan::Engine do
167
167
  let(:target) { '/monthly/2013-01' }
168
168
 
169
169
  (1..31).each do |day|
170
- it { expect(sources).to include '/table/y=2013/m=01/d=%02d' % day }
170
+ it { expect(sources).to include format('/table/y=2013/m=01/d=%02d', day) }
171
171
  end
172
172
  it { expect(sources.size).to eq(31) }
173
173
  end
174
174
 
175
175
  context 'invalid target' do
176
176
  let(:rule) { 'derived_daily' }
177
- let(:target) { '/table/y=2013/m=01/d=01' }
178
- it { expect { subject }.to raise_error /Cannot bind_input/ }
177
+ let(:target) { '/table/y=2013/m=01/d=01' }
178
+ it { expect { subject }.to raise_error(/Cannot bind_input/) }
179
179
  end
180
180
  end
181
181
 
@@ -204,7 +204,7 @@ describe Masamune::DataPlan::Engine do
204
204
 
205
205
  context 'invalid target' do
206
206
  let(:target) { '/daily' }
207
- it { expect { subject }.to raise_error /No rule matches/ }
207
+ it { expect { subject }.to raise_error(/No rule matches/) }
208
208
  end
209
209
  end
210
210
 
@@ -224,7 +224,7 @@ describe Masamune::DataPlan::Engine do
224
224
  context 'with :targets' do
225
225
  let(:rule) { 'primary' }
226
226
 
227
- let(:options) { {targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
227
+ let(:options) { { targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
228
228
 
229
229
  it { expect(targets).to include '/table/y=2013/m=01/d=01' }
230
230
  it { expect(targets).to include '/table/y=2013/m=01/d=02' }
@@ -235,10 +235,10 @@ describe Masamune::DataPlan::Engine do
235
235
  context 'with :sources' do
236
236
  let(:rule) { 'derived_daily' }
237
237
 
238
- let(:options) { {sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
238
+ let(:options) { { sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
239
239
 
240
- it { expect(targets).to include "/daily/2013-01-01" }
241
- it { expect(targets).to include "/daily/2013-01-02" }
240
+ it { expect(targets).to include '/daily/2013-01-01' }
241
+ it { expect(targets).to include '/daily/2013-01-02' }
242
242
  it { expect(sources).to include '/table/y=2013/m=01/d=01' }
243
243
  it { expect(sources).to include '/table/y=2013/m=01/d=02' }
244
244
  end
@@ -257,10 +257,13 @@ describe Masamune::DataPlan::Engine do
257
257
 
258
258
  context 'primary rule' do
259
259
  let(:rule) { 'primary' }
260
- let(:targets) { [
261
- '/table/y=2013/m=01/d=01',
262
- '/table/y=2013/m=01/d=02',
263
- '/table/y=2013/m=01/d=03' ] }
260
+ let(:targets) do
261
+ [
262
+ '/table/y=2013/m=01/d=01',
263
+ '/table/y=2013/m=01/d=02',
264
+ '/table/y=2013/m=01/d=03'
265
+ ]
266
+ end
264
267
 
265
268
  context 'when target data exists' do
266
269
  before do
@@ -269,7 +272,8 @@ describe Masamune::DataPlan::Engine do
269
272
  execute
270
273
  end
271
274
 
272
- it 'should not call touch!' do; end
275
+ it 'should not call touch!' do
276
+ end
273
277
  end
274
278
 
275
279
  context 'when partial target data exists' do
@@ -280,7 +284,8 @@ describe Masamune::DataPlan::Engine do
280
284
  execute
281
285
  end
282
286
 
283
- it 'should call touch!' do; end
287
+ it 'should call touch!' do
288
+ end
284
289
  end
285
290
 
286
291
  context 'when source data does not exist' do
@@ -289,13 +294,14 @@ describe Masamune::DataPlan::Engine do
289
294
  execute
290
295
  end
291
296
 
292
- it 'should not call touch!' do; end
297
+ it 'should not call touch!' do
298
+ end
293
299
  end
294
300
  end
295
301
 
296
302
  shared_examples_for 'derived daily data' do
297
303
  context 'when primary target data exists' do
298
- let(:derived_targets) { ['/table/y=2013/m=01/d=01/DATA', '/table/y=2013/m=01/d=02/DATA', '/table/y=2013/m=01/d=03/DATA'] }
304
+ let(:derived_targets) { ['/table/y=2013/m=01/d=01/DATA', '/table/y=2013/m=01/d=02/DATA', '/table/y=2013/m=01/d=03/DATA'] }
299
305
 
300
306
  before do
301
307
  engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
@@ -304,11 +310,12 @@ describe Masamune::DataPlan::Engine do
304
310
  execute
305
311
  end
306
312
 
307
- it 'should call touch!' do; end
313
+ it 'should call touch!' do
314
+ end
308
315
  end
309
316
 
310
317
  context 'when primary target data exists and :resolve is false' do
311
- let(:options) { {resolve: false} }
318
+ let(:options) { { resolve: false } }
312
319
 
313
320
  before do
314
321
  engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
@@ -316,7 +323,8 @@ describe Masamune::DataPlan::Engine do
316
323
  execute
317
324
  end
318
325
 
319
- it 'should not call touch!' do; end
326
+ it 'should not call touch!' do
327
+ end
320
328
  end
321
329
  end
322
330
 
@@ -331,7 +339,7 @@ describe Masamune::DataPlan::Engine do
331
339
 
332
340
  context 'derived_monthly rule' do
333
341
  let(:rule) { 'derived_monthly' }
334
- let(:targets) { ['/monthly/2013-01/DATA'] }
342
+ let(:targets) { ['/monthly/2013-01/DATA'] }
335
343
 
336
344
  it_behaves_like 'derived daily data' do
337
345
  let(:derived_command) { derived_monthly_command }
@@ -348,7 +356,7 @@ describe Masamune::DataPlan::Engine do
348
356
  end
349
357
 
350
358
  it 'should raise exception' do
351
- expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error /Max depth .* exceeded for rule 'derived'/
359
+ expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error(/Max depth .* exceeded for rule 'derived'/)
352
360
  end
353
361
  end
354
362
  end
@@ -25,7 +25,7 @@ describe Masamune::DataPlan::Rule do
25
25
  let(:name) { 'primary' }
26
26
  let(:type) { :target }
27
27
  let(:pattern) { 'report/%Y-%m-%d/%H' }
28
- let(:options) { {path: pattern} }
28
+ let(:options) { { path: pattern } }
29
29
 
30
30
  let(:instance) { described_class.new(engine, name, type, options) }
31
31
 
@@ -40,7 +40,7 @@ describe Masamune::DataPlan::Rule do
40
40
  end
41
41
 
42
42
  context 'with lambda' do
43
- let(:pattern) { lambda { |_| 'report/%Y-%m-%d/%H' } }
43
+ let(:pattern) { ->(_) { 'report/%Y-%m-%d/%H' } }
44
44
  it { is_expected.to eq('report/%Y-%m-%d/%H') }
45
45
  end
46
46
  end
@@ -59,7 +59,7 @@ describe Masamune::DataPlan::Rule do
59
59
  end
60
60
 
61
61
  context 'with DateTime input' do
62
- let(:input) { DateTime.civil(2013,04,05,23,13) }
62
+ let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
63
63
 
64
64
  describe '#path' do
65
65
  subject { elem.path }
@@ -69,7 +69,7 @@ describe Masamune::DataPlan::Rule do
69
69
 
70
70
  context 'with DateTime input and unix timestamp pattern' do
71
71
  let(:pattern) { 'logs/%H-s.log' }
72
- let(:input) { DateTime.civil(2013,04,05,23,13) }
72
+ let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
73
73
 
74
74
  describe '#path' do
75
75
  subject { elem.path }
@@ -78,7 +78,7 @@ describe Masamune::DataPlan::Rule do
78
78
  end
79
79
 
80
80
  context 'with Date input' do
81
- let(:input) { Date.civil(2013,04,05) }
81
+ let(:input) { Date.civil(2013, 04, 05) }
82
82
 
83
83
  describe '#path' do
84
84
  subject { elem.path }
@@ -92,7 +92,7 @@ describe Masamune::DataPlan::Rule do
92
92
 
93
93
  context 'with default' do
94
94
  let(:input) { 'report/2013-04-05/23' }
95
- let(:output_date) { DateTime.civil(2013,04,05,23) }
95
+ let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
96
96
 
97
97
  describe '#path' do
98
98
  subject { elem.path }
@@ -113,7 +113,7 @@ describe Masamune::DataPlan::Rule do
113
113
  context 'with unix timestamp pattern' do
114
114
  let(:pattern) { 'logs/%H-s.log' }
115
115
  let(:input) { 'logs/1365202800.log' }
116
- let(:output_date) { DateTime.civil(2013,04,05,23) }
116
+ let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
117
117
 
118
118
  describe '#path' do
119
119
  subject { elem.path }
@@ -140,7 +140,7 @@ describe Masamune::DataPlan::Rule do
140
140
  context 'with wildcard pattern' do
141
141
  let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
142
142
  let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
143
- let(:output_date) { DateTime.civil(2013,04,30,20) }
143
+ let(:output_date) { DateTime.civil(2013, 04, 30, 20) }
144
144
 
145
145
  describe '#path' do
146
146
  subject { elem.path }
@@ -160,8 +160,8 @@ describe Masamune::DataPlan::Rule do
160
160
  end
161
161
 
162
162
  describe '#unify' do
163
- let(:primary) { described_class.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
164
- let(:induced) { described_class.new(engine, name, type, {path: 'table/y=%Y/m=%m/d=%d/h=%H'}) }
163
+ let(:primary) { described_class.new(engine, name, type, path: 'report/%Y-%m-%d/%H') }
164
+ let(:induced) { described_class.new(engine, name, type, path: 'table/y=%Y/m=%m/d=%d/h=%H') }
165
165
  let(:elem) { primary.bind_input(input) }
166
166
 
167
167
  subject(:new_elem) { instance.unify(elem, induced) }
@@ -176,7 +176,7 @@ describe Masamune::DataPlan::Rule do
176
176
  end
177
177
 
178
178
  context 'when input partially matches basis pattern' do
179
- let(:induced) { described_class.new(engine, name, type, {path: 'table/%Y-%m'}) }
179
+ let(:induced) { described_class.new(engine, name, type, path: 'table/%Y-%m') }
180
180
 
181
181
  let(:input) { 'report/2013-01-02/00' }
182
182
 
@@ -240,14 +240,14 @@ describe Masamune::DataPlan::Rule do
240
240
 
241
241
  describe '#generate' do
242
242
  context 'with a block' do
243
- let(:start_date) { DateTime.civil(2013,04,05,20) }
244
- let(:stop_date) { DateTime.civil(2013,04,05,20) }
243
+ let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
244
+ let(:stop_date) { DateTime.civil(2013, 04, 05, 20) }
245
245
  specify { expect { |b| instance.generate(start_date, stop_date, &b) }.to yield_control }
246
246
  end
247
247
 
248
248
  context 'without a block' do
249
- let(:start_date) { DateTime.civil(2013,04,05,20) }
250
- let(:stop_date) { DateTime.civil(2013,04,05,22) }
249
+ let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
250
+ let(:stop_date) { DateTime.civil(2013, 04, 05, 22) }
251
251
 
252
252
  subject(:elems) do
253
253
  instance.generate(start_date, stop_date)
@@ -307,7 +307,7 @@ describe Masamune::DataPlan::Rule do
307
307
  end
308
308
 
309
309
  describe '#time_round' do
310
- let(:input_time) { DateTime.civil(2013,9,13,23,13) }
310
+ let(:input_time) { DateTime.civil(2013, 9, 13, 23, 13) }
311
311
  subject { instance.time_round(input_time) }
312
312
 
313
313
  before do
@@ -316,15 +316,15 @@ describe Masamune::DataPlan::Rule do
316
316
 
317
317
  context 'hourly' do
318
318
  let(:time_step) { :hours }
319
- it { is_expected.to eq(DateTime.civil(2013,9,13,23)) }
319
+ it { is_expected.to eq(DateTime.civil(2013, 9, 13, 23)) }
320
320
  end
321
321
  context 'daily' do
322
322
  let(:time_step) { :days }
323
- it { is_expected.to eq(DateTime.civil(2013,9,13)) }
323
+ it { is_expected.to eq(DateTime.civil(2013, 9, 13)) }
324
324
  end
325
325
  context 'monthly' do
326
326
  let(:time_step) { :months }
327
- it { is_expected.to eq(DateTime.civil(2013,9)) }
327
+ it { is_expected.to eq(DateTime.civil(2013, 9)) }
328
328
  end
329
329
  context 'yearly' do
330
330
  let(:time_step) { :years }