masamune 0.17.12 → 0.17.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +3 -3
  3. data/bin/masamune-aws-emr +1 -1
  4. data/bin/masamune-dump +1 -1
  5. data/bin/masamune-hive +1 -1
  6. data/bin/masamune-psql +1 -1
  7. data/bin/masamune-shell +1 -1
  8. data/lib/masamune.rb +1 -1
  9. data/lib/masamune/actions/aws_emr.rb +2 -2
  10. data/lib/masamune/actions/data_flow.rb +10 -10
  11. data/lib/masamune/actions/date_parse.rb +4 -6
  12. data/lib/masamune/actions/execute.rb +3 -3
  13. data/lib/masamune/actions/filesystem.rb +1 -1
  14. data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
  15. data/lib/masamune/actions/hive.rb +5 -5
  16. data/lib/masamune/actions/invoke_parallel.rb +5 -5
  17. data/lib/masamune/actions/postgres.rb +6 -6
  18. data/lib/masamune/actions/postgres_admin.rb +1 -1
  19. data/lib/masamune/actions/s3cmd.rb +3 -3
  20. data/lib/masamune/actions/transform.rb +4 -4
  21. data/lib/masamune/after_initialize_callbacks.rb +4 -4
  22. data/lib/masamune/cached_filesystem.rb +7 -11
  23. data/lib/masamune/commands/aws_emr.rb +12 -12
  24. data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
  25. data/lib/masamune/commands/hadoop_streaming.rb +16 -16
  26. data/lib/masamune/commands/hive.rb +31 -35
  27. data/lib/masamune/commands/postgres.rb +30 -32
  28. data/lib/masamune/commands/postgres_admin.rb +17 -16
  29. data/lib/masamune/commands/postgres_common.rb +1 -1
  30. data/lib/masamune/commands/retry_with_backoff.rb +19 -19
  31. data/lib/masamune/commands/s3cmd.rb +7 -7
  32. data/lib/masamune/commands/shell.rb +25 -38
  33. data/lib/masamune/configuration.rb +8 -9
  34. data/lib/masamune/data_plan/builder.rb +5 -5
  35. data/lib/masamune/data_plan/elem.rb +7 -7
  36. data/lib/masamune/data_plan/engine.rb +12 -15
  37. data/lib/masamune/data_plan/rule.rb +14 -12
  38. data/lib/masamune/data_plan/set.rb +13 -13
  39. data/lib/masamune/environment.rb +3 -3
  40. data/lib/masamune/filesystem.rb +29 -28
  41. data/lib/masamune/helpers/postgres.rb +8 -5
  42. data/lib/masamune/io.rb +2 -2
  43. data/lib/masamune/last_element.rb +3 -3
  44. data/lib/masamune/method_logger.rb +1 -1
  45. data/lib/masamune/schema/catalog.rb +9 -12
  46. data/lib/masamune/schema/column.rb +12 -10
  47. data/lib/masamune/schema/dimension.rb +5 -4
  48. data/lib/masamune/schema/fact.rb +2 -2
  49. data/lib/masamune/schema/map.rb +9 -13
  50. data/lib/masamune/schema/row.rb +3 -2
  51. data/lib/masamune/schema/store.rb +13 -14
  52. data/lib/masamune/schema/table.rb +24 -28
  53. data/lib/masamune/schema/table_reference.rb +7 -7
  54. data/lib/masamune/spec_helper.rb +1 -1
  55. data/lib/masamune/string_format.rb +1 -1
  56. data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
  57. data/lib/masamune/tasks/dump_thor.rb +5 -5
  58. data/lib/masamune/tasks/hive_thor.rb +13 -13
  59. data/lib/masamune/tasks/postgres_thor.rb +8 -8
  60. data/lib/masamune/tasks/shell_thor.rb +3 -3
  61. data/lib/masamune/template.rb +4 -4
  62. data/lib/masamune/thor.rb +26 -25
  63. data/lib/masamune/transform/common/denormalize_table.rb +6 -6
  64. data/lib/masamune/transform/define_table.rb +1 -1
  65. data/lib/masamune/transform/hive/define_table.rb +0 -2
  66. data/lib/masamune/transform/insert_reference_values.rb +1 -1
  67. data/lib/masamune/transform/operator.rb +3 -2
  68. data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
  69. data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
  70. data/lib/masamune/transform/postgres/define_table.rb +2 -2
  71. data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
  72. data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
  73. data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
  74. data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
  75. data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
  76. data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
  77. data/lib/masamune/version.rb +1 -1
  78. data/spec/masamune/actions/aws_emr_spec.rb +8 -8
  79. data/spec/masamune/actions/execute_spec.rb +6 -6
  80. data/spec/masamune/actions/hive_spec.rb +20 -15
  81. data/spec/masamune/actions/postgres_spec.rb +17 -10
  82. data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
  83. data/spec/masamune/commands/aws_emr_spec.rb +10 -10
  84. data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
  85. data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
  86. data/spec/masamune/commands/hive_spec.rb +11 -11
  87. data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
  88. data/spec/masamune/commands/postgres_spec.rb +15 -15
  89. data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
  90. data/spec/masamune/commands/s3cmd_spec.rb +3 -3
  91. data/spec/masamune/commands/shell_spec.rb +16 -16
  92. data/spec/masamune/data_plan/builder_spec.rb +22 -19
  93. data/spec/masamune/data_plan/elem_spec.rb +20 -20
  94. data/spec/masamune/data_plan/engine_spec.rb +41 -33
  95. data/spec/masamune/data_plan/rule_spec.rb +19 -19
  96. data/spec/masamune/data_plan/set_spec.rb +8 -6
  97. data/spec/masamune/environment_spec.rb +5 -5
  98. data/spec/masamune/filesystem_spec.rb +87 -87
  99. data/spec/masamune/helpers/postgres_spec.rb +1 -1
  100. data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
  101. data/spec/masamune/schema/catalog_spec.rb +16 -16
  102. data/spec/masamune/schema/column_spec.rb +19 -19
  103. data/spec/masamune/schema/dimension_spec.rb +2 -2
  104. data/spec/masamune/schema/fact_spec.rb +3 -3
  105. data/spec/masamune/schema/map_spec.rb +41 -41
  106. data/spec/masamune/schema/table_spec.rb +16 -16
  107. data/spec/masamune/string_format_spec.rb +7 -7
  108. data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
  109. data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
  110. data/spec/masamune/template_spec.rb +3 -2
  111. data/spec/masamune/thor_spec.rb +8 -8
  112. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
  113. data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
  114. data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
  115. data/spec/masamune/transform/define_table.table_spec.rb +6 -6
  116. data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
  117. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
  118. data/spec/masamune/transform/load_dimension_spec.rb +1 -1
  119. data/spec/masamune/transform/load_fact_spec.rb +1 -1
  120. data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
  121. data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
  122. data/spec/masamune/transform/stage_fact_spec.rb +4 -4
  123. data/spec/spec_helper.rb +1 -1
  124. data/spec/support/masamune/example_group.rb +2 -2
  125. data/spec/support/masamune/job_example_group.rb +3 -3
  126. data/spec/support/masamune/job_fixture.rb +4 -4
  127. data/spec/support/masamune/mock_command.rb +10 -10
  128. data/spec/support/masamune/mock_delegate.rb +2 -2
  129. data/spec/support/masamune/mock_filesystem.rb +3 -3
  130. data/spec/support/masamune/shared_example_group.rb +14 -16
  131. data/spec/support/masamune/step_example_group.rb +4 -4
  132. data/spec/support/masamune/step_fixture.rb +1 -1
  133. data/spec/support/masamune/task_example_group.rb +1 -1
  134. data/spec/support/masamune/thor_mute.rb +3 -2
  135. data/spec/support/rspec/example/action_example_group.rb +1 -1
  136. data/spec/support/rspec/example/task_example_group.rb +7 -4
  137. data/spec/support/rspec/example/transform_example_group.rb +1 -1
  138. data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
  139. metadata +16 -2
@@ -27,27 +27,28 @@ describe Masamune::DataPlan::Builder do
27
27
  end
28
28
 
29
29
  context 'with multiple namespaces' do
30
- let(:namespaces) { ['a', 'b'] }
31
- let(:commands) { ['load', 'store'] }
30
+ let(:namespaces) { %w(a b) }
31
+ let(:commands) { %w(load store) }
32
32
  let(:sources) { [{ path: 'log/%Y%m%d.*.log' }, { path: 'table/y=%Y/m=%m/d=%d' }] }
33
33
  let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d' }, { path: 'daily/%Y-%m-%d' }] }
34
34
 
35
35
  before do
36
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', { path: 'table/y=%Y/m=%m/d=%d'})
37
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', { path: 'log/%Y%m%d.*.log' })
36
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', path: 'table/y=%Y/m=%m/d=%d')
37
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', path: 'log/%Y%m%d.*.log')
38
38
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:load', an_instance_of(Proc))
39
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', { path: 'daily/%Y-%m-%d' })
40
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', { path: 'table/y=%Y/m=%m/d=%d' })
39
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', path: 'daily/%Y-%m-%d')
40
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', path: 'table/y=%Y/m=%m/d=%d')
41
41
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('b:store', an_instance_of(Proc))
42
42
  subject
43
43
  end
44
44
 
45
- it 'should build a Masamune::DataPlan::Engine instance' do; end
45
+ it 'should build a Masamune::DataPlan::Engine instance' do
46
+ end
46
47
  end
47
48
 
48
49
  context 'with :for option' do
49
- let(:namespaces) { ['a', 'a', 'a'] }
50
- let(:commands) { ['missing_before', 'override', 'missing_after'] }
50
+ let(:namespaces) { %w(a a a) }
51
+ let(:commands) { %w(missing_before override missing_after) }
51
52
  let(:sources) { [{ path: 'log/%Y%m%d.*.log', for: 'override' }] }
52
53
  let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d', for: 'override' }] }
53
54
 
@@ -55,8 +56,8 @@ describe Masamune::DataPlan::Builder do
55
56
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
56
57
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
57
58
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
58
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', { path: 'table/y=%Y/m=%m/d=%d' })
59
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', { path: 'log/%Y%m%d.*.log' })
59
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
60
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
60
61
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
61
62
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_after', anything)
62
63
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_after', anything)
@@ -64,26 +65,28 @@ describe Masamune::DataPlan::Builder do
64
65
  subject
65
66
  end
66
67
 
67
- it 'should build a Masamune::DataPlan::Engine instance' do; end
68
+ it 'should build a Masamune::DataPlan::Engine instance' do
69
+ end
68
70
  end
69
71
 
70
72
  context 'with :skip option' do
71
- let(:namespaces) { ['a', 'a', 'a'] }
72
- let(:commands) { ['missing_before', 'override'] }
73
- let(:sources) { [{ skip: true}, { path: 'log/%Y%m%d.*.log' }] }
74
- let(:targets) { [{ skip: true}, { path: 'table/y=%Y/m=%m/d=%d' }] }
73
+ let(:namespaces) { %w(a a a) }
74
+ let(:commands) { %w(missing_before override) }
75
+ let(:sources) { [{ skip: true }, { path: 'log/%Y%m%d.*.log' }] }
76
+ let(:targets) { [{ skip: true }, { path: 'table/y=%Y/m=%m/d=%d' }] }
75
77
 
76
78
  before do
77
79
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
78
80
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
79
81
  expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
80
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', { path: 'table/y=%Y/m=%m/d=%d' })
81
- expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', { path: 'log/%Y%m%d.*.log' })
82
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
83
+ expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
82
84
  expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
83
85
  subject
84
86
  end
85
87
 
86
- it 'should build a Masamune::DataPlan::Engine instance' do; end
88
+ it 'should build a Masamune::DataPlan::Engine instance' do
89
+ end
87
90
  end
88
91
  end
89
92
  end
@@ -32,14 +32,14 @@ describe Masamune::DataPlan::Elem do
32
32
 
33
33
  let(:name) { 'primary' }
34
34
  let(:type) { :target }
35
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
36
- let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/log/%Y%m%d.*.log'}) }
35
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
36
+ let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/log/%Y%m%d.*.log') }
37
37
 
38
- let(:start_time) { DateTime.civil(2013,07,19,11,07) }
39
- let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
38
+ let(:start_time) { DateTime.civil(2013, 07, 19, 11, 07) }
39
+ let(:other_start_time) { DateTime.civil(2013, 07, 20, 0, 0) }
40
40
 
41
- let(:options) { {tz: 'EST'} }
42
- let(:other_options) { {tz: 'PST'} }
41
+ let(:options) { { tz: 'EST' } }
42
+ let(:other_options) { { tz: 'PST' } }
43
43
 
44
44
  let(:instance) { described_class.new(rule, start_time, options) }
45
45
 
@@ -80,8 +80,8 @@ describe Masamune::DataPlan::Elem do
80
80
  end
81
81
 
82
82
  describe '#last_modified_at' do
83
- let(:early) { Time.parse("2014-05-01 00:00:00 +0000") }
84
- let(:later) { Time.parse("2014-06-01 00:00:00 +0000") }
83
+ let(:early) { Time.parse('2014-05-01 00:00:00 +0000') }
84
+ let(:later) { Time.parse('2014-06-01 00:00:00 +0000') }
85
85
 
86
86
  subject do
87
87
  instance.last_modified_at.utc
@@ -89,8 +89,8 @@ describe Masamune::DataPlan::Elem do
89
89
 
90
90
  context 'with missing mtime' do
91
91
  before do
92
- expect(rule.engine.filesystem).to receive(:stat).with(instance.path).
93
- and_return(nil)
92
+ expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
93
+ .and_return(nil)
94
94
  end
95
95
 
96
96
  it { is_expected.to eq(Masamune::DataPlan::Elem::MISSING_MODIFIED_AT) }
@@ -98,8 +98,8 @@ describe Masamune::DataPlan::Elem do
98
98
 
99
99
  context 'with single mtime' do
100
100
  before do
101
- expect(rule.engine.filesystem).to receive(:stat).with(instance.path).
102
- and_return(OpenStruct.new(mtime: early))
101
+ expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
102
+ .and_return(OpenStruct.new(mtime: early))
103
103
  end
104
104
 
105
105
  it { is_expected.to eq(early) }
@@ -110,7 +110,7 @@ describe Masamune::DataPlan::Elem do
110
110
  subject { instance.explode.map(&:path) }
111
111
 
112
112
  context 'with free path and existing files' do
113
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
113
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
114
114
  before do
115
115
  engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
116
116
  end
@@ -118,12 +118,12 @@ describe Masamune::DataPlan::Elem do
118
118
  end
119
119
 
120
120
  context 'with free path and missing files' do
121
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
121
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
122
122
  it { is_expected.to be_empty }
123
123
  end
124
124
 
125
125
  context 'with bound path and existing files' do
126
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
126
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
127
127
  before do
128
128
  engine.filesystem.touch!('/report/file')
129
129
  end
@@ -131,12 +131,12 @@ describe Masamune::DataPlan::Elem do
131
131
  end
132
132
 
133
133
  context 'with bound path and missing files' do
134
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
134
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
135
135
  it { is_expected.to be_empty }
136
136
  end
137
137
 
138
138
  context 'with free table and existing table' do
139
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
139
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
140
140
  before do
141
141
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
142
142
  end
@@ -144,7 +144,7 @@ describe Masamune::DataPlan::Elem do
144
144
  end
145
145
 
146
146
  context 'with free table and missing table' do
147
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
147
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
148
148
  before do
149
149
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
150
150
  end
@@ -152,7 +152,7 @@ describe Masamune::DataPlan::Elem do
152
152
  end
153
153
 
154
154
  context 'with bound table and existing table' do
155
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
155
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
156
156
  before do
157
157
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
158
158
  end
@@ -160,7 +160,7 @@ describe Masamune::DataPlan::Elem do
160
160
  end
161
161
 
162
162
  context 'with bound table and missing table' do
163
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
163
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
164
164
  before do
165
165
  expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
166
166
  end
@@ -31,7 +31,7 @@ describe Masamune::DataPlan::Engine do
31
31
  end
32
32
 
33
33
  let(:command) do
34
- Proc.new do |engine, rule|
34
+ proc do |engine, rule|
35
35
  missing_targets = []
36
36
  engine.targets(rule).missing.each do |target|
37
37
  missing_targets << target.path if target.sources.existing.any?
@@ -43,7 +43,7 @@ describe Masamune::DataPlan::Engine do
43
43
  before do
44
44
  engine.add_target_rule('non_primary', path: '/table/y=%Y/m=%m/d=%d', primary: false)
45
45
  engine.add_source_rule('non_primary', path: '/log/%Y%m%d.*.log', primary: false)
46
- engine.add_command_rule('non_primary', ->(*_) { fail } )
46
+ engine.add_command_rule('non_primary', ->(*_) { raise })
47
47
  engine.add_target_rule('primary', path: '/table/y=%Y/m=%m/d=%d')
48
48
  engine.add_source_rule('primary', path: '/log/%Y%m%d.*.log')
49
49
  engine.add_command_rule('primary', command)
@@ -61,8 +61,8 @@ describe Masamune::DataPlan::Engine do
61
61
  end
62
62
 
63
63
  describe '#targets_for_date_range' do
64
- let(:start) { Date.civil(2013,01,01) }
65
- let(:stop) { Date.civil(2013,01,03) }
64
+ let(:start) { Date.civil(2013, 01, 01) }
65
+ let(:stop) { Date.civil(2013, 01, 03) }
66
66
 
67
67
  subject { engine.targets_for_date_range(rule, start, stop).map(&:path) }
68
68
 
@@ -104,26 +104,26 @@ describe Masamune::DataPlan::Engine do
104
104
  let(:rule) { 'primary' }
105
105
  let(:source) { '/log/20130101.random.log' }
106
106
 
107
- it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
108
- it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
107
+ it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
108
+ it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
109
109
  it { expect(targets.first.path).to eq('/table/y=2013/m=01/d=01') }
110
110
  end
111
111
 
112
112
  context 'derived_daily' do
113
113
  let(:rule) { 'derived_daily' }
114
- let(:source) { '/table/y=2013/m=01/d=01' }
114
+ let(:source) { '/table/y=2013/m=01/d=01' }
115
115
 
116
- it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
117
- it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
116
+ it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
117
+ it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
118
118
  it { expect(targets.first.path).to eq('/daily/2013-01-01') }
119
119
  end
120
120
 
121
121
  context 'derived_monthly' do
122
122
  let(:rule) { 'derived_monthly' }
123
- let(:source) { '/table/y=2013/m=01/d=01' }
123
+ let(:source) { '/table/y=2013/m=01/d=01' }
124
124
 
125
- it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
126
- it { expect(targets.first.stop_time).to eq(Date.civil(2013,02,01)) }
125
+ it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
126
+ it { expect(targets.first.stop_time).to eq(Date.civil(2013, 02, 01)) }
127
127
  it { expect(targets.first.path).to eq('/monthly/2013-01') }
128
128
  end
129
129
  end
@@ -167,15 +167,15 @@ describe Masamune::DataPlan::Engine do
167
167
  let(:target) { '/monthly/2013-01' }
168
168
 
169
169
  (1..31).each do |day|
170
- it { expect(sources).to include '/table/y=2013/m=01/d=%02d' % day }
170
+ it { expect(sources).to include format('/table/y=2013/m=01/d=%02d', day) }
171
171
  end
172
172
  it { expect(sources.size).to eq(31) }
173
173
  end
174
174
 
175
175
  context 'invalid target' do
176
176
  let(:rule) { 'derived_daily' }
177
- let(:target) { '/table/y=2013/m=01/d=01' }
178
- it { expect { subject }.to raise_error /Cannot bind_input/ }
177
+ let(:target) { '/table/y=2013/m=01/d=01' }
178
+ it { expect { subject }.to raise_error(/Cannot bind_input/) }
179
179
  end
180
180
  end
181
181
 
@@ -204,7 +204,7 @@ describe Masamune::DataPlan::Engine do
204
204
 
205
205
  context 'invalid target' do
206
206
  let(:target) { '/daily' }
207
- it { expect { subject }.to raise_error /No rule matches/ }
207
+ it { expect { subject }.to raise_error(/No rule matches/) }
208
208
  end
209
209
  end
210
210
 
@@ -224,7 +224,7 @@ describe Masamune::DataPlan::Engine do
224
224
  context 'with :targets' do
225
225
  let(:rule) { 'primary' }
226
226
 
227
- let(:options) { {targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
227
+ let(:options) { { targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
228
228
 
229
229
  it { expect(targets).to include '/table/y=2013/m=01/d=01' }
230
230
  it { expect(targets).to include '/table/y=2013/m=01/d=02' }
@@ -235,10 +235,10 @@ describe Masamune::DataPlan::Engine do
235
235
  context 'with :sources' do
236
236
  let(:rule) { 'derived_daily' }
237
237
 
238
- let(:options) { {sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
238
+ let(:options) { { sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
239
239
 
240
- it { expect(targets).to include "/daily/2013-01-01" }
241
- it { expect(targets).to include "/daily/2013-01-02" }
240
+ it { expect(targets).to include '/daily/2013-01-01' }
241
+ it { expect(targets).to include '/daily/2013-01-02' }
242
242
  it { expect(sources).to include '/table/y=2013/m=01/d=01' }
243
243
  it { expect(sources).to include '/table/y=2013/m=01/d=02' }
244
244
  end
@@ -257,10 +257,13 @@ describe Masamune::DataPlan::Engine do
257
257
 
258
258
  context 'primary rule' do
259
259
  let(:rule) { 'primary' }
260
- let(:targets) { [
261
- '/table/y=2013/m=01/d=01',
262
- '/table/y=2013/m=01/d=02',
263
- '/table/y=2013/m=01/d=03' ] }
260
+ let(:targets) do
261
+ [
262
+ '/table/y=2013/m=01/d=01',
263
+ '/table/y=2013/m=01/d=02',
264
+ '/table/y=2013/m=01/d=03'
265
+ ]
266
+ end
264
267
 
265
268
  context 'when target data exists' do
266
269
  before do
@@ -269,7 +272,8 @@ describe Masamune::DataPlan::Engine do
269
272
  execute
270
273
  end
271
274
 
272
- it 'should not call touch!' do; end
275
+ it 'should not call touch!' do
276
+ end
273
277
  end
274
278
 
275
279
  context 'when partial target data exists' do
@@ -280,7 +284,8 @@ describe Masamune::DataPlan::Engine do
280
284
  execute
281
285
  end
282
286
 
283
- it 'should call touch!' do; end
287
+ it 'should call touch!' do
288
+ end
284
289
  end
285
290
 
286
291
  context 'when source data does not exist' do
@@ -289,13 +294,14 @@ describe Masamune::DataPlan::Engine do
289
294
  execute
290
295
  end
291
296
 
292
- it 'should not call touch!' do; end
297
+ it 'should not call touch!' do
298
+ end
293
299
  end
294
300
  end
295
301
 
296
302
  shared_examples_for 'derived daily data' do
297
303
  context 'when primary target data exists' do
298
- let(:derived_targets) { ['/table/y=2013/m=01/d=01/DATA', '/table/y=2013/m=01/d=02/DATA', '/table/y=2013/m=01/d=03/DATA'] }
304
+ let(:derived_targets) { ['/table/y=2013/m=01/d=01/DATA', '/table/y=2013/m=01/d=02/DATA', '/table/y=2013/m=01/d=03/DATA'] }
299
305
 
300
306
  before do
301
307
  engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
@@ -304,11 +310,12 @@ describe Masamune::DataPlan::Engine do
304
310
  execute
305
311
  end
306
312
 
307
- it 'should call touch!' do; end
313
+ it 'should call touch!' do
314
+ end
308
315
  end
309
316
 
310
317
  context 'when primary target data exists and :resolve is false' do
311
- let(:options) { {resolve: false} }
318
+ let(:options) { { resolve: false } }
312
319
 
313
320
  before do
314
321
  engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
@@ -316,7 +323,8 @@ describe Masamune::DataPlan::Engine do
316
323
  execute
317
324
  end
318
325
 
319
- it 'should not call touch!' do; end
326
+ it 'should not call touch!' do
327
+ end
320
328
  end
321
329
  end
322
330
 
@@ -331,7 +339,7 @@ describe Masamune::DataPlan::Engine do
331
339
 
332
340
  context 'derived_monthly rule' do
333
341
  let(:rule) { 'derived_monthly' }
334
- let(:targets) { ['/monthly/2013-01/DATA'] }
342
+ let(:targets) { ['/monthly/2013-01/DATA'] }
335
343
 
336
344
  it_behaves_like 'derived daily data' do
337
345
  let(:derived_command) { derived_monthly_command }
@@ -348,7 +356,7 @@ describe Masamune::DataPlan::Engine do
348
356
  end
349
357
 
350
358
  it 'should raise exception' do
351
- expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error /Max depth .* exceeded for rule 'derived'/
359
+ expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error(/Max depth .* exceeded for rule 'derived'/)
352
360
  end
353
361
  end
354
362
  end
@@ -25,7 +25,7 @@ describe Masamune::DataPlan::Rule do
25
25
  let(:name) { 'primary' }
26
26
  let(:type) { :target }
27
27
  let(:pattern) { 'report/%Y-%m-%d/%H' }
28
- let(:options) { {path: pattern} }
28
+ let(:options) { { path: pattern } }
29
29
 
30
30
  let(:instance) { described_class.new(engine, name, type, options) }
31
31
 
@@ -40,7 +40,7 @@ describe Masamune::DataPlan::Rule do
40
40
  end
41
41
 
42
42
  context 'with lambda' do
43
- let(:pattern) { lambda { |_| 'report/%Y-%m-%d/%H' } }
43
+ let(:pattern) { ->(_) { 'report/%Y-%m-%d/%H' } }
44
44
  it { is_expected.to eq('report/%Y-%m-%d/%H') }
45
45
  end
46
46
  end
@@ -59,7 +59,7 @@ describe Masamune::DataPlan::Rule do
59
59
  end
60
60
 
61
61
  context 'with DateTime input' do
62
- let(:input) { DateTime.civil(2013,04,05,23,13) }
62
+ let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
63
63
 
64
64
  describe '#path' do
65
65
  subject { elem.path }
@@ -69,7 +69,7 @@ describe Masamune::DataPlan::Rule do
69
69
 
70
70
  context 'with DateTime input and unix timestamp pattern' do
71
71
  let(:pattern) { 'logs/%H-s.log' }
72
- let(:input) { DateTime.civil(2013,04,05,23,13) }
72
+ let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
73
73
 
74
74
  describe '#path' do
75
75
  subject { elem.path }
@@ -78,7 +78,7 @@ describe Masamune::DataPlan::Rule do
78
78
  end
79
79
 
80
80
  context 'with Date input' do
81
- let(:input) { Date.civil(2013,04,05) }
81
+ let(:input) { Date.civil(2013, 04, 05) }
82
82
 
83
83
  describe '#path' do
84
84
  subject { elem.path }
@@ -92,7 +92,7 @@ describe Masamune::DataPlan::Rule do
92
92
 
93
93
  context 'with default' do
94
94
  let(:input) { 'report/2013-04-05/23' }
95
- let(:output_date) { DateTime.civil(2013,04,05,23) }
95
+ let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
96
96
 
97
97
  describe '#path' do
98
98
  subject { elem.path }
@@ -113,7 +113,7 @@ describe Masamune::DataPlan::Rule do
113
113
  context 'with unix timestamp pattern' do
114
114
  let(:pattern) { 'logs/%H-s.log' }
115
115
  let(:input) { 'logs/1365202800.log' }
116
- let(:output_date) { DateTime.civil(2013,04,05,23) }
116
+ let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
117
117
 
118
118
  describe '#path' do
119
119
  subject { elem.path }
@@ -140,7 +140,7 @@ describe Masamune::DataPlan::Rule do
140
140
  context 'with wildcard pattern' do
141
141
  let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
142
142
  let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
143
- let(:output_date) { DateTime.civil(2013,04,30,20) }
143
+ let(:output_date) { DateTime.civil(2013, 04, 30, 20) }
144
144
 
145
145
  describe '#path' do
146
146
  subject { elem.path }
@@ -160,8 +160,8 @@ describe Masamune::DataPlan::Rule do
160
160
  end
161
161
 
162
162
  describe '#unify' do
163
- let(:primary) { described_class.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
164
- let(:induced) { described_class.new(engine, name, type, {path: 'table/y=%Y/m=%m/d=%d/h=%H'}) }
163
+ let(:primary) { described_class.new(engine, name, type, path: 'report/%Y-%m-%d/%H') }
164
+ let(:induced) { described_class.new(engine, name, type, path: 'table/y=%Y/m=%m/d=%d/h=%H') }
165
165
  let(:elem) { primary.bind_input(input) }
166
166
 
167
167
  subject(:new_elem) { instance.unify(elem, induced) }
@@ -176,7 +176,7 @@ describe Masamune::DataPlan::Rule do
176
176
  end
177
177
 
178
178
  context 'when input partially matches basis pattern' do
179
- let(:induced) { described_class.new(engine, name, type, {path: 'table/%Y-%m'}) }
179
+ let(:induced) { described_class.new(engine, name, type, path: 'table/%Y-%m') }
180
180
 
181
181
  let(:input) { 'report/2013-01-02/00' }
182
182
 
@@ -240,14 +240,14 @@ describe Masamune::DataPlan::Rule do
240
240
 
241
241
  describe '#generate' do
242
242
  context 'with a block' do
243
- let(:start_date) { DateTime.civil(2013,04,05,20) }
244
- let(:stop_date) { DateTime.civil(2013,04,05,20) }
243
+ let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
244
+ let(:stop_date) { DateTime.civil(2013, 04, 05, 20) }
245
245
  specify { expect { |b| instance.generate(start_date, stop_date, &b) }.to yield_control }
246
246
  end
247
247
 
248
248
  context 'without a block' do
249
- let(:start_date) { DateTime.civil(2013,04,05,20) }
250
- let(:stop_date) { DateTime.civil(2013,04,05,22) }
249
+ let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
250
+ let(:stop_date) { DateTime.civil(2013, 04, 05, 22) }
251
251
 
252
252
  subject(:elems) do
253
253
  instance.generate(start_date, stop_date)
@@ -307,7 +307,7 @@ describe Masamune::DataPlan::Rule do
307
307
  end
308
308
 
309
309
  describe '#time_round' do
310
- let(:input_time) { DateTime.civil(2013,9,13,23,13) }
310
+ let(:input_time) { DateTime.civil(2013, 9, 13, 23, 13) }
311
311
  subject { instance.time_round(input_time) }
312
312
 
313
313
  before do
@@ -316,15 +316,15 @@ describe Masamune::DataPlan::Rule do
316
316
 
317
317
  context 'hourly' do
318
318
  let(:time_step) { :hours }
319
- it { is_expected.to eq(DateTime.civil(2013,9,13,23)) }
319
+ it { is_expected.to eq(DateTime.civil(2013, 9, 13, 23)) }
320
320
  end
321
321
  context 'daily' do
322
322
  let(:time_step) { :days }
323
- it { is_expected.to eq(DateTime.civil(2013,9,13)) }
323
+ it { is_expected.to eq(DateTime.civil(2013, 9, 13)) }
324
324
  end
325
325
  context 'monthly' do
326
326
  let(:time_step) { :months }
327
- it { is_expected.to eq(DateTime.civil(2013,9)) }
327
+ it { is_expected.to eq(DateTime.civil(2013, 9)) }
328
328
  end
329
329
  context 'yearly' do
330
330
  let(:time_step) { :years }