masamune 0.17.12 → 0.17.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -27,27 +27,28 @@ describe Masamune::DataPlan::Builder do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
context 'with multiple namespaces' do
|
30
|
-
let(:namespaces) {
|
31
|
-
let(:commands) {
|
30
|
+
let(:namespaces) { %w(a b) }
|
31
|
+
let(:commands) { %w(load store) }
|
32
32
|
let(:sources) { [{ path: 'log/%Y%m%d.*.log' }, { path: 'table/y=%Y/m=%m/d=%d' }] }
|
33
33
|
let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d' }, { path: 'daily/%Y-%m-%d' }] }
|
34
34
|
|
35
35
|
before do
|
36
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load',
|
37
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load',
|
36
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', path: 'table/y=%Y/m=%m/d=%d')
|
37
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', path: 'log/%Y%m%d.*.log')
|
38
38
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:load', an_instance_of(Proc))
|
39
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store',
|
40
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store',
|
39
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', path: 'daily/%Y-%m-%d')
|
40
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', path: 'table/y=%Y/m=%m/d=%d')
|
41
41
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('b:store', an_instance_of(Proc))
|
42
42
|
subject
|
43
43
|
end
|
44
44
|
|
45
|
-
it 'should build a Masamune::DataPlan::Engine instance' do
|
45
|
+
it 'should build a Masamune::DataPlan::Engine instance' do
|
46
|
+
end
|
46
47
|
end
|
47
48
|
|
48
49
|
context 'with :for option' do
|
49
|
-
let(:namespaces) {
|
50
|
-
let(:commands) {
|
50
|
+
let(:namespaces) { %w(a a a) }
|
51
|
+
let(:commands) { %w(missing_before override missing_after) }
|
51
52
|
let(:sources) { [{ path: 'log/%Y%m%d.*.log', for: 'override' }] }
|
52
53
|
let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d', for: 'override' }] }
|
53
54
|
|
@@ -55,8 +56,8 @@ describe Masamune::DataPlan::Builder do
|
|
55
56
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
|
56
57
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
|
57
58
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
|
58
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override',
|
59
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override',
|
59
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
|
60
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
|
60
61
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
|
61
62
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_after', anything)
|
62
63
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_after', anything)
|
@@ -64,26 +65,28 @@ describe Masamune::DataPlan::Builder do
|
|
64
65
|
subject
|
65
66
|
end
|
66
67
|
|
67
|
-
it 'should build a Masamune::DataPlan::Engine instance' do
|
68
|
+
it 'should build a Masamune::DataPlan::Engine instance' do
|
69
|
+
end
|
68
70
|
end
|
69
71
|
|
70
72
|
context 'with :skip option' do
|
71
|
-
let(:namespaces) {
|
72
|
-
let(:commands) {
|
73
|
-
let(:sources) { [{ skip: true}, { path: 'log/%Y%m%d.*.log' }] }
|
74
|
-
let(:targets) { [{ skip: true}, { path: 'table/y=%Y/m=%m/d=%d' }] }
|
73
|
+
let(:namespaces) { %w(a a a) }
|
74
|
+
let(:commands) { %w(missing_before override) }
|
75
|
+
let(:sources) { [{ skip: true }, { path: 'log/%Y%m%d.*.log' }] }
|
76
|
+
let(:targets) { [{ skip: true }, { path: 'table/y=%Y/m=%m/d=%d' }] }
|
75
77
|
|
76
78
|
before do
|
77
79
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
|
78
80
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
|
79
81
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
|
80
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override',
|
81
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override',
|
82
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
|
83
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
|
82
84
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
|
83
85
|
subject
|
84
86
|
end
|
85
87
|
|
86
|
-
it 'should build a Masamune::DataPlan::Engine instance' do
|
88
|
+
it 'should build a Masamune::DataPlan::Engine instance' do
|
89
|
+
end
|
87
90
|
end
|
88
91
|
end
|
89
92
|
end
|
@@ -32,14 +32,14 @@ describe Masamune::DataPlan::Elem do
|
|
32
32
|
|
33
33
|
let(:name) { 'primary' }
|
34
34
|
let(:type) { :target }
|
35
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
36
|
-
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
35
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
|
36
|
+
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/log/%Y%m%d.*.log') }
|
37
37
|
|
38
|
-
let(:start_time) { DateTime.civil(2013,07,19,11,07) }
|
39
|
-
let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
|
38
|
+
let(:start_time) { DateTime.civil(2013, 07, 19, 11, 07) }
|
39
|
+
let(:other_start_time) { DateTime.civil(2013, 07, 20, 0, 0) }
|
40
40
|
|
41
|
-
let(:options) { {tz: 'EST'} }
|
42
|
-
let(:other_options) { {tz: 'PST'} }
|
41
|
+
let(:options) { { tz: 'EST' } }
|
42
|
+
let(:other_options) { { tz: 'PST' } }
|
43
43
|
|
44
44
|
let(:instance) { described_class.new(rule, start_time, options) }
|
45
45
|
|
@@ -80,8 +80,8 @@ describe Masamune::DataPlan::Elem do
|
|
80
80
|
end
|
81
81
|
|
82
82
|
describe '#last_modified_at' do
|
83
|
-
let(:early) { Time.parse(
|
84
|
-
let(:later) { Time.parse(
|
83
|
+
let(:early) { Time.parse('2014-05-01 00:00:00 +0000') }
|
84
|
+
let(:later) { Time.parse('2014-06-01 00:00:00 +0000') }
|
85
85
|
|
86
86
|
subject do
|
87
87
|
instance.last_modified_at.utc
|
@@ -89,8 +89,8 @@ describe Masamune::DataPlan::Elem do
|
|
89
89
|
|
90
90
|
context 'with missing mtime' do
|
91
91
|
before do
|
92
|
-
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
93
|
-
and_return(nil)
|
92
|
+
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
93
|
+
.and_return(nil)
|
94
94
|
end
|
95
95
|
|
96
96
|
it { is_expected.to eq(Masamune::DataPlan::Elem::MISSING_MODIFIED_AT) }
|
@@ -98,8 +98,8 @@ describe Masamune::DataPlan::Elem do
|
|
98
98
|
|
99
99
|
context 'with single mtime' do
|
100
100
|
before do
|
101
|
-
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
102
|
-
and_return(OpenStruct.new(mtime: early))
|
101
|
+
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
102
|
+
.and_return(OpenStruct.new(mtime: early))
|
103
103
|
end
|
104
104
|
|
105
105
|
it { is_expected.to eq(early) }
|
@@ -110,7 +110,7 @@ describe Masamune::DataPlan::Elem do
|
|
110
110
|
subject { instance.explode.map(&:path) }
|
111
111
|
|
112
112
|
context 'with free path and existing files' do
|
113
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
113
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
|
114
114
|
before do
|
115
115
|
engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
|
116
116
|
end
|
@@ -118,12 +118,12 @@ describe Masamune::DataPlan::Elem do
|
|
118
118
|
end
|
119
119
|
|
120
120
|
context 'with free path and missing files' do
|
121
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
121
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
|
122
122
|
it { is_expected.to be_empty }
|
123
123
|
end
|
124
124
|
|
125
125
|
context 'with bound path and existing files' do
|
126
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
126
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
|
127
127
|
before do
|
128
128
|
engine.filesystem.touch!('/report/file')
|
129
129
|
end
|
@@ -131,12 +131,12 @@ describe Masamune::DataPlan::Elem do
|
|
131
131
|
end
|
132
132
|
|
133
133
|
context 'with bound path and missing files' do
|
134
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
134
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
|
135
135
|
it { is_expected.to be_empty }
|
136
136
|
end
|
137
137
|
|
138
138
|
context 'with free table and existing table' do
|
139
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
139
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
|
140
140
|
before do
|
141
141
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
142
142
|
end
|
@@ -144,7 +144,7 @@ describe Masamune::DataPlan::Elem do
|
|
144
144
|
end
|
145
145
|
|
146
146
|
context 'with free table and missing table' do
|
147
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
147
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
|
148
148
|
before do
|
149
149
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
150
150
|
end
|
@@ -152,7 +152,7 @@ describe Masamune::DataPlan::Elem do
|
|
152
152
|
end
|
153
153
|
|
154
154
|
context 'with bound table and existing table' do
|
155
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
155
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
|
156
156
|
before do
|
157
157
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
158
158
|
end
|
@@ -160,7 +160,7 @@ describe Masamune::DataPlan::Elem do
|
|
160
160
|
end
|
161
161
|
|
162
162
|
context 'with bound table and missing table' do
|
163
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
163
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
|
164
164
|
before do
|
165
165
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
166
166
|
end
|
@@ -31,7 +31,7 @@ describe Masamune::DataPlan::Engine do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
let(:command) do
|
34
|
-
|
34
|
+
proc do |engine, rule|
|
35
35
|
missing_targets = []
|
36
36
|
engine.targets(rule).missing.each do |target|
|
37
37
|
missing_targets << target.path if target.sources.existing.any?
|
@@ -43,7 +43,7 @@ describe Masamune::DataPlan::Engine do
|
|
43
43
|
before do
|
44
44
|
engine.add_target_rule('non_primary', path: '/table/y=%Y/m=%m/d=%d', primary: false)
|
45
45
|
engine.add_source_rule('non_primary', path: '/log/%Y%m%d.*.log', primary: false)
|
46
|
-
engine.add_command_rule('non_primary', ->(*_) {
|
46
|
+
engine.add_command_rule('non_primary', ->(*_) { raise })
|
47
47
|
engine.add_target_rule('primary', path: '/table/y=%Y/m=%m/d=%d')
|
48
48
|
engine.add_source_rule('primary', path: '/log/%Y%m%d.*.log')
|
49
49
|
engine.add_command_rule('primary', command)
|
@@ -61,8 +61,8 @@ describe Masamune::DataPlan::Engine do
|
|
61
61
|
end
|
62
62
|
|
63
63
|
describe '#targets_for_date_range' do
|
64
|
-
let(:start) { Date.civil(2013,01,01) }
|
65
|
-
let(:stop) { Date.civil(2013,01,03) }
|
64
|
+
let(:start) { Date.civil(2013, 01, 01) }
|
65
|
+
let(:stop) { Date.civil(2013, 01, 03) }
|
66
66
|
|
67
67
|
subject { engine.targets_for_date_range(rule, start, stop).map(&:path) }
|
68
68
|
|
@@ -104,26 +104,26 @@ describe Masamune::DataPlan::Engine do
|
|
104
104
|
let(:rule) { 'primary' }
|
105
105
|
let(:source) { '/log/20130101.random.log' }
|
106
106
|
|
107
|
-
it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
|
108
|
-
it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
|
107
|
+
it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
|
108
|
+
it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
|
109
109
|
it { expect(targets.first.path).to eq('/table/y=2013/m=01/d=01') }
|
110
110
|
end
|
111
111
|
|
112
112
|
context 'derived_daily' do
|
113
113
|
let(:rule) { 'derived_daily' }
|
114
|
-
let(:source)
|
114
|
+
let(:source) { '/table/y=2013/m=01/d=01' }
|
115
115
|
|
116
|
-
it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
|
117
|
-
it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
|
116
|
+
it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
|
117
|
+
it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
|
118
118
|
it { expect(targets.first.path).to eq('/daily/2013-01-01') }
|
119
119
|
end
|
120
120
|
|
121
121
|
context 'derived_monthly' do
|
122
122
|
let(:rule) { 'derived_monthly' }
|
123
|
-
let(:source)
|
123
|
+
let(:source) { '/table/y=2013/m=01/d=01' }
|
124
124
|
|
125
|
-
it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
|
126
|
-
it { expect(targets.first.stop_time).to eq(Date.civil(2013,02,01)) }
|
125
|
+
it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
|
126
|
+
it { expect(targets.first.stop_time).to eq(Date.civil(2013, 02, 01)) }
|
127
127
|
it { expect(targets.first.path).to eq('/monthly/2013-01') }
|
128
128
|
end
|
129
129
|
end
|
@@ -167,15 +167,15 @@ describe Masamune::DataPlan::Engine do
|
|
167
167
|
let(:target) { '/monthly/2013-01' }
|
168
168
|
|
169
169
|
(1..31).each do |day|
|
170
|
-
it { expect(sources).to include '/table/y=2013/m=01/d=%02d'
|
170
|
+
it { expect(sources).to include format('/table/y=2013/m=01/d=%02d', day) }
|
171
171
|
end
|
172
172
|
it { expect(sources.size).to eq(31) }
|
173
173
|
end
|
174
174
|
|
175
175
|
context 'invalid target' do
|
176
176
|
let(:rule) { 'derived_daily' }
|
177
|
-
let(:target) {
|
178
|
-
it { expect { subject }.to raise_error
|
177
|
+
let(:target) { '/table/y=2013/m=01/d=01' }
|
178
|
+
it { expect { subject }.to raise_error(/Cannot bind_input/) }
|
179
179
|
end
|
180
180
|
end
|
181
181
|
|
@@ -204,7 +204,7 @@ describe Masamune::DataPlan::Engine do
|
|
204
204
|
|
205
205
|
context 'invalid target' do
|
206
206
|
let(:target) { '/daily' }
|
207
|
-
it { expect { subject }.to raise_error
|
207
|
+
it { expect { subject }.to raise_error(/No rule matches/) }
|
208
208
|
end
|
209
209
|
end
|
210
210
|
|
@@ -224,7 +224,7 @@ describe Masamune::DataPlan::Engine do
|
|
224
224
|
context 'with :targets' do
|
225
225
|
let(:rule) { 'primary' }
|
226
226
|
|
227
|
-
let(:options) { {targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
|
227
|
+
let(:options) { { targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
|
228
228
|
|
229
229
|
it { expect(targets).to include '/table/y=2013/m=01/d=01' }
|
230
230
|
it { expect(targets).to include '/table/y=2013/m=01/d=02' }
|
@@ -235,10 +235,10 @@ describe Masamune::DataPlan::Engine do
|
|
235
235
|
context 'with :sources' do
|
236
236
|
let(:rule) { 'derived_daily' }
|
237
237
|
|
238
|
-
let(:options) { {sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
|
238
|
+
let(:options) { { sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
|
239
239
|
|
240
|
-
it { expect(targets).to include
|
241
|
-
it { expect(targets).to include
|
240
|
+
it { expect(targets).to include '/daily/2013-01-01' }
|
241
|
+
it { expect(targets).to include '/daily/2013-01-02' }
|
242
242
|
it { expect(sources).to include '/table/y=2013/m=01/d=01' }
|
243
243
|
it { expect(sources).to include '/table/y=2013/m=01/d=02' }
|
244
244
|
end
|
@@ -257,10 +257,13 @@ describe Masamune::DataPlan::Engine do
|
|
257
257
|
|
258
258
|
context 'primary rule' do
|
259
259
|
let(:rule) { 'primary' }
|
260
|
-
let(:targets)
|
261
|
-
|
262
|
-
|
263
|
-
|
260
|
+
let(:targets) do
|
261
|
+
[
|
262
|
+
'/table/y=2013/m=01/d=01',
|
263
|
+
'/table/y=2013/m=01/d=02',
|
264
|
+
'/table/y=2013/m=01/d=03'
|
265
|
+
]
|
266
|
+
end
|
264
267
|
|
265
268
|
context 'when target data exists' do
|
266
269
|
before do
|
@@ -269,7 +272,8 @@ describe Masamune::DataPlan::Engine do
|
|
269
272
|
execute
|
270
273
|
end
|
271
274
|
|
272
|
-
it 'should not call touch!' do
|
275
|
+
it 'should not call touch!' do
|
276
|
+
end
|
273
277
|
end
|
274
278
|
|
275
279
|
context 'when partial target data exists' do
|
@@ -280,7 +284,8 @@ describe Masamune::DataPlan::Engine do
|
|
280
284
|
execute
|
281
285
|
end
|
282
286
|
|
283
|
-
it 'should call touch!' do
|
287
|
+
it 'should call touch!' do
|
288
|
+
end
|
284
289
|
end
|
285
290
|
|
286
291
|
context 'when source data does not exist' do
|
@@ -289,13 +294,14 @@ describe Masamune::DataPlan::Engine do
|
|
289
294
|
execute
|
290
295
|
end
|
291
296
|
|
292
|
-
it 'should not call touch!' do
|
297
|
+
it 'should not call touch!' do
|
298
|
+
end
|
293
299
|
end
|
294
300
|
end
|
295
301
|
|
296
302
|
shared_examples_for 'derived daily data' do
|
297
303
|
context 'when primary target data exists' do
|
298
|
-
let(:derived_targets) {
|
304
|
+
let(:derived_targets) { ['/table/y=2013/m=01/d=01/DATA', '/table/y=2013/m=01/d=02/DATA', '/table/y=2013/m=01/d=03/DATA'] }
|
299
305
|
|
300
306
|
before do
|
301
307
|
engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
|
@@ -304,11 +310,12 @@ describe Masamune::DataPlan::Engine do
|
|
304
310
|
execute
|
305
311
|
end
|
306
312
|
|
307
|
-
it 'should call touch!' do
|
313
|
+
it 'should call touch!' do
|
314
|
+
end
|
308
315
|
end
|
309
316
|
|
310
317
|
context 'when primary target data exists and :resolve is false' do
|
311
|
-
let(:options) { {resolve: false} }
|
318
|
+
let(:options) { { resolve: false } }
|
312
319
|
|
313
320
|
before do
|
314
321
|
engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
|
@@ -316,7 +323,8 @@ describe Masamune::DataPlan::Engine do
|
|
316
323
|
execute
|
317
324
|
end
|
318
325
|
|
319
|
-
it 'should not call touch!' do
|
326
|
+
it 'should not call touch!' do
|
327
|
+
end
|
320
328
|
end
|
321
329
|
end
|
322
330
|
|
@@ -331,7 +339,7 @@ describe Masamune::DataPlan::Engine do
|
|
331
339
|
|
332
340
|
context 'derived_monthly rule' do
|
333
341
|
let(:rule) { 'derived_monthly' }
|
334
|
-
let(:targets) {
|
342
|
+
let(:targets) { ['/monthly/2013-01/DATA'] }
|
335
343
|
|
336
344
|
it_behaves_like 'derived daily data' do
|
337
345
|
let(:derived_command) { derived_monthly_command }
|
@@ -348,7 +356,7 @@ describe Masamune::DataPlan::Engine do
|
|
348
356
|
end
|
349
357
|
|
350
358
|
it 'should raise exception' do
|
351
|
-
expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error
|
359
|
+
expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error(/Max depth .* exceeded for rule 'derived'/)
|
352
360
|
end
|
353
361
|
end
|
354
362
|
end
|
@@ -25,7 +25,7 @@ describe Masamune::DataPlan::Rule do
|
|
25
25
|
let(:name) { 'primary' }
|
26
26
|
let(:type) { :target }
|
27
27
|
let(:pattern) { 'report/%Y-%m-%d/%H' }
|
28
|
-
let(:options) { {path: pattern} }
|
28
|
+
let(:options) { { path: pattern } }
|
29
29
|
|
30
30
|
let(:instance) { described_class.new(engine, name, type, options) }
|
31
31
|
|
@@ -40,7 +40,7 @@ describe Masamune::DataPlan::Rule do
|
|
40
40
|
end
|
41
41
|
|
42
42
|
context 'with lambda' do
|
43
|
-
let(:pattern) {
|
43
|
+
let(:pattern) { ->(_) { 'report/%Y-%m-%d/%H' } }
|
44
44
|
it { is_expected.to eq('report/%Y-%m-%d/%H') }
|
45
45
|
end
|
46
46
|
end
|
@@ -59,7 +59,7 @@ describe Masamune::DataPlan::Rule do
|
|
59
59
|
end
|
60
60
|
|
61
61
|
context 'with DateTime input' do
|
62
|
-
let(:input) { DateTime.civil(2013,04,05,23,13) }
|
62
|
+
let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
|
63
63
|
|
64
64
|
describe '#path' do
|
65
65
|
subject { elem.path }
|
@@ -69,7 +69,7 @@ describe Masamune::DataPlan::Rule do
|
|
69
69
|
|
70
70
|
context 'with DateTime input and unix timestamp pattern' do
|
71
71
|
let(:pattern) { 'logs/%H-s.log' }
|
72
|
-
let(:input) { DateTime.civil(2013,04,05,23,13) }
|
72
|
+
let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
|
73
73
|
|
74
74
|
describe '#path' do
|
75
75
|
subject { elem.path }
|
@@ -78,7 +78,7 @@ describe Masamune::DataPlan::Rule do
|
|
78
78
|
end
|
79
79
|
|
80
80
|
context 'with Date input' do
|
81
|
-
let(:input) { Date.civil(2013,04,05) }
|
81
|
+
let(:input) { Date.civil(2013, 04, 05) }
|
82
82
|
|
83
83
|
describe '#path' do
|
84
84
|
subject { elem.path }
|
@@ -92,7 +92,7 @@ describe Masamune::DataPlan::Rule do
|
|
92
92
|
|
93
93
|
context 'with default' do
|
94
94
|
let(:input) { 'report/2013-04-05/23' }
|
95
|
-
let(:output_date) { DateTime.civil(2013,04,05,23) }
|
95
|
+
let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
|
96
96
|
|
97
97
|
describe '#path' do
|
98
98
|
subject { elem.path }
|
@@ -113,7 +113,7 @@ describe Masamune::DataPlan::Rule do
|
|
113
113
|
context 'with unix timestamp pattern' do
|
114
114
|
let(:pattern) { 'logs/%H-s.log' }
|
115
115
|
let(:input) { 'logs/1365202800.log' }
|
116
|
-
let(:output_date) { DateTime.civil(2013,04,05,23) }
|
116
|
+
let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
|
117
117
|
|
118
118
|
describe '#path' do
|
119
119
|
subject { elem.path }
|
@@ -140,7 +140,7 @@ describe Masamune::DataPlan::Rule do
|
|
140
140
|
context 'with wildcard pattern' do
|
141
141
|
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
|
142
142
|
let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
|
143
|
-
let(:output_date) { DateTime.civil(2013,04,30,20) }
|
143
|
+
let(:output_date) { DateTime.civil(2013, 04, 30, 20) }
|
144
144
|
|
145
145
|
describe '#path' do
|
146
146
|
subject { elem.path }
|
@@ -160,8 +160,8 @@ describe Masamune::DataPlan::Rule do
|
|
160
160
|
end
|
161
161
|
|
162
162
|
describe '#unify' do
|
163
|
-
let(:primary) { described_class.new(engine, name, type,
|
164
|
-
let(:induced) { described_class.new(engine, name, type,
|
163
|
+
let(:primary) { described_class.new(engine, name, type, path: 'report/%Y-%m-%d/%H') }
|
164
|
+
let(:induced) { described_class.new(engine, name, type, path: 'table/y=%Y/m=%m/d=%d/h=%H') }
|
165
165
|
let(:elem) { primary.bind_input(input) }
|
166
166
|
|
167
167
|
subject(:new_elem) { instance.unify(elem, induced) }
|
@@ -176,7 +176,7 @@ describe Masamune::DataPlan::Rule do
|
|
176
176
|
end
|
177
177
|
|
178
178
|
context 'when input partially matches basis pattern' do
|
179
|
-
let(:induced) { described_class.new(engine, name, type,
|
179
|
+
let(:induced) { described_class.new(engine, name, type, path: 'table/%Y-%m') }
|
180
180
|
|
181
181
|
let(:input) { 'report/2013-01-02/00' }
|
182
182
|
|
@@ -240,14 +240,14 @@ describe Masamune::DataPlan::Rule do
|
|
240
240
|
|
241
241
|
describe '#generate' do
|
242
242
|
context 'with a block' do
|
243
|
-
let(:start_date) { DateTime.civil(2013,04,05,20) }
|
244
|
-
let(:stop_date) { DateTime.civil(2013,04,05,20) }
|
243
|
+
let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
|
244
|
+
let(:stop_date) { DateTime.civil(2013, 04, 05, 20) }
|
245
245
|
specify { expect { |b| instance.generate(start_date, stop_date, &b) }.to yield_control }
|
246
246
|
end
|
247
247
|
|
248
248
|
context 'without a block' do
|
249
|
-
let(:start_date) { DateTime.civil(2013,04,05,20) }
|
250
|
-
let(:stop_date) { DateTime.civil(2013,04,05,22) }
|
249
|
+
let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
|
250
|
+
let(:stop_date) { DateTime.civil(2013, 04, 05, 22) }
|
251
251
|
|
252
252
|
subject(:elems) do
|
253
253
|
instance.generate(start_date, stop_date)
|
@@ -307,7 +307,7 @@ describe Masamune::DataPlan::Rule do
|
|
307
307
|
end
|
308
308
|
|
309
309
|
describe '#time_round' do
|
310
|
-
let(:input_time) { DateTime.civil(2013,9,13,23,13) }
|
310
|
+
let(:input_time) { DateTime.civil(2013, 9, 13, 23, 13) }
|
311
311
|
subject { instance.time_round(input_time) }
|
312
312
|
|
313
313
|
before do
|
@@ -316,15 +316,15 @@ describe Masamune::DataPlan::Rule do
|
|
316
316
|
|
317
317
|
context 'hourly' do
|
318
318
|
let(:time_step) { :hours }
|
319
|
-
it { is_expected.to eq(DateTime.civil(2013,9,13,23)) }
|
319
|
+
it { is_expected.to eq(DateTime.civil(2013, 9, 13, 23)) }
|
320
320
|
end
|
321
321
|
context 'daily' do
|
322
322
|
let(:time_step) { :days }
|
323
|
-
it { is_expected.to eq(DateTime.civil(2013,9,13)) }
|
323
|
+
it { is_expected.to eq(DateTime.civil(2013, 9, 13)) }
|
324
324
|
end
|
325
325
|
context 'monthly' do
|
326
326
|
let(:time_step) { :months }
|
327
|
-
it { is_expected.to eq(DateTime.civil(2013,9)) }
|
327
|
+
it { is_expected.to eq(DateTime.civil(2013, 9)) }
|
328
328
|
end
|
329
329
|
context 'yearly' do
|
330
330
|
let(:time_step) { :years }
|