masamune 0.17.12 → 0.17.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -27,27 +27,28 @@ describe Masamune::DataPlan::Builder do
|
|
27
27
|
end
|
28
28
|
|
29
29
|
context 'with multiple namespaces' do
|
30
|
-
let(:namespaces) {
|
31
|
-
let(:commands) {
|
30
|
+
let(:namespaces) { %w(a b) }
|
31
|
+
let(:commands) { %w(load store) }
|
32
32
|
let(:sources) { [{ path: 'log/%Y%m%d.*.log' }, { path: 'table/y=%Y/m=%m/d=%d' }] }
|
33
33
|
let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d' }, { path: 'daily/%Y-%m-%d' }] }
|
34
34
|
|
35
35
|
before do
|
36
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load',
|
37
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load',
|
36
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:load', path: 'table/y=%Y/m=%m/d=%d')
|
37
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:load', path: 'log/%Y%m%d.*.log')
|
38
38
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:load', an_instance_of(Proc))
|
39
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store',
|
40
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store',
|
39
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('b:store', path: 'daily/%Y-%m-%d')
|
40
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('b:store', path: 'table/y=%Y/m=%m/d=%d')
|
41
41
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('b:store', an_instance_of(Proc))
|
42
42
|
subject
|
43
43
|
end
|
44
44
|
|
45
|
-
it 'should build a Masamune::DataPlan::Engine instance' do
|
45
|
+
it 'should build a Masamune::DataPlan::Engine instance' do
|
46
|
+
end
|
46
47
|
end
|
47
48
|
|
48
49
|
context 'with :for option' do
|
49
|
-
let(:namespaces) {
|
50
|
-
let(:commands) {
|
50
|
+
let(:namespaces) { %w(a a a) }
|
51
|
+
let(:commands) { %w(missing_before override missing_after) }
|
51
52
|
let(:sources) { [{ path: 'log/%Y%m%d.*.log', for: 'override' }] }
|
52
53
|
let(:targets) { [{ path: 'table/y=%Y/m=%m/d=%d', for: 'override' }] }
|
53
54
|
|
@@ -55,8 +56,8 @@ describe Masamune::DataPlan::Builder do
|
|
55
56
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
|
56
57
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
|
57
58
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
|
58
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override',
|
59
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override',
|
59
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
|
60
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
|
60
61
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
|
61
62
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_after', anything)
|
62
63
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_after', anything)
|
@@ -64,26 +65,28 @@ describe Masamune::DataPlan::Builder do
|
|
64
65
|
subject
|
65
66
|
end
|
66
67
|
|
67
|
-
it 'should build a Masamune::DataPlan::Engine instance' do
|
68
|
+
it 'should build a Masamune::DataPlan::Engine instance' do
|
69
|
+
end
|
68
70
|
end
|
69
71
|
|
70
72
|
context 'with :skip option' do
|
71
|
-
let(:namespaces) {
|
72
|
-
let(:commands) {
|
73
|
-
let(:sources) { [{ skip: true}, { path: 'log/%Y%m%d.*.log' }] }
|
74
|
-
let(:targets) { [{ skip: true}, { path: 'table/y=%Y/m=%m/d=%d' }] }
|
73
|
+
let(:namespaces) { %w(a a a) }
|
74
|
+
let(:commands) { %w(missing_before override) }
|
75
|
+
let(:sources) { [{ skip: true }, { path: 'log/%Y%m%d.*.log' }] }
|
76
|
+
let(:targets) { [{ skip: true }, { path: 'table/y=%Y/m=%m/d=%d' }] }
|
75
77
|
|
76
78
|
before do
|
77
79
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_target_rule).with('a:missing_before', anything)
|
78
80
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_source_rule).with('a:missing_before', anything)
|
79
81
|
expect_any_instance_of(Masamune::DataPlan::Engine).not_to receive(:add_command_rule).with('a:missing_before', an_instance_of(Proc))
|
80
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override',
|
81
|
-
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override',
|
82
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_target_rule).with('a:override', path: 'table/y=%Y/m=%m/d=%d')
|
83
|
+
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_source_rule).with('a:override', path: 'log/%Y%m%d.*.log')
|
82
84
|
expect_any_instance_of(Masamune::DataPlan::Engine).to receive(:add_command_rule).with('a:override', an_instance_of(Proc))
|
83
85
|
subject
|
84
86
|
end
|
85
87
|
|
86
|
-
it 'should build a Masamune::DataPlan::Engine instance' do
|
88
|
+
it 'should build a Masamune::DataPlan::Engine instance' do
|
89
|
+
end
|
87
90
|
end
|
88
91
|
end
|
89
92
|
end
|
@@ -32,14 +32,14 @@ describe Masamune::DataPlan::Elem do
|
|
32
32
|
|
33
33
|
let(:name) { 'primary' }
|
34
34
|
let(:type) { :target }
|
35
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
36
|
-
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
35
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
|
36
|
+
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/log/%Y%m%d.*.log') }
|
37
37
|
|
38
|
-
let(:start_time) { DateTime.civil(2013,07,19,11,07) }
|
39
|
-
let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
|
38
|
+
let(:start_time) { DateTime.civil(2013, 07, 19, 11, 07) }
|
39
|
+
let(:other_start_time) { DateTime.civil(2013, 07, 20, 0, 0) }
|
40
40
|
|
41
|
-
let(:options) { {tz: 'EST'} }
|
42
|
-
let(:other_options) { {tz: 'PST'} }
|
41
|
+
let(:options) { { tz: 'EST' } }
|
42
|
+
let(:other_options) { { tz: 'PST' } }
|
43
43
|
|
44
44
|
let(:instance) { described_class.new(rule, start_time, options) }
|
45
45
|
|
@@ -80,8 +80,8 @@ describe Masamune::DataPlan::Elem do
|
|
80
80
|
end
|
81
81
|
|
82
82
|
describe '#last_modified_at' do
|
83
|
-
let(:early) { Time.parse(
|
84
|
-
let(:later) { Time.parse(
|
83
|
+
let(:early) { Time.parse('2014-05-01 00:00:00 +0000') }
|
84
|
+
let(:later) { Time.parse('2014-06-01 00:00:00 +0000') }
|
85
85
|
|
86
86
|
subject do
|
87
87
|
instance.last_modified_at.utc
|
@@ -89,8 +89,8 @@ describe Masamune::DataPlan::Elem do
|
|
89
89
|
|
90
90
|
context 'with missing mtime' do
|
91
91
|
before do
|
92
|
-
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
93
|
-
and_return(nil)
|
92
|
+
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
93
|
+
.and_return(nil)
|
94
94
|
end
|
95
95
|
|
96
96
|
it { is_expected.to eq(Masamune::DataPlan::Elem::MISSING_MODIFIED_AT) }
|
@@ -98,8 +98,8 @@ describe Masamune::DataPlan::Elem do
|
|
98
98
|
|
99
99
|
context 'with single mtime' do
|
100
100
|
before do
|
101
|
-
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
102
|
-
and_return(OpenStruct.new(mtime: early))
|
101
|
+
expect(rule.engine.filesystem).to receive(:stat).with(instance.path)
|
102
|
+
.and_return(OpenStruct.new(mtime: early))
|
103
103
|
end
|
104
104
|
|
105
105
|
it { is_expected.to eq(early) }
|
@@ -110,7 +110,7 @@ describe Masamune::DataPlan::Elem do
|
|
110
110
|
subject { instance.explode.map(&:path) }
|
111
111
|
|
112
112
|
context 'with free path and existing files' do
|
113
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
113
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
|
114
114
|
before do
|
115
115
|
engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
|
116
116
|
end
|
@@ -118,12 +118,12 @@ describe Masamune::DataPlan::Elem do
|
|
118
118
|
end
|
119
119
|
|
120
120
|
context 'with free path and missing files' do
|
121
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
121
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/%Y-%m-%d/%H') }
|
122
122
|
it { is_expected.to be_empty }
|
123
123
|
end
|
124
124
|
|
125
125
|
context 'with bound path and existing files' do
|
126
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
126
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
|
127
127
|
before do
|
128
128
|
engine.filesystem.touch!('/report/file')
|
129
129
|
end
|
@@ -131,12 +131,12 @@ describe Masamune::DataPlan::Elem do
|
|
131
131
|
end
|
132
132
|
|
133
133
|
context 'with bound path and missing files' do
|
134
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
134
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, path: '/report/file') }
|
135
135
|
it { is_expected.to be_empty }
|
136
136
|
end
|
137
137
|
|
138
138
|
context 'with free table and existing table' do
|
139
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
139
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
|
140
140
|
before do
|
141
141
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
142
142
|
end
|
@@ -144,7 +144,7 @@ describe Masamune::DataPlan::Elem do
|
|
144
144
|
end
|
145
145
|
|
146
146
|
context 'with free table and missing table' do
|
147
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
147
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact', partition: 'y%Y%m') }
|
148
148
|
before do
|
149
149
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
150
150
|
end
|
@@ -152,7 +152,7 @@ describe Masamune::DataPlan::Elem do
|
|
152
152
|
end
|
153
153
|
|
154
154
|
context 'with bound table and existing table' do
|
155
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
155
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
|
156
156
|
before do
|
157
157
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
158
158
|
end
|
@@ -160,7 +160,7 @@ describe Masamune::DataPlan::Elem do
|
|
160
160
|
end
|
161
161
|
|
162
162
|
context 'with bound table and missing table' do
|
163
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type,
|
163
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, table: 'visits_fact') }
|
164
164
|
before do
|
165
165
|
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
166
166
|
end
|
@@ -31,7 +31,7 @@ describe Masamune::DataPlan::Engine do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
let(:command) do
|
34
|
-
|
34
|
+
proc do |engine, rule|
|
35
35
|
missing_targets = []
|
36
36
|
engine.targets(rule).missing.each do |target|
|
37
37
|
missing_targets << target.path if target.sources.existing.any?
|
@@ -43,7 +43,7 @@ describe Masamune::DataPlan::Engine do
|
|
43
43
|
before do
|
44
44
|
engine.add_target_rule('non_primary', path: '/table/y=%Y/m=%m/d=%d', primary: false)
|
45
45
|
engine.add_source_rule('non_primary', path: '/log/%Y%m%d.*.log', primary: false)
|
46
|
-
engine.add_command_rule('non_primary', ->(*_) {
|
46
|
+
engine.add_command_rule('non_primary', ->(*_) { raise })
|
47
47
|
engine.add_target_rule('primary', path: '/table/y=%Y/m=%m/d=%d')
|
48
48
|
engine.add_source_rule('primary', path: '/log/%Y%m%d.*.log')
|
49
49
|
engine.add_command_rule('primary', command)
|
@@ -61,8 +61,8 @@ describe Masamune::DataPlan::Engine do
|
|
61
61
|
end
|
62
62
|
|
63
63
|
describe '#targets_for_date_range' do
|
64
|
-
let(:start) { Date.civil(2013,01,01) }
|
65
|
-
let(:stop) { Date.civil(2013,01,03) }
|
64
|
+
let(:start) { Date.civil(2013, 01, 01) }
|
65
|
+
let(:stop) { Date.civil(2013, 01, 03) }
|
66
66
|
|
67
67
|
subject { engine.targets_for_date_range(rule, start, stop).map(&:path) }
|
68
68
|
|
@@ -104,26 +104,26 @@ describe Masamune::DataPlan::Engine do
|
|
104
104
|
let(:rule) { 'primary' }
|
105
105
|
let(:source) { '/log/20130101.random.log' }
|
106
106
|
|
107
|
-
it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
|
108
|
-
it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
|
107
|
+
it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
|
108
|
+
it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
|
109
109
|
it { expect(targets.first.path).to eq('/table/y=2013/m=01/d=01') }
|
110
110
|
end
|
111
111
|
|
112
112
|
context 'derived_daily' do
|
113
113
|
let(:rule) { 'derived_daily' }
|
114
|
-
let(:source)
|
114
|
+
let(:source) { '/table/y=2013/m=01/d=01' }
|
115
115
|
|
116
|
-
it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
|
117
|
-
it { expect(targets.first.stop_time).to eq(Date.civil(2013,01,02)) }
|
116
|
+
it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
|
117
|
+
it { expect(targets.first.stop_time).to eq(Date.civil(2013, 01, 02)) }
|
118
118
|
it { expect(targets.first.path).to eq('/daily/2013-01-01') }
|
119
119
|
end
|
120
120
|
|
121
121
|
context 'derived_monthly' do
|
122
122
|
let(:rule) { 'derived_monthly' }
|
123
|
-
let(:source)
|
123
|
+
let(:source) { '/table/y=2013/m=01/d=01' }
|
124
124
|
|
125
|
-
it { expect(targets.first.start_time).to eq(Date.civil(2013,01,01)) }
|
126
|
-
it { expect(targets.first.stop_time).to eq(Date.civil(2013,02,01)) }
|
125
|
+
it { expect(targets.first.start_time).to eq(Date.civil(2013, 01, 01)) }
|
126
|
+
it { expect(targets.first.stop_time).to eq(Date.civil(2013, 02, 01)) }
|
127
127
|
it { expect(targets.first.path).to eq('/monthly/2013-01') }
|
128
128
|
end
|
129
129
|
end
|
@@ -167,15 +167,15 @@ describe Masamune::DataPlan::Engine do
|
|
167
167
|
let(:target) { '/monthly/2013-01' }
|
168
168
|
|
169
169
|
(1..31).each do |day|
|
170
|
-
it { expect(sources).to include '/table/y=2013/m=01/d=%02d'
|
170
|
+
it { expect(sources).to include format('/table/y=2013/m=01/d=%02d', day) }
|
171
171
|
end
|
172
172
|
it { expect(sources.size).to eq(31) }
|
173
173
|
end
|
174
174
|
|
175
175
|
context 'invalid target' do
|
176
176
|
let(:rule) { 'derived_daily' }
|
177
|
-
let(:target) {
|
178
|
-
it { expect { subject }.to raise_error
|
177
|
+
let(:target) { '/table/y=2013/m=01/d=01' }
|
178
|
+
it { expect { subject }.to raise_error(/Cannot bind_input/) }
|
179
179
|
end
|
180
180
|
end
|
181
181
|
|
@@ -204,7 +204,7 @@ describe Masamune::DataPlan::Engine do
|
|
204
204
|
|
205
205
|
context 'invalid target' do
|
206
206
|
let(:target) { '/daily' }
|
207
|
-
it { expect { subject }.to raise_error
|
207
|
+
it { expect { subject }.to raise_error(/No rule matches/) }
|
208
208
|
end
|
209
209
|
end
|
210
210
|
|
@@ -224,7 +224,7 @@ describe Masamune::DataPlan::Engine do
|
|
224
224
|
context 'with :targets' do
|
225
225
|
let(:rule) { 'primary' }
|
226
226
|
|
227
|
-
let(:options) { {targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
|
227
|
+
let(:options) { { targets: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
|
228
228
|
|
229
229
|
it { expect(targets).to include '/table/y=2013/m=01/d=01' }
|
230
230
|
it { expect(targets).to include '/table/y=2013/m=01/d=02' }
|
@@ -235,10 +235,10 @@ describe Masamune::DataPlan::Engine do
|
|
235
235
|
context 'with :sources' do
|
236
236
|
let(:rule) { 'derived_daily' }
|
237
237
|
|
238
|
-
let(:options) { {sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02']} }
|
238
|
+
let(:options) { { sources: ['/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02', '/table/y=2013/m=01/d=02'] } }
|
239
239
|
|
240
|
-
it { expect(targets).to include
|
241
|
-
it { expect(targets).to include
|
240
|
+
it { expect(targets).to include '/daily/2013-01-01' }
|
241
|
+
it { expect(targets).to include '/daily/2013-01-02' }
|
242
242
|
it { expect(sources).to include '/table/y=2013/m=01/d=01' }
|
243
243
|
it { expect(sources).to include '/table/y=2013/m=01/d=02' }
|
244
244
|
end
|
@@ -257,10 +257,13 @@ describe Masamune::DataPlan::Engine do
|
|
257
257
|
|
258
258
|
context 'primary rule' do
|
259
259
|
let(:rule) { 'primary' }
|
260
|
-
let(:targets)
|
261
|
-
|
262
|
-
|
263
|
-
|
260
|
+
let(:targets) do
|
261
|
+
[
|
262
|
+
'/table/y=2013/m=01/d=01',
|
263
|
+
'/table/y=2013/m=01/d=02',
|
264
|
+
'/table/y=2013/m=01/d=03'
|
265
|
+
]
|
266
|
+
end
|
264
267
|
|
265
268
|
context 'when target data exists' do
|
266
269
|
before do
|
@@ -269,7 +272,8 @@ describe Masamune::DataPlan::Engine do
|
|
269
272
|
execute
|
270
273
|
end
|
271
274
|
|
272
|
-
it 'should not call touch!' do
|
275
|
+
it 'should not call touch!' do
|
276
|
+
end
|
273
277
|
end
|
274
278
|
|
275
279
|
context 'when partial target data exists' do
|
@@ -280,7 +284,8 @@ describe Masamune::DataPlan::Engine do
|
|
280
284
|
execute
|
281
285
|
end
|
282
286
|
|
283
|
-
it 'should call touch!' do
|
287
|
+
it 'should call touch!' do
|
288
|
+
end
|
284
289
|
end
|
285
290
|
|
286
291
|
context 'when source data does not exist' do
|
@@ -289,13 +294,14 @@ describe Masamune::DataPlan::Engine do
|
|
289
294
|
execute
|
290
295
|
end
|
291
296
|
|
292
|
-
it 'should not call touch!' do
|
297
|
+
it 'should not call touch!' do
|
298
|
+
end
|
293
299
|
end
|
294
300
|
end
|
295
301
|
|
296
302
|
shared_examples_for 'derived daily data' do
|
297
303
|
context 'when primary target data exists' do
|
298
|
-
let(:derived_targets) {
|
304
|
+
let(:derived_targets) { ['/table/y=2013/m=01/d=01/DATA', '/table/y=2013/m=01/d=02/DATA', '/table/y=2013/m=01/d=03/DATA'] }
|
299
305
|
|
300
306
|
before do
|
301
307
|
engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
|
@@ -304,11 +310,12 @@ describe Masamune::DataPlan::Engine do
|
|
304
310
|
execute
|
305
311
|
end
|
306
312
|
|
307
|
-
it 'should call touch!' do
|
313
|
+
it 'should call touch!' do
|
314
|
+
end
|
308
315
|
end
|
309
316
|
|
310
317
|
context 'when primary target data exists and :resolve is false' do
|
311
|
-
let(:options) { {resolve: false} }
|
318
|
+
let(:options) { { resolve: false } }
|
312
319
|
|
313
320
|
before do
|
314
321
|
engine.filesystem.touch!('/log/20130101.app1.log', '/log/20130102.app1.log', '/log/20130103.app1.log')
|
@@ -316,7 +323,8 @@ describe Masamune::DataPlan::Engine do
|
|
316
323
|
execute
|
317
324
|
end
|
318
325
|
|
319
|
-
it 'should not call touch!' do
|
326
|
+
it 'should not call touch!' do
|
327
|
+
end
|
320
328
|
end
|
321
329
|
end
|
322
330
|
|
@@ -331,7 +339,7 @@ describe Masamune::DataPlan::Engine do
|
|
331
339
|
|
332
340
|
context 'derived_monthly rule' do
|
333
341
|
let(:rule) { 'derived_monthly' }
|
334
|
-
let(:targets) {
|
342
|
+
let(:targets) { ['/monthly/2013-01/DATA'] }
|
335
343
|
|
336
344
|
it_behaves_like 'derived daily data' do
|
337
345
|
let(:derived_command) { derived_monthly_command }
|
@@ -348,7 +356,7 @@ describe Masamune::DataPlan::Engine do
|
|
348
356
|
end
|
349
357
|
|
350
358
|
it 'should raise exception' do
|
351
|
-
expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error
|
359
|
+
expect { engine.prepare('derived', targets: ['/log/20140228.wtf.log']) }.to raise_error(/Max depth .* exceeded for rule 'derived'/)
|
352
360
|
end
|
353
361
|
end
|
354
362
|
end
|
@@ -25,7 +25,7 @@ describe Masamune::DataPlan::Rule do
|
|
25
25
|
let(:name) { 'primary' }
|
26
26
|
let(:type) { :target }
|
27
27
|
let(:pattern) { 'report/%Y-%m-%d/%H' }
|
28
|
-
let(:options) { {path: pattern} }
|
28
|
+
let(:options) { { path: pattern } }
|
29
29
|
|
30
30
|
let(:instance) { described_class.new(engine, name, type, options) }
|
31
31
|
|
@@ -40,7 +40,7 @@ describe Masamune::DataPlan::Rule do
|
|
40
40
|
end
|
41
41
|
|
42
42
|
context 'with lambda' do
|
43
|
-
let(:pattern) {
|
43
|
+
let(:pattern) { ->(_) { 'report/%Y-%m-%d/%H' } }
|
44
44
|
it { is_expected.to eq('report/%Y-%m-%d/%H') }
|
45
45
|
end
|
46
46
|
end
|
@@ -59,7 +59,7 @@ describe Masamune::DataPlan::Rule do
|
|
59
59
|
end
|
60
60
|
|
61
61
|
context 'with DateTime input' do
|
62
|
-
let(:input) { DateTime.civil(2013,04,05,23,13) }
|
62
|
+
let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
|
63
63
|
|
64
64
|
describe '#path' do
|
65
65
|
subject { elem.path }
|
@@ -69,7 +69,7 @@ describe Masamune::DataPlan::Rule do
|
|
69
69
|
|
70
70
|
context 'with DateTime input and unix timestamp pattern' do
|
71
71
|
let(:pattern) { 'logs/%H-s.log' }
|
72
|
-
let(:input) { DateTime.civil(2013,04,05,23,13) }
|
72
|
+
let(:input) { DateTime.civil(2013, 04, 05, 23, 13) }
|
73
73
|
|
74
74
|
describe '#path' do
|
75
75
|
subject { elem.path }
|
@@ -78,7 +78,7 @@ describe Masamune::DataPlan::Rule do
|
|
78
78
|
end
|
79
79
|
|
80
80
|
context 'with Date input' do
|
81
|
-
let(:input) { Date.civil(2013,04,05) }
|
81
|
+
let(:input) { Date.civil(2013, 04, 05) }
|
82
82
|
|
83
83
|
describe '#path' do
|
84
84
|
subject { elem.path }
|
@@ -92,7 +92,7 @@ describe Masamune::DataPlan::Rule do
|
|
92
92
|
|
93
93
|
context 'with default' do
|
94
94
|
let(:input) { 'report/2013-04-05/23' }
|
95
|
-
let(:output_date) { DateTime.civil(2013,04,05,23) }
|
95
|
+
let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
|
96
96
|
|
97
97
|
describe '#path' do
|
98
98
|
subject { elem.path }
|
@@ -113,7 +113,7 @@ describe Masamune::DataPlan::Rule do
|
|
113
113
|
context 'with unix timestamp pattern' do
|
114
114
|
let(:pattern) { 'logs/%H-s.log' }
|
115
115
|
let(:input) { 'logs/1365202800.log' }
|
116
|
-
let(:output_date) { DateTime.civil(2013,04,05,23) }
|
116
|
+
let(:output_date) { DateTime.civil(2013, 04, 05, 23) }
|
117
117
|
|
118
118
|
describe '#path' do
|
119
119
|
subject { elem.path }
|
@@ -140,7 +140,7 @@ describe Masamune::DataPlan::Rule do
|
|
140
140
|
context 'with wildcard pattern' do
|
141
141
|
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
|
142
142
|
let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
|
143
|
-
let(:output_date) { DateTime.civil(2013,04,30,20) }
|
143
|
+
let(:output_date) { DateTime.civil(2013, 04, 30, 20) }
|
144
144
|
|
145
145
|
describe '#path' do
|
146
146
|
subject { elem.path }
|
@@ -160,8 +160,8 @@ describe Masamune::DataPlan::Rule do
|
|
160
160
|
end
|
161
161
|
|
162
162
|
describe '#unify' do
|
163
|
-
let(:primary) { described_class.new(engine, name, type,
|
164
|
-
let(:induced) { described_class.new(engine, name, type,
|
163
|
+
let(:primary) { described_class.new(engine, name, type, path: 'report/%Y-%m-%d/%H') }
|
164
|
+
let(:induced) { described_class.new(engine, name, type, path: 'table/y=%Y/m=%m/d=%d/h=%H') }
|
165
165
|
let(:elem) { primary.bind_input(input) }
|
166
166
|
|
167
167
|
subject(:new_elem) { instance.unify(elem, induced) }
|
@@ -176,7 +176,7 @@ describe Masamune::DataPlan::Rule do
|
|
176
176
|
end
|
177
177
|
|
178
178
|
context 'when input partially matches basis pattern' do
|
179
|
-
let(:induced) { described_class.new(engine, name, type,
|
179
|
+
let(:induced) { described_class.new(engine, name, type, path: 'table/%Y-%m') }
|
180
180
|
|
181
181
|
let(:input) { 'report/2013-01-02/00' }
|
182
182
|
|
@@ -240,14 +240,14 @@ describe Masamune::DataPlan::Rule do
|
|
240
240
|
|
241
241
|
describe '#generate' do
|
242
242
|
context 'with a block' do
|
243
|
-
let(:start_date) { DateTime.civil(2013,04,05,20) }
|
244
|
-
let(:stop_date) { DateTime.civil(2013,04,05,20) }
|
243
|
+
let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
|
244
|
+
let(:stop_date) { DateTime.civil(2013, 04, 05, 20) }
|
245
245
|
specify { expect { |b| instance.generate(start_date, stop_date, &b) }.to yield_control }
|
246
246
|
end
|
247
247
|
|
248
248
|
context 'without a block' do
|
249
|
-
let(:start_date) { DateTime.civil(2013,04,05,20) }
|
250
|
-
let(:stop_date) { DateTime.civil(2013,04,05,22) }
|
249
|
+
let(:start_date) { DateTime.civil(2013, 04, 05, 20) }
|
250
|
+
let(:stop_date) { DateTime.civil(2013, 04, 05, 22) }
|
251
251
|
|
252
252
|
subject(:elems) do
|
253
253
|
instance.generate(start_date, stop_date)
|
@@ -307,7 +307,7 @@ describe Masamune::DataPlan::Rule do
|
|
307
307
|
end
|
308
308
|
|
309
309
|
describe '#time_round' do
|
310
|
-
let(:input_time) { DateTime.civil(2013,9,13,23,13) }
|
310
|
+
let(:input_time) { DateTime.civil(2013, 9, 13, 23, 13) }
|
311
311
|
subject { instance.time_round(input_time) }
|
312
312
|
|
313
313
|
before do
|
@@ -316,15 +316,15 @@ describe Masamune::DataPlan::Rule do
|
|
316
316
|
|
317
317
|
context 'hourly' do
|
318
318
|
let(:time_step) { :hours }
|
319
|
-
it { is_expected.to eq(DateTime.civil(2013,9,13,23)) }
|
319
|
+
it { is_expected.to eq(DateTime.civil(2013, 9, 13, 23)) }
|
320
320
|
end
|
321
321
|
context 'daily' do
|
322
322
|
let(:time_step) { :days }
|
323
|
-
it { is_expected.to eq(DateTime.civil(2013,9,13)) }
|
323
|
+
it { is_expected.to eq(DateTime.civil(2013, 9, 13)) }
|
324
324
|
end
|
325
325
|
context 'monthly' do
|
326
326
|
let(:time_step) { :months }
|
327
|
-
it { is_expected.to eq(DateTime.civil(2013,9)) }
|
327
|
+
it { is_expected.to eq(DateTime.civil(2013, 9)) }
|
328
328
|
end
|
329
329
|
context 'yearly' do
|
330
330
|
let(:time_step) { :years }
|