masamune 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +54 -0
- data/Rakefile +15 -0
- data/bin/masamune-elastic-mapreduce +4 -0
- data/bin/masamune-hive +4 -0
- data/bin/masamune-psql +4 -0
- data/bin/masamune-shell +4 -0
- data/lib/masamune.rb +56 -0
- data/lib/masamune/accumulate.rb +60 -0
- data/lib/masamune/actions.rb +38 -0
- data/lib/masamune/actions/data_flow.rb +131 -0
- data/lib/masamune/actions/date_parse.rb +75 -0
- data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
- data/lib/masamune/actions/execute.rb +52 -0
- data/lib/masamune/actions/filesystem.rb +37 -0
- data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
- data/lib/masamune/actions/hadoop_streaming.rb +41 -0
- data/lib/masamune/actions/hive.rb +74 -0
- data/lib/masamune/actions/postgres.rb +76 -0
- data/lib/masamune/actions/postgres_admin.rb +34 -0
- data/lib/masamune/actions/s3cmd.rb +44 -0
- data/lib/masamune/actions/transform.rb +89 -0
- data/lib/masamune/after_initialize_callbacks.rb +55 -0
- data/lib/masamune/cached_filesystem.rb +110 -0
- data/lib/masamune/commands.rb +37 -0
- data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
- data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
- data/lib/masamune/commands/hadoop_streaming.rb +116 -0
- data/lib/masamune/commands/hive.rb +178 -0
- data/lib/masamune/commands/interactive.rb +37 -0
- data/lib/masamune/commands/postgres.rb +128 -0
- data/lib/masamune/commands/postgres_admin.rb +72 -0
- data/lib/masamune/commands/postgres_common.rb +33 -0
- data/lib/masamune/commands/retry_with_backoff.rb +60 -0
- data/lib/masamune/commands/s3cmd.rb +70 -0
- data/lib/masamune/commands/shell.rb +202 -0
- data/lib/masamune/configuration.rb +195 -0
- data/lib/masamune/data_plan.rb +31 -0
- data/lib/masamune/data_plan/builder.rb +66 -0
- data/lib/masamune/data_plan/elem.rb +190 -0
- data/lib/masamune/data_plan/engine.rb +162 -0
- data/lib/masamune/data_plan/rule.rb +292 -0
- data/lib/masamune/data_plan/set.rb +176 -0
- data/lib/masamune/environment.rb +164 -0
- data/lib/masamune/filesystem.rb +567 -0
- data/lib/masamune/has_environment.rb +40 -0
- data/lib/masamune/helpers.rb +27 -0
- data/lib/masamune/helpers/postgres.rb +84 -0
- data/lib/masamune/io.rb +33 -0
- data/lib/masamune/last_element.rb +53 -0
- data/lib/masamune/method_logger.rb +41 -0
- data/lib/masamune/multi_io.rb +39 -0
- data/lib/masamune/schema.rb +36 -0
- data/lib/masamune/schema/catalog.rb +233 -0
- data/lib/masamune/schema/column.rb +527 -0
- data/lib/masamune/schema/dimension.rb +133 -0
- data/lib/masamune/schema/event.rb +121 -0
- data/lib/masamune/schema/fact.rb +133 -0
- data/lib/masamune/schema/map.rb +265 -0
- data/lib/masamune/schema/row.rb +133 -0
- data/lib/masamune/schema/store.rb +115 -0
- data/lib/masamune/schema/table.rb +308 -0
- data/lib/masamune/schema/table_reference.rb +76 -0
- data/lib/masamune/spec_helper.rb +23 -0
- data/lib/masamune/string_format.rb +34 -0
- data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
- data/lib/masamune/tasks/hive_thor.rb +55 -0
- data/lib/masamune/tasks/postgres_thor.rb +47 -0
- data/lib/masamune/tasks/shell_thor.rb +63 -0
- data/lib/masamune/template.rb +77 -0
- data/lib/masamune/thor.rb +186 -0
- data/lib/masamune/thor_loader.rb +38 -0
- data/lib/masamune/topological_hash.rb +34 -0
- data/lib/masamune/transform.rb +47 -0
- data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
- data/lib/masamune/transform/bulk_upsert.rb +52 -0
- data/lib/masamune/transform/consolidate_dimension.rb +54 -0
- data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
- data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
- data/lib/masamune/transform/define_event_view.hql.erb +51 -0
- data/lib/masamune/transform/define_event_view.rb +60 -0
- data/lib/masamune/transform/define_index.psql.erb +34 -0
- data/lib/masamune/transform/define_schema.hql.erb +23 -0
- data/lib/masamune/transform/define_schema.psql.erb +79 -0
- data/lib/masamune/transform/define_schema.rb +56 -0
- data/lib/masamune/transform/define_table.hql.erb +34 -0
- data/lib/masamune/transform/define_table.psql.erb +95 -0
- data/lib/masamune/transform/define_table.rb +40 -0
- data/lib/masamune/transform/define_unique.psql.erb +30 -0
- data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
- data/lib/masamune/transform/insert_reference_values.rb +64 -0
- data/lib/masamune/transform/load_dimension.rb +47 -0
- data/lib/masamune/transform/load_fact.rb +45 -0
- data/lib/masamune/transform/operator.rb +96 -0
- data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
- data/lib/masamune/transform/relabel_dimension.rb +39 -0
- data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
- data/lib/masamune/transform/rollup_fact.rb +149 -0
- data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
- data/lib/masamune/transform/snapshot_dimension.rb +74 -0
- data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
- data/lib/masamune/transform/stage_dimension.rb +83 -0
- data/lib/masamune/transform/stage_fact.psql.erb +80 -0
- data/lib/masamune/transform/stage_fact.rb +111 -0
- data/lib/masamune/version.rb +25 -0
- data/spec/fixtures/aggregate.sql.erb +25 -0
- data/spec/fixtures/comment.sql.erb +27 -0
- data/spec/fixtures/invalid.sql.erb +23 -0
- data/spec/fixtures/relative.sql.erb +23 -0
- data/spec/fixtures/simple.sql.erb +28 -0
- data/spec/fixtures/whitespace.sql.erb +30 -0
- data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
- data/spec/masamune/actions/execute_spec.rb +50 -0
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
- data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
- data/spec/masamune/actions/hive_spec.rb +117 -0
- data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
- data/spec/masamune/actions/postgres_spec.rb +134 -0
- data/spec/masamune/actions/s3cmd_spec.rb +44 -0
- data/spec/masamune/actions/transform_spec.rb +144 -0
- data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
- data/spec/masamune/cached_filesystem_spec.rb +167 -0
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
- data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
- data/spec/masamune/commands/hive_spec.rb +117 -0
- data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
- data/spec/masamune/commands/postgres_spec.rb +100 -0
- data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
- data/spec/masamune/commands/s3cmd_spec.rb +50 -0
- data/spec/masamune/commands/shell_spec.rb +101 -0
- data/spec/masamune/configuration_spec.rb +102 -0
- data/spec/masamune/data_plan/builder_spec.rb +91 -0
- data/spec/masamune/data_plan/elem_spec.rb +102 -0
- data/spec/masamune/data_plan/engine_spec.rb +356 -0
- data/spec/masamune/data_plan/rule_spec.rb +407 -0
- data/spec/masamune/data_plan/set_spec.rb +517 -0
- data/spec/masamune/environment_spec.rb +65 -0
- data/spec/masamune/filesystem_spec.rb +1421 -0
- data/spec/masamune/helpers/postgres_spec.rb +95 -0
- data/spec/masamune/schema/catalog_spec.rb +613 -0
- data/spec/masamune/schema/column_spec.rb +696 -0
- data/spec/masamune/schema/dimension_spec.rb +137 -0
- data/spec/masamune/schema/event_spec.rb +75 -0
- data/spec/masamune/schema/fact_spec.rb +117 -0
- data/spec/masamune/schema/map_spec.rb +593 -0
- data/spec/masamune/schema/row_spec.rb +28 -0
- data/spec/masamune/schema/store_spec.rb +49 -0
- data/spec/masamune/schema/table_spec.rb +395 -0
- data/spec/masamune/string_format_spec.rb +60 -0
- data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
- data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
- data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
- data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
- data/spec/masamune/template_spec.rb +77 -0
- data/spec/masamune/thor_spec.rb +238 -0
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
- data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
- data/spec/masamune/transform/define_event_view_spec.rb +84 -0
- data/spec/masamune/transform/define_schema_spec.rb +83 -0
- data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
- data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
- data/spec/masamune/transform/define_table.table_spec.rb +525 -0
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
- data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
- data/spec/masamune/transform/load_dimension_spec.rb +76 -0
- data/spec/masamune/transform/load_fact_spec.rb +89 -0
- data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
- data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
- data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
- data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
- data/spec/masamune/transform/stage_fact_spec.rb +204 -0
- data/spec/masamune_spec.rb +32 -0
- data/spec/spec_helper.rb +41 -0
- data/spec/support/masamune/example_group.rb +36 -0
- data/spec/support/masamune/mock_command.rb +99 -0
- data/spec/support/masamune/mock_delegate.rb +51 -0
- data/spec/support/masamune/mock_filesystem.rb +96 -0
- data/spec/support/masamune/thor_mute.rb +35 -0
- data/spec/support/rspec/example/action_example_group.rb +34 -0
- data/spec/support/rspec/example/task_example_group.rb +80 -0
- data/spec/support/rspec/example/transform_example_group.rb +36 -0
- data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
- metadata +462 -0
@@ -0,0 +1,58 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Actions::PostgresAdmin do
|
26
|
+
let(:klass) do
|
27
|
+
Class.new do
|
28
|
+
include Masamune::HasEnvironment
|
29
|
+
include Masamune::Actions::PostgresAdmin
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
let(:instance) { klass.new }
|
34
|
+
|
35
|
+
describe '.postgres_admin' do
|
36
|
+
subject { instance.postgres_admin(action: action, database: 'zombo') }
|
37
|
+
|
38
|
+
context 'with :action :create' do
|
39
|
+
let(:action) { :create }
|
40
|
+
|
41
|
+
before do
|
42
|
+
mock_command(/\Acreatedb/, mock_success)
|
43
|
+
end
|
44
|
+
|
45
|
+
it { is_expected.to be_success }
|
46
|
+
end
|
47
|
+
|
48
|
+
context 'with :action :drop' do
|
49
|
+
let(:action) { :drop }
|
50
|
+
|
51
|
+
before do
|
52
|
+
mock_command(/\Adropdb/, mock_success)
|
53
|
+
end
|
54
|
+
|
55
|
+
it { is_expected.to be_success }
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Actions::Postgres do
|
26
|
+
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
|
+
|
28
|
+
let(:klass) do
|
29
|
+
Class.new do
|
30
|
+
include Masamune::HasEnvironment
|
31
|
+
include Masamune::AfterInitializeCallbacks
|
32
|
+
include Masamune::Actions::Postgres
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
let(:instance) { klass.new }
|
37
|
+
let(:configuration) { {database: 'test'} }
|
38
|
+
let(:postgres_helper) { double }
|
39
|
+
let(:catalog) { double }
|
40
|
+
|
41
|
+
before do
|
42
|
+
allow(instance).to receive(:filesystem) { filesystem }
|
43
|
+
allow(instance).to receive(:catalog) { catalog }
|
44
|
+
allow(instance).to receive(:postgres_helper) { postgres_helper }
|
45
|
+
allow(instance).to receive_message_chain(:configuration, :postgres).and_return(configuration)
|
46
|
+
allow(instance).to receive_message_chain(:configuration, :with_quiet).and_yield
|
47
|
+
allow(instance).to receive_message_chain(:define_schema, :to_file) { 'catalog.psql' }
|
48
|
+
end
|
49
|
+
|
50
|
+
describe '.postgres' do
|
51
|
+
before do
|
52
|
+
mock_command(/\Apsql/, mock_success)
|
53
|
+
end
|
54
|
+
|
55
|
+
subject { instance.postgres }
|
56
|
+
|
57
|
+
it { is_expected.to be_success }
|
58
|
+
end
|
59
|
+
|
60
|
+
describe '.after_initialize' do
|
61
|
+
let(:options) { {} }
|
62
|
+
let(:setup_files) { [] }
|
63
|
+
let(:schema_files) { [] }
|
64
|
+
let(:configuration) { {database: 'test', setup_files: setup_files, schema_files: schema_files} }
|
65
|
+
|
66
|
+
subject(:after_initialize_invoke) do
|
67
|
+
instance.after_initialize_invoke(options)
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'when database does not exist' do
|
71
|
+
before do
|
72
|
+
expect(postgres_helper).to receive(:database_exists?).and_return(false)
|
73
|
+
expect(instance).to receive(:postgres_admin).with(action: :create, database: 'test', safe: true).once
|
74
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
|
75
|
+
after_initialize_invoke
|
76
|
+
end
|
77
|
+
it 'should call posgres_admin once' do; end
|
78
|
+
end
|
79
|
+
|
80
|
+
context 'when database exists' do
|
81
|
+
before do
|
82
|
+
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
83
|
+
expect(instance).to receive(:postgres_admin).never
|
84
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
|
85
|
+
after_initialize_invoke
|
86
|
+
end
|
87
|
+
it 'should not call postgres_admin' do; end
|
88
|
+
end
|
89
|
+
|
90
|
+
context 'when setup_files are configured' do
|
91
|
+
let(:setup_files) { ['setup.psql'] }
|
92
|
+
before do
|
93
|
+
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
94
|
+
expect(instance).to receive(:postgres).with(file: setup_files.first).once
|
95
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
|
96
|
+
after_initialize_invoke
|
97
|
+
end
|
98
|
+
it 'should call postgres with setup_files' do; end
|
99
|
+
end
|
100
|
+
|
101
|
+
context 'when schema_files are configured' do
|
102
|
+
let(:schema_files) { ['schema.psql'] }
|
103
|
+
before do
|
104
|
+
filesystem.touch!(*schema_files)
|
105
|
+
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
106
|
+
expect(instance).to receive(:postgres).once
|
107
|
+
after_initialize_invoke
|
108
|
+
end
|
109
|
+
it 'should call postgres with schema_files' do; end
|
110
|
+
end
|
111
|
+
|
112
|
+
context 'when schema_files that are globs are configured' do
|
113
|
+
let(:schema_files) { ['schema*.psql'] }
|
114
|
+
before do
|
115
|
+
filesystem.touch!('schema_1.psql', 'schema_2.psql')
|
116
|
+
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
117
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
|
118
|
+
after_initialize_invoke
|
119
|
+
end
|
120
|
+
it 'should call postgres with schema_files' do; end
|
121
|
+
end
|
122
|
+
|
123
|
+
context 'when ruby schema_files configured' do
|
124
|
+
let(:schema_files) { ['schema.rb'] }
|
125
|
+
before do
|
126
|
+
filesystem.touch!('schema.rb')
|
127
|
+
expect(postgres_helper).to receive(:database_exists?).and_return(true)
|
128
|
+
expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
|
129
|
+
after_initialize_invoke
|
130
|
+
end
|
131
|
+
it 'should call postgres with schema_files' do; end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Actions::S3Cmd do
|
26
|
+
let(:klass) do
|
27
|
+
Class.new do
|
28
|
+
include Masamune::HasEnvironment
|
29
|
+
include Masamune::Actions::S3Cmd
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
let(:instance) { klass.new }
|
34
|
+
|
35
|
+
describe '.s3cmd' do
|
36
|
+
before do
|
37
|
+
mock_command(/\As3cmd/, mock_success)
|
38
|
+
end
|
39
|
+
|
40
|
+
subject { instance.s3cmd 'ls', 's3://fake-bucket' }
|
41
|
+
|
42
|
+
it { is_expected.to be_success }
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Actions::Transform do
|
26
|
+
let(:environment) { double }
|
27
|
+
let(:catalog) { Masamune::Schema::Catalog.new(environment) }
|
28
|
+
|
29
|
+
before do
|
30
|
+
catalog.schema :postgres do
|
31
|
+
dimension 'date', type: :date do
|
32
|
+
column 'date_id', type: :integer, unique: true, index: true, natural_key: true
|
33
|
+
column 'date_epoch', type: :integer
|
34
|
+
column 'month_epoch', type: :integer
|
35
|
+
column 'year_epoch', type: :integer
|
36
|
+
end
|
37
|
+
|
38
|
+
dimension 'user', type: :four do
|
39
|
+
column 'tenant_id', type: :integer, index: true
|
40
|
+
column 'user_id', type: :integer, index: true, surrogate_key: true
|
41
|
+
end
|
42
|
+
|
43
|
+
file 'user' do
|
44
|
+
column 'id', type: :integer
|
45
|
+
column 'tenant_id', type: :integer
|
46
|
+
column 'updated_at', type: :timestamp
|
47
|
+
end
|
48
|
+
|
49
|
+
map from: postgres.user_file, to: postgres.user_dimension do |row|
|
50
|
+
{
|
51
|
+
user_id: row[:id],
|
52
|
+
tenant_id: row[:tenant_id],
|
53
|
+
source_kind: 'users',
|
54
|
+
start_at: row[:updated_at],
|
55
|
+
delta: 1
|
56
|
+
}
|
57
|
+
end
|
58
|
+
|
59
|
+
fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
|
60
|
+
references :date
|
61
|
+
references :user
|
62
|
+
|
63
|
+
measure 'total', type: :integer
|
64
|
+
end
|
65
|
+
|
66
|
+
file 'visits_hourly' do
|
67
|
+
column 'user.tenant_id', type: :integer
|
68
|
+
column 'user.user_id', type: :integer
|
69
|
+
column 'time_key', type: :integer
|
70
|
+
column 'total', type: :integer
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
let(:source_file) { Tempfile.new('masamune') }
|
76
|
+
|
77
|
+
let(:klass) do
|
78
|
+
Class.new do
|
79
|
+
include Masamune::HasEnvironment
|
80
|
+
include Masamune::Actions::Transform
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
let(:instance) { klass.new }
|
85
|
+
let(:postgres) { catalog.postgres }
|
86
|
+
|
87
|
+
before do
|
88
|
+
instance.environment = Masamune::ExampleGroup
|
89
|
+
end
|
90
|
+
|
91
|
+
describe '.load_dimension' do
|
92
|
+
before do
|
93
|
+
mock_command(/\Apsql/, mock_success)
|
94
|
+
end
|
95
|
+
|
96
|
+
subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
|
97
|
+
|
98
|
+
it { is_expected.to be_success }
|
99
|
+
end
|
100
|
+
|
101
|
+
describe '.relabel_dimension' do
|
102
|
+
before do
|
103
|
+
mock_command(/\Apsql/, mock_success)
|
104
|
+
end
|
105
|
+
|
106
|
+
subject { instance.relabel_dimension(postgres.user_dimension) }
|
107
|
+
|
108
|
+
it { is_expected.to be_success }
|
109
|
+
end
|
110
|
+
|
111
|
+
describe '.consolidate_dimension' do
|
112
|
+
before do
|
113
|
+
mock_command(/\Apsql/, mock_success)
|
114
|
+
end
|
115
|
+
|
116
|
+
subject { instance.consolidate_dimension(postgres.user_dimension) }
|
117
|
+
|
118
|
+
it { is_expected.to be_success }
|
119
|
+
end
|
120
|
+
|
121
|
+
describe '.load_fact' do
|
122
|
+
let(:date) { DateTime.civil(2014, 8) }
|
123
|
+
|
124
|
+
before do
|
125
|
+
mock_command(/\Apsql/, mock_success)
|
126
|
+
end
|
127
|
+
|
128
|
+
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
|
129
|
+
|
130
|
+
it { is_expected.to be_success }
|
131
|
+
end
|
132
|
+
|
133
|
+
describe '.rollup_fact' do
|
134
|
+
let(:date) { DateTime.civil(2014, 8) }
|
135
|
+
|
136
|
+
before do
|
137
|
+
mock_command(/\Apsql/, mock_success)
|
138
|
+
end
|
139
|
+
|
140
|
+
subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date) }
|
141
|
+
|
142
|
+
it { is_expected.to be_success }
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::AfterInitializeCallbacks do
|
26
|
+
let(:klass) do
|
27
|
+
Class.new do
|
28
|
+
include Masamune::AfterInitializeCallbacks
|
29
|
+
|
30
|
+
def first_callback; end
|
31
|
+
def early_callback; end
|
32
|
+
def default_callback; end
|
33
|
+
def unknown_callback; end
|
34
|
+
def later_callback; end
|
35
|
+
def final_callback; end
|
36
|
+
|
37
|
+
after_initialize(:first) { |o| o.first_callback }
|
38
|
+
after_initialize(:early) { |o| o.early_callback }
|
39
|
+
after_initialize(:default) { |o| o.default_callback }
|
40
|
+
after_initialize(:unknown) { |o| o.unknown_callback }
|
41
|
+
after_initialize(:later) { |o| o.later_callback }
|
42
|
+
after_initialize(:final) { |o| o.final_callback}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
let(:instance) { klass.new }
|
47
|
+
|
48
|
+
describe '.after_initialize_invoke' do
|
49
|
+
before do
|
50
|
+
expect(instance).to receive(:first_callback).once.ordered
|
51
|
+
expect(instance).to receive(:early_callback).once.ordered
|
52
|
+
expect(instance).to receive(:default_callback).once.ordered
|
53
|
+
expect(instance).to receive(:unknown_callback).once.ordered
|
54
|
+
expect(instance).to receive(:later_callback).once.ordered
|
55
|
+
expect(instance).to receive(:final_callback).once.ordered
|
56
|
+
instance.after_initialize_invoke
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'should call callbacks in priority order' do; end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::CachedFilesystem do
|
26
|
+
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
|
+
let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
|
28
|
+
|
29
|
+
context 'when path is present, top down traversal' do
|
30
|
+
before do
|
31
|
+
filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
|
32
|
+
expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
|
33
|
+
expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
|
34
|
+
expect(filesystem).to receive(:glob_stat).with('/*').never
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'calls Filesystem#glob_stat once for multiple calls' do
|
38
|
+
expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
|
39
|
+
expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
|
40
|
+
expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
|
41
|
+
expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
|
42
|
+
expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
|
43
|
+
expect(cached_filesystem.glob('/a/b/c/*')).not_to be_empty
|
44
|
+
expect(cached_filesystem.glob('/a/b/c/*.txt')).not_to be_empty
|
45
|
+
expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
|
46
|
+
expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
|
47
|
+
expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
|
48
|
+
expect(cached_filesystem.stat('/a/b/c/4.txt')).to be_nil
|
49
|
+
expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
|
50
|
+
expect(cached_filesystem.stat('/a/b')).to_not be_nil
|
51
|
+
expect(cached_filesystem.stat('/a')).to_not be_nil
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'when path is present, bottom up traversal' do
|
56
|
+
before do
|
57
|
+
filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
|
58
|
+
expect(filesystem).to receive(:glob_stat).with('/a/*').once.and_call_original
|
59
|
+
expect(filesystem).to receive(:glob_stat).with('/*').never
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'calls Filesystem#glob_stat once for multiple calls' do
|
63
|
+
expect(cached_filesystem.glob('/a/b/*')).to eq(['/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt'])
|
64
|
+
expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
|
65
|
+
expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
|
66
|
+
expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
|
67
|
+
expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
|
68
|
+
expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
|
69
|
+
expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
|
70
|
+
expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
|
71
|
+
expect(cached_filesystem.stat('/a/b/c/4.txt')).to be_nil
|
72
|
+
expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
|
73
|
+
expect(cached_filesystem.stat('/a/b')).to_not be_nil
|
74
|
+
expect(cached_filesystem.stat('/a')).to_not be_nil
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
context 'when path is present, checking for similar non existant paths' do
|
79
|
+
before do
|
80
|
+
filesystem.touch!('/y=2013/m=1/d=22/00000')
|
81
|
+
expect(filesystem).to receive(:glob_stat).with('/y=2013/m=1/*').once.and_call_original
|
82
|
+
expect(filesystem).to receive(:glob_stat).with('/y=2013/*').once.and_call_original
|
83
|
+
expect(filesystem).to receive(:glob_stat).with('/*').never
|
84
|
+
end
|
85
|
+
|
86
|
+
it 'calls Filesystem#glob_stat once for multiple calls' do
|
87
|
+
expect(cached_filesystem.exists?('/y=2013/m=1/d=22/00000')).to eq(true)
|
88
|
+
expect(cached_filesystem.exists?('/y=2013/m=1/d=22')).to eq(true)
|
89
|
+
expect(cached_filesystem.exists?('/y=2013/m=1/d=2')).to eq(false)
|
90
|
+
expect(cached_filesystem.glob('/y=2013/m=1/*')).not_to be_empty
|
91
|
+
expect(cached_filesystem.glob('/y=2013/m=1/d=22/*')).not_to be_empty
|
92
|
+
expect(cached_filesystem.stat('/y=2013/m=1/d=22/00000')).not_to be_nil
|
93
|
+
expect(cached_filesystem.stat('/y=2013/m=1/d=22')).not_to be_nil
|
94
|
+
expect(cached_filesystem.stat('/y=2013/m=1')).not_to be_nil
|
95
|
+
expect(cached_filesystem.stat('/y=2013')).not_to be_nil
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
context 'when path is present, checking for similar existing paths' do
|
100
|
+
before do
|
101
|
+
filesystem.touch!('/logs/box1_123.txt', '/logs/box2_123.txt', '/logs/box3_123.txt')
|
102
|
+
expect(filesystem).to receive(:glob_stat).with('/logs/*').once.and_call_original
|
103
|
+
expect(filesystem).to receive(:glob_stat).with('/*').never
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'calls Filesystem#glob_stat once for multiple calls' do
|
107
|
+
expect(cached_filesystem.exists?('/logs/box1_123.txt')).to eq(true)
|
108
|
+
expect(cached_filesystem.exists?('/logs/box1_456.txt')).to eq(false)
|
109
|
+
expect(cached_filesystem.exists?('/logs/box2_123.txt')).to eq(true)
|
110
|
+
expect(cached_filesystem.exists?('/logs/box2_456.txt')).to eq(false)
|
111
|
+
expect(cached_filesystem.exists?('/logs/box3_123.txt')).to eq(true)
|
112
|
+
expect(cached_filesystem.exists?('/logs/box3_456.txt')).to eq(false)
|
113
|
+
expect(cached_filesystem.exists?('/logs/box4_123.txt')).to eq(false)
|
114
|
+
expect(cached_filesystem.exists?('/logs/box4_456.txt')).to eq(false)
|
115
|
+
expect(cached_filesystem.exists?('/logs/box')).to eq(false)
|
116
|
+
expect(cached_filesystem.glob('/logs/*')).not_to be_empty
|
117
|
+
expect(cached_filesystem.glob('/logs/*.txt')).not_to be_empty
|
118
|
+
expect(cached_filesystem.glob('/logs/box1_*.txt')).not_to be_empty
|
119
|
+
expect(cached_filesystem.glob('/logs/box2_*.txt')).not_to be_empty
|
120
|
+
expect(cached_filesystem.glob('/logs/box3_*.txt')).not_to be_empty
|
121
|
+
expect(cached_filesystem.glob('/logs/box*.txt').size).to eq(3)
|
122
|
+
expect(cached_filesystem.glob('/logs/box*.csv')).to be_empty
|
123
|
+
expect(cached_filesystem.glob('/logs/box')).to be_empty
|
124
|
+
expect(cached_filesystem.glob('/logs/box/*')).to be_empty
|
125
|
+
expect(cached_filesystem.stat('/logs/box1_123.txt')).to_not be_nil
|
126
|
+
expect(cached_filesystem.stat('/logs/box1_456.txt')).to be_nil
|
127
|
+
expect(cached_filesystem.stat('/logs/box2_123.txt')).to_not be_nil
|
128
|
+
expect(cached_filesystem.stat('/logs/box2_456.txt')).to be_nil
|
129
|
+
expect(cached_filesystem.stat('/logs/box3_123.txt')).to_not be_nil
|
130
|
+
expect(cached_filesystem.stat('/logs/box3_456.txt')).to be_nil
|
131
|
+
expect(cached_filesystem.stat('/logs/box4_123.txt')).to be_nil
|
132
|
+
expect(cached_filesystem.stat('/logs/box4_456.txt')).to be_nil
|
133
|
+
expect(cached_filesystem.glob('/logs')).not_to be_nil
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
context 'when path is missing' do
|
138
|
+
before do
|
139
|
+
filesystem.touch!('/a/b/c')
|
140
|
+
expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
|
141
|
+
expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
|
142
|
+
expect(filesystem).to receive(:glob_stat).with('/*').never
|
143
|
+
end
|
144
|
+
|
145
|
+
it 'calls Filesystem#glob_stat once for multiple calls' do
|
146
|
+
expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(false)
|
147
|
+
expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(false)
|
148
|
+
expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(false)
|
149
|
+
expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
|
150
|
+
expect(cached_filesystem.exists?('/a/b')).to eq(true)
|
151
|
+
expect(cached_filesystem.exists?('/a')).to eq(true)
|
152
|
+
expect(cached_filesystem.glob('/a')).to be_empty
|
153
|
+
expect(cached_filesystem.glob('/a/*')).not_to be_empty
|
154
|
+
expect(cached_filesystem.glob('/a/b')).to be_empty
|
155
|
+
expect(cached_filesystem.glob('/a/b/*')).not_to be_empty
|
156
|
+
expect(cached_filesystem.glob('/a/b/c')).not_to be_empty
|
157
|
+
expect(cached_filesystem.glob('/a/b/c/*')).to be_empty
|
158
|
+
expect(cached_filesystem.glob('/a/b/c/*.txt')).to be_empty
|
159
|
+
expect(cached_filesystem.stat('/a/b/c/1.txt')).to be_nil
|
160
|
+
expect(cached_filesystem.stat('/a/b/c/2.txt')).to be_nil
|
161
|
+
expect(cached_filesystem.stat('/a/b/c/3.txt')).to be_nil
|
162
|
+
expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
|
163
|
+
expect(cached_filesystem.stat('/a/b')).to_not be_nil
|
164
|
+
expect(cached_filesystem.stat('/a')).to_not be_nil
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|