masamune 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +54 -0
- data/Rakefile +15 -0
- data/bin/masamune-elastic-mapreduce +4 -0
- data/bin/masamune-hive +4 -0
- data/bin/masamune-psql +4 -0
- data/bin/masamune-shell +4 -0
- data/lib/masamune.rb +56 -0
- data/lib/masamune/accumulate.rb +60 -0
- data/lib/masamune/actions.rb +38 -0
- data/lib/masamune/actions/data_flow.rb +131 -0
- data/lib/masamune/actions/date_parse.rb +75 -0
- data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
- data/lib/masamune/actions/execute.rb +52 -0
- data/lib/masamune/actions/filesystem.rb +37 -0
- data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
- data/lib/masamune/actions/hadoop_streaming.rb +41 -0
- data/lib/masamune/actions/hive.rb +74 -0
- data/lib/masamune/actions/postgres.rb +76 -0
- data/lib/masamune/actions/postgres_admin.rb +34 -0
- data/lib/masamune/actions/s3cmd.rb +44 -0
- data/lib/masamune/actions/transform.rb +89 -0
- data/lib/masamune/after_initialize_callbacks.rb +55 -0
- data/lib/masamune/cached_filesystem.rb +110 -0
- data/lib/masamune/commands.rb +37 -0
- data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
- data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
- data/lib/masamune/commands/hadoop_streaming.rb +116 -0
- data/lib/masamune/commands/hive.rb +178 -0
- data/lib/masamune/commands/interactive.rb +37 -0
- data/lib/masamune/commands/postgres.rb +128 -0
- data/lib/masamune/commands/postgres_admin.rb +72 -0
- data/lib/masamune/commands/postgres_common.rb +33 -0
- data/lib/masamune/commands/retry_with_backoff.rb +60 -0
- data/lib/masamune/commands/s3cmd.rb +70 -0
- data/lib/masamune/commands/shell.rb +202 -0
- data/lib/masamune/configuration.rb +195 -0
- data/lib/masamune/data_plan.rb +31 -0
- data/lib/masamune/data_plan/builder.rb +66 -0
- data/lib/masamune/data_plan/elem.rb +190 -0
- data/lib/masamune/data_plan/engine.rb +162 -0
- data/lib/masamune/data_plan/rule.rb +292 -0
- data/lib/masamune/data_plan/set.rb +176 -0
- data/lib/masamune/environment.rb +164 -0
- data/lib/masamune/filesystem.rb +567 -0
- data/lib/masamune/has_environment.rb +40 -0
- data/lib/masamune/helpers.rb +27 -0
- data/lib/masamune/helpers/postgres.rb +84 -0
- data/lib/masamune/io.rb +33 -0
- data/lib/masamune/last_element.rb +53 -0
- data/lib/masamune/method_logger.rb +41 -0
- data/lib/masamune/multi_io.rb +39 -0
- data/lib/masamune/schema.rb +36 -0
- data/lib/masamune/schema/catalog.rb +233 -0
- data/lib/masamune/schema/column.rb +527 -0
- data/lib/masamune/schema/dimension.rb +133 -0
- data/lib/masamune/schema/event.rb +121 -0
- data/lib/masamune/schema/fact.rb +133 -0
- data/lib/masamune/schema/map.rb +265 -0
- data/lib/masamune/schema/row.rb +133 -0
- data/lib/masamune/schema/store.rb +115 -0
- data/lib/masamune/schema/table.rb +308 -0
- data/lib/masamune/schema/table_reference.rb +76 -0
- data/lib/masamune/spec_helper.rb +23 -0
- data/lib/masamune/string_format.rb +34 -0
- data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
- data/lib/masamune/tasks/hive_thor.rb +55 -0
- data/lib/masamune/tasks/postgres_thor.rb +47 -0
- data/lib/masamune/tasks/shell_thor.rb +63 -0
- data/lib/masamune/template.rb +77 -0
- data/lib/masamune/thor.rb +186 -0
- data/lib/masamune/thor_loader.rb +38 -0
- data/lib/masamune/topological_hash.rb +34 -0
- data/lib/masamune/transform.rb +47 -0
- data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
- data/lib/masamune/transform/bulk_upsert.rb +52 -0
- data/lib/masamune/transform/consolidate_dimension.rb +54 -0
- data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
- data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
- data/lib/masamune/transform/define_event_view.hql.erb +51 -0
- data/lib/masamune/transform/define_event_view.rb +60 -0
- data/lib/masamune/transform/define_index.psql.erb +34 -0
- data/lib/masamune/transform/define_schema.hql.erb +23 -0
- data/lib/masamune/transform/define_schema.psql.erb +79 -0
- data/lib/masamune/transform/define_schema.rb +56 -0
- data/lib/masamune/transform/define_table.hql.erb +34 -0
- data/lib/masamune/transform/define_table.psql.erb +95 -0
- data/lib/masamune/transform/define_table.rb +40 -0
- data/lib/masamune/transform/define_unique.psql.erb +30 -0
- data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
- data/lib/masamune/transform/insert_reference_values.rb +64 -0
- data/lib/masamune/transform/load_dimension.rb +47 -0
- data/lib/masamune/transform/load_fact.rb +45 -0
- data/lib/masamune/transform/operator.rb +96 -0
- data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
- data/lib/masamune/transform/relabel_dimension.rb +39 -0
- data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
- data/lib/masamune/transform/rollup_fact.rb +149 -0
- data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
- data/lib/masamune/transform/snapshot_dimension.rb +74 -0
- data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
- data/lib/masamune/transform/stage_dimension.rb +83 -0
- data/lib/masamune/transform/stage_fact.psql.erb +80 -0
- data/lib/masamune/transform/stage_fact.rb +111 -0
- data/lib/masamune/version.rb +25 -0
- data/spec/fixtures/aggregate.sql.erb +25 -0
- data/spec/fixtures/comment.sql.erb +27 -0
- data/spec/fixtures/invalid.sql.erb +23 -0
- data/spec/fixtures/relative.sql.erb +23 -0
- data/spec/fixtures/simple.sql.erb +28 -0
- data/spec/fixtures/whitespace.sql.erb +30 -0
- data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
- data/spec/masamune/actions/execute_spec.rb +50 -0
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
- data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
- data/spec/masamune/actions/hive_spec.rb +117 -0
- data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
- data/spec/masamune/actions/postgres_spec.rb +134 -0
- data/spec/masamune/actions/s3cmd_spec.rb +44 -0
- data/spec/masamune/actions/transform_spec.rb +144 -0
- data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
- data/spec/masamune/cached_filesystem_spec.rb +167 -0
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
- data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
- data/spec/masamune/commands/hive_spec.rb +117 -0
- data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
- data/spec/masamune/commands/postgres_spec.rb +100 -0
- data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
- data/spec/masamune/commands/s3cmd_spec.rb +50 -0
- data/spec/masamune/commands/shell_spec.rb +101 -0
- data/spec/masamune/configuration_spec.rb +102 -0
- data/spec/masamune/data_plan/builder_spec.rb +91 -0
- data/spec/masamune/data_plan/elem_spec.rb +102 -0
- data/spec/masamune/data_plan/engine_spec.rb +356 -0
- data/spec/masamune/data_plan/rule_spec.rb +407 -0
- data/spec/masamune/data_plan/set_spec.rb +517 -0
- data/spec/masamune/environment_spec.rb +65 -0
- data/spec/masamune/filesystem_spec.rb +1421 -0
- data/spec/masamune/helpers/postgres_spec.rb +95 -0
- data/spec/masamune/schema/catalog_spec.rb +613 -0
- data/spec/masamune/schema/column_spec.rb +696 -0
- data/spec/masamune/schema/dimension_spec.rb +137 -0
- data/spec/masamune/schema/event_spec.rb +75 -0
- data/spec/masamune/schema/fact_spec.rb +117 -0
- data/spec/masamune/schema/map_spec.rb +593 -0
- data/spec/masamune/schema/row_spec.rb +28 -0
- data/spec/masamune/schema/store_spec.rb +49 -0
- data/spec/masamune/schema/table_spec.rb +395 -0
- data/spec/masamune/string_format_spec.rb +60 -0
- data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
- data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
- data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
- data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
- data/spec/masamune/template_spec.rb +77 -0
- data/spec/masamune/thor_spec.rb +238 -0
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
- data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
- data/spec/masamune/transform/define_event_view_spec.rb +84 -0
- data/spec/masamune/transform/define_schema_spec.rb +83 -0
- data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
- data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
- data/spec/masamune/transform/define_table.table_spec.rb +525 -0
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
- data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
- data/spec/masamune/transform/load_dimension_spec.rb +76 -0
- data/spec/masamune/transform/load_fact_spec.rb +89 -0
- data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
- data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
- data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
- data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
- data/spec/masamune/transform/stage_fact_spec.rb +204 -0
- data/spec/masamune_spec.rb +32 -0
- data/spec/spec_helper.rb +41 -0
- data/spec/support/masamune/example_group.rb +36 -0
- data/spec/support/masamune/mock_command.rb +99 -0
- data/spec/support/masamune/mock_delegate.rb +51 -0
- data/spec/support/masamune/mock_filesystem.rb +96 -0
- data/spec/support/masamune/thor_mute.rb +35 -0
- data/spec/support/rspec/example/action_example_group.rb +34 -0
- data/spec/support/rspec/example/task_example_group.rb +80 -0
- data/spec/support/rspec/example/transform_example_group.rb +36 -0
- data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
- metadata +462 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::StringFormat do
|
26
|
+
let(:instance) { Object.new.extend(described_class) }
|
27
|
+
|
28
|
+
describe '.strip_sql' do
|
29
|
+
subject { instance.strip_sql(input) }
|
30
|
+
|
31
|
+
context 'with quoted sql' do
|
32
|
+
let(:input) { %q('SELECT * FROM table;') }
|
33
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'with ; terminated sql' do
|
37
|
+
let(:input) { %q(SELECT * FROM table;;) }
|
38
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'with multi line sql' do
|
42
|
+
let(:input) do
|
43
|
+
<<-EOS
|
44
|
+
SELECT
|
45
|
+
*
|
46
|
+
FROM
|
47
|
+
table
|
48
|
+
;
|
49
|
+
|
50
|
+
EOS
|
51
|
+
end
|
52
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'with un-quoted sql' do
|
56
|
+
let(:input) { %q(SELECT * FROM table) }
|
57
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/elastic_mapreduce_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::ElasticMapreduceThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
it_behaves_like 'command usage'
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'with -j' do
|
35
|
+
let(:options) { ['-j', 'j-XYZ'] }
|
36
|
+
it do
|
37
|
+
expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(jobflow: 'j-XYZ', extra: ['--ssh'])).once.and_return(mock_success)
|
38
|
+
cli_invocation
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'with --jobflow' do
|
43
|
+
let(:options) { ['--jobflow=j-XYZ'] }
|
44
|
+
it do
|
45
|
+
expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(jobflow: 'j-XYZ', extra: ['--ssh'])).once.and_return(mock_success)
|
46
|
+
cli_invocation
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'with -- --list' do
|
51
|
+
let(:options) { ['--', '--list'] }
|
52
|
+
it do
|
53
|
+
expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(extra: ['--list'])).once.and_return(mock_success)
|
54
|
+
cli_invocation
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/hive_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::HiveThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
before do
|
32
|
+
expect_any_instance_of(described_class).to receive(:hive).never
|
33
|
+
end
|
34
|
+
it_behaves_like 'command usage'
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with command options' do
|
38
|
+
before do
|
39
|
+
expect_any_instance_of(described_class).to receive(:hive).with(exec: 'CREATE DATABASE IF NOT EXISTS masamune;', database: nil).and_return(mock_success)
|
40
|
+
expect_any_instance_of(described_class).to receive(:hive).with(file: instance_of(String)).and_return(mock_success)
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'with --file' do
|
44
|
+
let(:options) { ['--file=zombo.hql'] }
|
45
|
+
it do
|
46
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(file: File.expand_path('zombo.hql'))).once.and_return(mock_success)
|
47
|
+
cli_invocation
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context 'with --output' do
|
52
|
+
let(:options) { ['--output=report.txt'] }
|
53
|
+
it do
|
54
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(output: File.expand_path('report.txt'))).once.and_return(mock_success)
|
55
|
+
cli_invocation
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context 'with --variables=YEAR:2015 MONTH:1' do
|
60
|
+
let(:options) { ['--variables=YEAR:2015', 'MONTH:1'] }
|
61
|
+
it do
|
62
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(variables: { 'YEAR' => '2015', 'MONTH' => '1'})).once.and_return(mock_success)
|
63
|
+
cli_invocation
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'with -X YEAR:2015 MONTH:1' do
|
68
|
+
let(:options) { ['-X', 'YEAR:2015', 'MONTH:1'] }
|
69
|
+
it do
|
70
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(variables: { 'YEAR' => '2015', 'MONTH' => '1'})).once.and_return(mock_success)
|
71
|
+
cli_invocation
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/postgres_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::PostgresThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
it_behaves_like 'command usage'
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'with --file' do
|
35
|
+
let(:options) { ['--file=zombo.hql'] }
|
36
|
+
it do
|
37
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String)).once.and_return(mock_success)
|
38
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(file: 'zombo.hql')).once.and_return(mock_success)
|
39
|
+
cli_invocation
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/shell_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::ShellThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
it_behaves_like 'command usage'
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'with no arguments' do
|
35
|
+
it do
|
36
|
+
expect(Pry).to receive(:start)
|
37
|
+
cli_invocation
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'with --dump' do
|
42
|
+
let(:options) { ['--dump'] }
|
43
|
+
|
44
|
+
it 'exits with status code 0 and prints catalog' do
|
45
|
+
expect { cli_invocation }.to raise_error { |e|
|
46
|
+
expect(e).to be_a(SystemExit)
|
47
|
+
expect(e.status).to eq(0)
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Template do
|
26
|
+
describe '.render_to_file' do
|
27
|
+
let(:parameters) { {} }
|
28
|
+
|
29
|
+
subject do
|
30
|
+
File.read(described_class.render_to_file(template, parameters))
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'with invalid file' do
|
34
|
+
let(:template) { 'not_a_file.txt' }
|
35
|
+
it { expect { subject }.to raise_error IOError }
|
36
|
+
end
|
37
|
+
|
38
|
+
context 'with invalid template' do
|
39
|
+
let(:template) { File.expand_path('../../fixtures/invalid.sql.erb', __FILE__) }
|
40
|
+
it { expect { subject }.to raise_error IOError, /not_found.sql.erb/ }
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'with simple template' do
|
44
|
+
let(:template) { File.expand_path('../../fixtures/simple.sql.erb', __FILE__) }
|
45
|
+
let(:parameters) { {table: 'zombo'} }
|
46
|
+
|
47
|
+
it { is_expected.to eq("SELECT * FROM zombo;\n") }
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'with template with comments' do
|
51
|
+
let(:template) { File.expand_path('../../fixtures/comment.sql.erb', __FILE__) }
|
52
|
+
it { is_expected.to eq("SELECT 1;\n") }
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'with template with unnecessary whitespace' do
|
56
|
+
let(:template) { File.expand_path('../../fixtures/whitespace.sql.erb', __FILE__) }
|
57
|
+
|
58
|
+
it { is_expected.to eq("SELECT 1;\n\nSELECT 2;\n") }
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'with aggregate template' do
|
62
|
+
let(:template) { File.expand_path('../../fixtures/aggregate.sql.erb', __FILE__) }
|
63
|
+
|
64
|
+
it do is_expected.to eq <<-EOS.gsub(/^\s*/,'')
|
65
|
+
SHOW TABLES;
|
66
|
+
SELECT * FROM foo;
|
67
|
+
SELECT * FROM bar;
|
68
|
+
EOS
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context 'with aggregate template with relative path' do
|
73
|
+
let(:template) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'relative.sql.erb') }
|
74
|
+
it { is_expected.to eq("SELECT * FROM relative;\n") }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,238 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
describe Masamune::Thor do
|
27
|
+
let(:thor_class) do
|
28
|
+
Class.new(Thor) do
|
29
|
+
include Masamune::Thor
|
30
|
+
include Masamune::Actions::DataFlow
|
31
|
+
|
32
|
+
desc 'command', 'command'
|
33
|
+
target path: "target/%Y-%m-%d"
|
34
|
+
source path: "source/%Y%m%d*.log"
|
35
|
+
method_option :zombo, desc: 'Anything is possible'
|
36
|
+
def command
|
37
|
+
# NOP
|
38
|
+
end
|
39
|
+
|
40
|
+
desc 'other', 'other'
|
41
|
+
skip
|
42
|
+
def other
|
43
|
+
# NOP
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
before do
|
49
|
+
allow_any_instance_of(thor_class).to receive(:top_level?).and_return(true)
|
50
|
+
end
|
51
|
+
|
52
|
+
context 'CLI' do
|
53
|
+
context 'ouside of top level' do
|
54
|
+
let(:command) { 'command' }
|
55
|
+
let(:options) { ['--start', '2013-01-01'] }
|
56
|
+
|
57
|
+
before do
|
58
|
+
allow_any_instance_of(thor_class).to receive(:top_level?).and_return(false)
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'continues execution' do
|
62
|
+
expect { cli_invocation }.to_not raise_error
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'without command' do
|
67
|
+
it_behaves_like 'general usage'
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'with help command ' do
|
71
|
+
let(:command) { 'help' }
|
72
|
+
it_behaves_like 'general usage'
|
73
|
+
end
|
74
|
+
|
75
|
+
context 'with --help option' do
|
76
|
+
let(:command) { 'command' }
|
77
|
+
let(:options) { ['--help'] }
|
78
|
+
it_behaves_like 'command usage'
|
79
|
+
end
|
80
|
+
|
81
|
+
context 'with help subcommand ' do
|
82
|
+
let(:command) { 'help' }
|
83
|
+
let(:options) { ['command'] }
|
84
|
+
it_behaves_like 'command usage'
|
85
|
+
end
|
86
|
+
|
87
|
+
context 'with command and --version' do
|
88
|
+
let(:command) { 'command' }
|
89
|
+
let(:options) { ['--version'] }
|
90
|
+
it 'exits with status code 0 and prints version' do
|
91
|
+
expect { cli_invocation }.to raise_error { |e|
|
92
|
+
expect(e).to be_a(SystemExit)
|
93
|
+
expect(e.status).to eq(0)
|
94
|
+
}
|
95
|
+
expect(stdout.string).to match(/\Amasamune/)
|
96
|
+
expect(stderr.string).to be_blank
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
context 'with command and no input options' do
|
101
|
+
let(:command) { 'command' }
|
102
|
+
it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /No value provided for required options '--start'/ }
|
103
|
+
end
|
104
|
+
|
105
|
+
context 'with command and invalid --start' do
|
106
|
+
let(:command) { 'command' }
|
107
|
+
let(:options) { ['--start', 'xxx'] }
|
108
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--start'; got/ }
|
109
|
+
end
|
110
|
+
|
111
|
+
context 'with command and invalid --stop' do
|
112
|
+
let(:command) { 'command' }
|
113
|
+
let(:options) { ['--start', '2013-01-01', '--stop', 'xxx'] }
|
114
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--stop'; got/ }
|
115
|
+
end
|
116
|
+
|
117
|
+
context 'with command and invalid --sources' do
|
118
|
+
let(:command) { 'command' }
|
119
|
+
let(:options) { ['--sources', 'foo'] }
|
120
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--sources'; got/ }
|
121
|
+
end
|
122
|
+
|
123
|
+
context 'with command and invalid --targets' do
|
124
|
+
let(:command) { 'command' }
|
125
|
+
let(:options) { ['--targets', 'foo'] }
|
126
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--targets'; got/ }
|
127
|
+
end
|
128
|
+
|
129
|
+
context 'with command and both --sources and --targets' do
|
130
|
+
let(:command) { 'command' }
|
131
|
+
let(:options) { ['--sources', 'sources', '--targets', 'targets'] }
|
132
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Cannot specify both option '--sources' and option '--targets'/ }
|
133
|
+
end
|
134
|
+
|
135
|
+
context 'with command and --start and bad --config file' do
|
136
|
+
let(:command) { 'command' }
|
137
|
+
let(:options) { ['--start', '2013-01-01', '--config', 'xxx'] }
|
138
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Could not load file provided for '--config'/ }
|
139
|
+
end
|
140
|
+
|
141
|
+
context 'with command and --start and missing system --config file' do
|
142
|
+
let(:command) { 'command' }
|
143
|
+
let(:options) { ['--start', '2013-01-01'] }
|
144
|
+
before do
|
145
|
+
expect_any_instance_of(Masamune::Filesystem).to receive(:resolve_file)
|
146
|
+
end
|
147
|
+
it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /Option --config or valid system configuration file required/ }
|
148
|
+
end
|
149
|
+
|
150
|
+
context 'with command and -- --extra --args' do
|
151
|
+
let(:command) { 'command' }
|
152
|
+
let(:options) { ['--start', '2013-01-01', '--', '--extra', '--args'] }
|
153
|
+
before do
|
154
|
+
expect_any_instance_of(thor_class).to receive(:extra=).with(['--extra', '--args'])
|
155
|
+
end
|
156
|
+
it do
|
157
|
+
expect { cli_invocation }.to raise_error SystemExit
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'with command and --start' do
|
162
|
+
let(:command) { 'command' }
|
163
|
+
let(:options) { ['--start', '2013-01-01'] }
|
164
|
+
it 'exits with status code 0 without error message' do
|
165
|
+
expect { cli_invocation }.to raise_error { |e|
|
166
|
+
expect(e).to be_a(SystemExit)
|
167
|
+
expect(e.status).to eq(0)
|
168
|
+
}
|
169
|
+
expect(stdout.string).to match(/\AUsing '.*' for --start/)
|
170
|
+
expect(stderr.string).to eq('')
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
context 'with command and natural language --start' do
|
175
|
+
let(:command) { 'command' }
|
176
|
+
let(:options) { ['--start', 'yesterday'] }
|
177
|
+
it 'exits with status code 0 without error message' do
|
178
|
+
expect { cli_invocation }.to raise_error { |e|
|
179
|
+
expect(e).to be_a(SystemExit)
|
180
|
+
expect(e.status).to eq(0)
|
181
|
+
}
|
182
|
+
expect(stdout.string).to match(/\AUsing '.*' for --start/)
|
183
|
+
expect(stderr.string).to eq('')
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
context 'with command that raises exception before initialization' do
|
188
|
+
let(:command) { 'command' }
|
189
|
+
let(:options) { ['--start', '2013-01-01'] }
|
190
|
+
before do
|
191
|
+
expect_any_instance_of(Logger).to receive(:error).with(/random exception/)
|
192
|
+
allow(thor_class).to receive(:dispatch).and_raise('random exception')
|
193
|
+
end
|
194
|
+
it { expect { cli_invocation }.to raise_error /random exception/ }
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'with command that raises exception after initialization' do
|
198
|
+
let(:command) { 'command' }
|
199
|
+
let(:options) { ['--start', '2013-01-01'] }
|
200
|
+
before do
|
201
|
+
expect_any_instance_of(Logger).to receive(:error).with(/random exception/)
|
202
|
+
allow(thor_class).to receive(:after_initialize_invoke).and_raise('random exception')
|
203
|
+
end
|
204
|
+
it { expect { cli_invocation }.to raise_error /random exception/ }
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
context '.parse_extra' do
|
209
|
+
subject do
|
210
|
+
thor_class.parse_extra(argv)
|
211
|
+
end
|
212
|
+
|
213
|
+
context 'without --' do
|
214
|
+
let(:argv) { ['--flag', 'true'] }
|
215
|
+
it { is_expected.to eq([['--flag', 'true'],[]]) }
|
216
|
+
end
|
217
|
+
|
218
|
+
context 'with -- and no following arguments' do
|
219
|
+
let(:argv) { ['--flag', 'true', '--'] }
|
220
|
+
it { is_expected.to eq([['--flag', 'true'],[]]) }
|
221
|
+
end
|
222
|
+
|
223
|
+
context 'with -- and a single extra argument' do
|
224
|
+
let(:argv) { ['--flag', 'true', '--', '--more'] }
|
225
|
+
it { is_expected.to eq([['--flag', 'true'], ['--more']]) }
|
226
|
+
end
|
227
|
+
|
228
|
+
context 'with -- and multiple extra agruments' do
|
229
|
+
let(:argv) { ['--flag', 'true', '--', '--more', 'flag'] }
|
230
|
+
it { is_expected.to eq([['--flag', 'true'], ['--more', 'flag']]) }
|
231
|
+
end
|
232
|
+
|
233
|
+
context 'with leading -- and a single extra argument' do
|
234
|
+
let(:argv) { ['--', '--more'] }
|
235
|
+
it { is_expected.to eq([[], ['--more']]) }
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|