masamune 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +54 -0
- data/Rakefile +15 -0
- data/bin/masamune-elastic-mapreduce +4 -0
- data/bin/masamune-hive +4 -0
- data/bin/masamune-psql +4 -0
- data/bin/masamune-shell +4 -0
- data/lib/masamune.rb +56 -0
- data/lib/masamune/accumulate.rb +60 -0
- data/lib/masamune/actions.rb +38 -0
- data/lib/masamune/actions/data_flow.rb +131 -0
- data/lib/masamune/actions/date_parse.rb +75 -0
- data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
- data/lib/masamune/actions/execute.rb +52 -0
- data/lib/masamune/actions/filesystem.rb +37 -0
- data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
- data/lib/masamune/actions/hadoop_streaming.rb +41 -0
- data/lib/masamune/actions/hive.rb +74 -0
- data/lib/masamune/actions/postgres.rb +76 -0
- data/lib/masamune/actions/postgres_admin.rb +34 -0
- data/lib/masamune/actions/s3cmd.rb +44 -0
- data/lib/masamune/actions/transform.rb +89 -0
- data/lib/masamune/after_initialize_callbacks.rb +55 -0
- data/lib/masamune/cached_filesystem.rb +110 -0
- data/lib/masamune/commands.rb +37 -0
- data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
- data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
- data/lib/masamune/commands/hadoop_streaming.rb +116 -0
- data/lib/masamune/commands/hive.rb +178 -0
- data/lib/masamune/commands/interactive.rb +37 -0
- data/lib/masamune/commands/postgres.rb +128 -0
- data/lib/masamune/commands/postgres_admin.rb +72 -0
- data/lib/masamune/commands/postgres_common.rb +33 -0
- data/lib/masamune/commands/retry_with_backoff.rb +60 -0
- data/lib/masamune/commands/s3cmd.rb +70 -0
- data/lib/masamune/commands/shell.rb +202 -0
- data/lib/masamune/configuration.rb +195 -0
- data/lib/masamune/data_plan.rb +31 -0
- data/lib/masamune/data_plan/builder.rb +66 -0
- data/lib/masamune/data_plan/elem.rb +190 -0
- data/lib/masamune/data_plan/engine.rb +162 -0
- data/lib/masamune/data_plan/rule.rb +292 -0
- data/lib/masamune/data_plan/set.rb +176 -0
- data/lib/masamune/environment.rb +164 -0
- data/lib/masamune/filesystem.rb +567 -0
- data/lib/masamune/has_environment.rb +40 -0
- data/lib/masamune/helpers.rb +27 -0
- data/lib/masamune/helpers/postgres.rb +84 -0
- data/lib/masamune/io.rb +33 -0
- data/lib/masamune/last_element.rb +53 -0
- data/lib/masamune/method_logger.rb +41 -0
- data/lib/masamune/multi_io.rb +39 -0
- data/lib/masamune/schema.rb +36 -0
- data/lib/masamune/schema/catalog.rb +233 -0
- data/lib/masamune/schema/column.rb +527 -0
- data/lib/masamune/schema/dimension.rb +133 -0
- data/lib/masamune/schema/event.rb +121 -0
- data/lib/masamune/schema/fact.rb +133 -0
- data/lib/masamune/schema/map.rb +265 -0
- data/lib/masamune/schema/row.rb +133 -0
- data/lib/masamune/schema/store.rb +115 -0
- data/lib/masamune/schema/table.rb +308 -0
- data/lib/masamune/schema/table_reference.rb +76 -0
- data/lib/masamune/spec_helper.rb +23 -0
- data/lib/masamune/string_format.rb +34 -0
- data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
- data/lib/masamune/tasks/hive_thor.rb +55 -0
- data/lib/masamune/tasks/postgres_thor.rb +47 -0
- data/lib/masamune/tasks/shell_thor.rb +63 -0
- data/lib/masamune/template.rb +77 -0
- data/lib/masamune/thor.rb +186 -0
- data/lib/masamune/thor_loader.rb +38 -0
- data/lib/masamune/topological_hash.rb +34 -0
- data/lib/masamune/transform.rb +47 -0
- data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
- data/lib/masamune/transform/bulk_upsert.rb +52 -0
- data/lib/masamune/transform/consolidate_dimension.rb +54 -0
- data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
- data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
- data/lib/masamune/transform/define_event_view.hql.erb +51 -0
- data/lib/masamune/transform/define_event_view.rb +60 -0
- data/lib/masamune/transform/define_index.psql.erb +34 -0
- data/lib/masamune/transform/define_schema.hql.erb +23 -0
- data/lib/masamune/transform/define_schema.psql.erb +79 -0
- data/lib/masamune/transform/define_schema.rb +56 -0
- data/lib/masamune/transform/define_table.hql.erb +34 -0
- data/lib/masamune/transform/define_table.psql.erb +95 -0
- data/lib/masamune/transform/define_table.rb +40 -0
- data/lib/masamune/transform/define_unique.psql.erb +30 -0
- data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
- data/lib/masamune/transform/insert_reference_values.rb +64 -0
- data/lib/masamune/transform/load_dimension.rb +47 -0
- data/lib/masamune/transform/load_fact.rb +45 -0
- data/lib/masamune/transform/operator.rb +96 -0
- data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
- data/lib/masamune/transform/relabel_dimension.rb +39 -0
- data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
- data/lib/masamune/transform/rollup_fact.rb +149 -0
- data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
- data/lib/masamune/transform/snapshot_dimension.rb +74 -0
- data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
- data/lib/masamune/transform/stage_dimension.rb +83 -0
- data/lib/masamune/transform/stage_fact.psql.erb +80 -0
- data/lib/masamune/transform/stage_fact.rb +111 -0
- data/lib/masamune/version.rb +25 -0
- data/spec/fixtures/aggregate.sql.erb +25 -0
- data/spec/fixtures/comment.sql.erb +27 -0
- data/spec/fixtures/invalid.sql.erb +23 -0
- data/spec/fixtures/relative.sql.erb +23 -0
- data/spec/fixtures/simple.sql.erb +28 -0
- data/spec/fixtures/whitespace.sql.erb +30 -0
- data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
- data/spec/masamune/actions/execute_spec.rb +50 -0
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
- data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
- data/spec/masamune/actions/hive_spec.rb +117 -0
- data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
- data/spec/masamune/actions/postgres_spec.rb +134 -0
- data/spec/masamune/actions/s3cmd_spec.rb +44 -0
- data/spec/masamune/actions/transform_spec.rb +144 -0
- data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
- data/spec/masamune/cached_filesystem_spec.rb +167 -0
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
- data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
- data/spec/masamune/commands/hive_spec.rb +117 -0
- data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
- data/spec/masamune/commands/postgres_spec.rb +100 -0
- data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
- data/spec/masamune/commands/s3cmd_spec.rb +50 -0
- data/spec/masamune/commands/shell_spec.rb +101 -0
- data/spec/masamune/configuration_spec.rb +102 -0
- data/spec/masamune/data_plan/builder_spec.rb +91 -0
- data/spec/masamune/data_plan/elem_spec.rb +102 -0
- data/spec/masamune/data_plan/engine_spec.rb +356 -0
- data/spec/masamune/data_plan/rule_spec.rb +407 -0
- data/spec/masamune/data_plan/set_spec.rb +517 -0
- data/spec/masamune/environment_spec.rb +65 -0
- data/spec/masamune/filesystem_spec.rb +1421 -0
- data/spec/masamune/helpers/postgres_spec.rb +95 -0
- data/spec/masamune/schema/catalog_spec.rb +613 -0
- data/spec/masamune/schema/column_spec.rb +696 -0
- data/spec/masamune/schema/dimension_spec.rb +137 -0
- data/spec/masamune/schema/event_spec.rb +75 -0
- data/spec/masamune/schema/fact_spec.rb +117 -0
- data/spec/masamune/schema/map_spec.rb +593 -0
- data/spec/masamune/schema/row_spec.rb +28 -0
- data/spec/masamune/schema/store_spec.rb +49 -0
- data/spec/masamune/schema/table_spec.rb +395 -0
- data/spec/masamune/string_format_spec.rb +60 -0
- data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
- data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
- data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
- data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
- data/spec/masamune/template_spec.rb +77 -0
- data/spec/masamune/thor_spec.rb +238 -0
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
- data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
- data/spec/masamune/transform/define_event_view_spec.rb +84 -0
- data/spec/masamune/transform/define_schema_spec.rb +83 -0
- data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
- data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
- data/spec/masamune/transform/define_table.table_spec.rb +525 -0
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
- data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
- data/spec/masamune/transform/load_dimension_spec.rb +76 -0
- data/spec/masamune/transform/load_fact_spec.rb +89 -0
- data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
- data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
- data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
- data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
- data/spec/masamune/transform/stage_fact_spec.rb +204 -0
- data/spec/masamune_spec.rb +32 -0
- data/spec/spec_helper.rb +41 -0
- data/spec/support/masamune/example_group.rb +36 -0
- data/spec/support/masamune/mock_command.rb +99 -0
- data/spec/support/masamune/mock_delegate.rb +51 -0
- data/spec/support/masamune/mock_filesystem.rb +96 -0
- data/spec/support/masamune/thor_mute.rb +35 -0
- data/spec/support/rspec/example/action_example_group.rb +34 -0
- data/spec/support/rspec/example/task_example_group.rb +80 -0
- data/spec/support/rspec/example/transform_example_group.rb +36 -0
- data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
- metadata +462 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::StringFormat do
|
26
|
+
let(:instance) { Object.new.extend(described_class) }
|
27
|
+
|
28
|
+
describe '.strip_sql' do
|
29
|
+
subject { instance.strip_sql(input) }
|
30
|
+
|
31
|
+
context 'with quoted sql' do
|
32
|
+
let(:input) { %q('SELECT * FROM table;') }
|
33
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'with ; terminated sql' do
|
37
|
+
let(:input) { %q(SELECT * FROM table;;) }
|
38
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'with multi line sql' do
|
42
|
+
let(:input) do
|
43
|
+
<<-EOS
|
44
|
+
SELECT
|
45
|
+
*
|
46
|
+
FROM
|
47
|
+
table
|
48
|
+
;
|
49
|
+
|
50
|
+
EOS
|
51
|
+
end
|
52
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'with un-quoted sql' do
|
56
|
+
let(:input) { %q(SELECT * FROM table) }
|
57
|
+
it { is_expected.to eq(%q(SELECT * FROM table;)) }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/elastic_mapreduce_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::ElasticMapreduceThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
it_behaves_like 'command usage'
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'with -j' do
|
35
|
+
let(:options) { ['-j', 'j-XYZ'] }
|
36
|
+
it do
|
37
|
+
expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(jobflow: 'j-XYZ', extra: ['--ssh'])).once.and_return(mock_success)
|
38
|
+
cli_invocation
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
context 'with --jobflow' do
|
43
|
+
let(:options) { ['--jobflow=j-XYZ'] }
|
44
|
+
it do
|
45
|
+
expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(jobflow: 'j-XYZ', extra: ['--ssh'])).once.and_return(mock_success)
|
46
|
+
cli_invocation
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'with -- --list' do
|
51
|
+
let(:options) { ['--', '--list'] }
|
52
|
+
it do
|
53
|
+
expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(extra: ['--list'])).once.and_return(mock_success)
|
54
|
+
cli_invocation
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/hive_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::HiveThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
before do
|
32
|
+
expect_any_instance_of(described_class).to receive(:hive).never
|
33
|
+
end
|
34
|
+
it_behaves_like 'command usage'
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'with command options' do
|
38
|
+
before do
|
39
|
+
expect_any_instance_of(described_class).to receive(:hive).with(exec: 'CREATE DATABASE IF NOT EXISTS masamune;', database: nil).and_return(mock_success)
|
40
|
+
expect_any_instance_of(described_class).to receive(:hive).with(file: instance_of(String)).and_return(mock_success)
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'with --file' do
|
44
|
+
let(:options) { ['--file=zombo.hql'] }
|
45
|
+
it do
|
46
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(file: File.expand_path('zombo.hql'))).once.and_return(mock_success)
|
47
|
+
cli_invocation
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
context 'with --output' do
|
52
|
+
let(:options) { ['--output=report.txt'] }
|
53
|
+
it do
|
54
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(output: File.expand_path('report.txt'))).once.and_return(mock_success)
|
55
|
+
cli_invocation
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context 'with --variables=YEAR:2015 MONTH:1' do
|
60
|
+
let(:options) { ['--variables=YEAR:2015', 'MONTH:1'] }
|
61
|
+
it do
|
62
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(variables: { 'YEAR' => '2015', 'MONTH' => '1'})).once.and_return(mock_success)
|
63
|
+
cli_invocation
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'with -X YEAR:2015 MONTH:1' do
|
68
|
+
let(:options) { ['-X', 'YEAR:2015', 'MONTH:1'] }
|
69
|
+
it do
|
70
|
+
expect_any_instance_of(described_class).to receive(:hive).with(hash_including(variables: { 'YEAR' => '2015', 'MONTH' => '1'})).once.and_return(mock_success)
|
71
|
+
cli_invocation
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/postgres_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::PostgresThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
it_behaves_like 'command usage'
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'with --file' do
|
35
|
+
let(:options) { ['--file=zombo.hql'] }
|
36
|
+
it do
|
37
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String)).once.and_return(mock_success)
|
38
|
+
expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(file: 'zombo.hql')).once.and_return(mock_success)
|
39
|
+
cli_invocation
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
require 'masamune/tasks/shell_thor'
|
27
|
+
|
28
|
+
describe Masamune::Tasks::ShellThor do
|
29
|
+
context 'with help command ' do
|
30
|
+
let(:command) { 'help' }
|
31
|
+
it_behaves_like 'command usage'
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'with no arguments' do
|
35
|
+
it do
|
36
|
+
expect(Pry).to receive(:start)
|
37
|
+
cli_invocation
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
context 'with --dump' do
|
42
|
+
let(:options) { ['--dump'] }
|
43
|
+
|
44
|
+
it 'exits with status code 0 and prints catalog' do
|
45
|
+
expect { cli_invocation }.to raise_error { |e|
|
46
|
+
expect(e).to be_a(SystemExit)
|
47
|
+
expect(e.status).to eq(0)
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Template do
|
26
|
+
describe '.render_to_file' do
|
27
|
+
let(:parameters) { {} }
|
28
|
+
|
29
|
+
subject do
|
30
|
+
File.read(described_class.render_to_file(template, parameters))
|
31
|
+
end
|
32
|
+
|
33
|
+
context 'with invalid file' do
|
34
|
+
let(:template) { 'not_a_file.txt' }
|
35
|
+
it { expect { subject }.to raise_error IOError }
|
36
|
+
end
|
37
|
+
|
38
|
+
context 'with invalid template' do
|
39
|
+
let(:template) { File.expand_path('../../fixtures/invalid.sql.erb', __FILE__) }
|
40
|
+
it { expect { subject }.to raise_error IOError, /not_found.sql.erb/ }
|
41
|
+
end
|
42
|
+
|
43
|
+
context 'with simple template' do
|
44
|
+
let(:template) { File.expand_path('../../fixtures/simple.sql.erb', __FILE__) }
|
45
|
+
let(:parameters) { {table: 'zombo'} }
|
46
|
+
|
47
|
+
it { is_expected.to eq("SELECT * FROM zombo;\n") }
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'with template with comments' do
|
51
|
+
let(:template) { File.expand_path('../../fixtures/comment.sql.erb', __FILE__) }
|
52
|
+
it { is_expected.to eq("SELECT 1;\n") }
|
53
|
+
end
|
54
|
+
|
55
|
+
context 'with template with unnecessary whitespace' do
|
56
|
+
let(:template) { File.expand_path('../../fixtures/whitespace.sql.erb', __FILE__) }
|
57
|
+
|
58
|
+
it { is_expected.to eq("SELECT 1;\n\nSELECT 2;\n") }
|
59
|
+
end
|
60
|
+
|
61
|
+
context 'with aggregate template' do
|
62
|
+
let(:template) { File.expand_path('../../fixtures/aggregate.sql.erb', __FILE__) }
|
63
|
+
|
64
|
+
it do is_expected.to eq <<-EOS.gsub(/^\s*/,'')
|
65
|
+
SHOW TABLES;
|
66
|
+
SELECT * FROM foo;
|
67
|
+
SELECT * FROM bar;
|
68
|
+
EOS
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context 'with aggregate template with relative path' do
|
73
|
+
let(:template) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'relative.sql.erb') }
|
74
|
+
it { is_expected.to eq("SELECT * FROM relative;\n") }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,238 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
require 'thor'
|
25
|
+
|
26
|
+
describe Masamune::Thor do
|
27
|
+
let(:thor_class) do
|
28
|
+
Class.new(Thor) do
|
29
|
+
include Masamune::Thor
|
30
|
+
include Masamune::Actions::DataFlow
|
31
|
+
|
32
|
+
desc 'command', 'command'
|
33
|
+
target path: "target/%Y-%m-%d"
|
34
|
+
source path: "source/%Y%m%d*.log"
|
35
|
+
method_option :zombo, desc: 'Anything is possible'
|
36
|
+
def command
|
37
|
+
# NOP
|
38
|
+
end
|
39
|
+
|
40
|
+
desc 'other', 'other'
|
41
|
+
skip
|
42
|
+
def other
|
43
|
+
# NOP
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
before do
|
49
|
+
allow_any_instance_of(thor_class).to receive(:top_level?).and_return(true)
|
50
|
+
end
|
51
|
+
|
52
|
+
context 'CLI' do
|
53
|
+
context 'ouside of top level' do
|
54
|
+
let(:command) { 'command' }
|
55
|
+
let(:options) { ['--start', '2013-01-01'] }
|
56
|
+
|
57
|
+
before do
|
58
|
+
allow_any_instance_of(thor_class).to receive(:top_level?).and_return(false)
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'continues execution' do
|
62
|
+
expect { cli_invocation }.to_not raise_error
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'without command' do
|
67
|
+
it_behaves_like 'general usage'
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'with help command ' do
|
71
|
+
let(:command) { 'help' }
|
72
|
+
it_behaves_like 'general usage'
|
73
|
+
end
|
74
|
+
|
75
|
+
context 'with --help option' do
|
76
|
+
let(:command) { 'command' }
|
77
|
+
let(:options) { ['--help'] }
|
78
|
+
it_behaves_like 'command usage'
|
79
|
+
end
|
80
|
+
|
81
|
+
context 'with help subcommand ' do
|
82
|
+
let(:command) { 'help' }
|
83
|
+
let(:options) { ['command'] }
|
84
|
+
it_behaves_like 'command usage'
|
85
|
+
end
|
86
|
+
|
87
|
+
context 'with command and --version' do
|
88
|
+
let(:command) { 'command' }
|
89
|
+
let(:options) { ['--version'] }
|
90
|
+
it 'exits with status code 0 and prints version' do
|
91
|
+
expect { cli_invocation }.to raise_error { |e|
|
92
|
+
expect(e).to be_a(SystemExit)
|
93
|
+
expect(e.status).to eq(0)
|
94
|
+
}
|
95
|
+
expect(stdout.string).to match(/\Amasamune/)
|
96
|
+
expect(stderr.string).to be_blank
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
context 'with command and no input options' do
|
101
|
+
let(:command) { 'command' }
|
102
|
+
it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /No value provided for required options '--start'/ }
|
103
|
+
end
|
104
|
+
|
105
|
+
context 'with command and invalid --start' do
|
106
|
+
let(:command) { 'command' }
|
107
|
+
let(:options) { ['--start', 'xxx'] }
|
108
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--start'; got/ }
|
109
|
+
end
|
110
|
+
|
111
|
+
context 'with command and invalid --stop' do
|
112
|
+
let(:command) { 'command' }
|
113
|
+
let(:options) { ['--start', '2013-01-01', '--stop', 'xxx'] }
|
114
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--stop'; got/ }
|
115
|
+
end
|
116
|
+
|
117
|
+
context 'with command and invalid --sources' do
|
118
|
+
let(:command) { 'command' }
|
119
|
+
let(:options) { ['--sources', 'foo'] }
|
120
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--sources'; got/ }
|
121
|
+
end
|
122
|
+
|
123
|
+
context 'with command and invalid --targets' do
|
124
|
+
let(:command) { 'command' }
|
125
|
+
let(:options) { ['--targets', 'foo'] }
|
126
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--targets'; got/ }
|
127
|
+
end
|
128
|
+
|
129
|
+
context 'with command and both --sources and --targets' do
|
130
|
+
let(:command) { 'command' }
|
131
|
+
let(:options) { ['--sources', 'sources', '--targets', 'targets'] }
|
132
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Cannot specify both option '--sources' and option '--targets'/ }
|
133
|
+
end
|
134
|
+
|
135
|
+
context 'with command and --start and bad --config file' do
|
136
|
+
let(:command) { 'command' }
|
137
|
+
let(:options) { ['--start', '2013-01-01', '--config', 'xxx'] }
|
138
|
+
it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Could not load file provided for '--config'/ }
|
139
|
+
end
|
140
|
+
|
141
|
+
context 'with command and --start and missing system --config file' do
|
142
|
+
let(:command) { 'command' }
|
143
|
+
let(:options) { ['--start', '2013-01-01'] }
|
144
|
+
before do
|
145
|
+
expect_any_instance_of(Masamune::Filesystem).to receive(:resolve_file)
|
146
|
+
end
|
147
|
+
it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /Option --config or valid system configuration file required/ }
|
148
|
+
end
|
149
|
+
|
150
|
+
context 'with command and -- --extra --args' do
|
151
|
+
let(:command) { 'command' }
|
152
|
+
let(:options) { ['--start', '2013-01-01', '--', '--extra', '--args'] }
|
153
|
+
before do
|
154
|
+
expect_any_instance_of(thor_class).to receive(:extra=).with(['--extra', '--args'])
|
155
|
+
end
|
156
|
+
it do
|
157
|
+
expect { cli_invocation }.to raise_error SystemExit
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
context 'with command and --start' do
|
162
|
+
let(:command) { 'command' }
|
163
|
+
let(:options) { ['--start', '2013-01-01'] }
|
164
|
+
it 'exits with status code 0 without error message' do
|
165
|
+
expect { cli_invocation }.to raise_error { |e|
|
166
|
+
expect(e).to be_a(SystemExit)
|
167
|
+
expect(e.status).to eq(0)
|
168
|
+
}
|
169
|
+
expect(stdout.string).to match(/\AUsing '.*' for --start/)
|
170
|
+
expect(stderr.string).to eq('')
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
context 'with command and natural language --start' do
|
175
|
+
let(:command) { 'command' }
|
176
|
+
let(:options) { ['--start', 'yesterday'] }
|
177
|
+
it 'exits with status code 0 without error message' do
|
178
|
+
expect { cli_invocation }.to raise_error { |e|
|
179
|
+
expect(e).to be_a(SystemExit)
|
180
|
+
expect(e.status).to eq(0)
|
181
|
+
}
|
182
|
+
expect(stdout.string).to match(/\AUsing '.*' for --start/)
|
183
|
+
expect(stderr.string).to eq('')
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
context 'with command that raises exception before initialization' do
|
188
|
+
let(:command) { 'command' }
|
189
|
+
let(:options) { ['--start', '2013-01-01'] }
|
190
|
+
before do
|
191
|
+
expect_any_instance_of(Logger).to receive(:error).with(/random exception/)
|
192
|
+
allow(thor_class).to receive(:dispatch).and_raise('random exception')
|
193
|
+
end
|
194
|
+
it { expect { cli_invocation }.to raise_error /random exception/ }
|
195
|
+
end
|
196
|
+
|
197
|
+
context 'with command that raises exception after initialization' do
|
198
|
+
let(:command) { 'command' }
|
199
|
+
let(:options) { ['--start', '2013-01-01'] }
|
200
|
+
before do
|
201
|
+
expect_any_instance_of(Logger).to receive(:error).with(/random exception/)
|
202
|
+
allow(thor_class).to receive(:after_initialize_invoke).and_raise('random exception')
|
203
|
+
end
|
204
|
+
it { expect { cli_invocation }.to raise_error /random exception/ }
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
context '.parse_extra' do
|
209
|
+
subject do
|
210
|
+
thor_class.parse_extra(argv)
|
211
|
+
end
|
212
|
+
|
213
|
+
context 'without --' do
|
214
|
+
let(:argv) { ['--flag', 'true'] }
|
215
|
+
it { is_expected.to eq([['--flag', 'true'],[]]) }
|
216
|
+
end
|
217
|
+
|
218
|
+
context 'with -- and no following arguments' do
|
219
|
+
let(:argv) { ['--flag', 'true', '--'] }
|
220
|
+
it { is_expected.to eq([['--flag', 'true'],[]]) }
|
221
|
+
end
|
222
|
+
|
223
|
+
context 'with -- and a single extra argument' do
|
224
|
+
let(:argv) { ['--flag', 'true', '--', '--more'] }
|
225
|
+
it { is_expected.to eq([['--flag', 'true'], ['--more']]) }
|
226
|
+
end
|
227
|
+
|
228
|
+
context 'with -- and multiple extra agruments' do
|
229
|
+
let(:argv) { ['--flag', 'true', '--', '--more', 'flag'] }
|
230
|
+
it { is_expected.to eq([['--flag', 'true'], ['--more', 'flag']]) }
|
231
|
+
end
|
232
|
+
|
233
|
+
context 'with leading -- and a single extra argument' do
|
234
|
+
let(:argv) { ['--', '--more'] }
|
235
|
+
it { is_expected.to eq([[], ['--more']]) }
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|