masamune 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/README.md +54 -0
  4. data/Rakefile +15 -0
  5. data/bin/masamune-elastic-mapreduce +4 -0
  6. data/bin/masamune-hive +4 -0
  7. data/bin/masamune-psql +4 -0
  8. data/bin/masamune-shell +4 -0
  9. data/lib/masamune.rb +56 -0
  10. data/lib/masamune/accumulate.rb +60 -0
  11. data/lib/masamune/actions.rb +38 -0
  12. data/lib/masamune/actions/data_flow.rb +131 -0
  13. data/lib/masamune/actions/date_parse.rb +75 -0
  14. data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
  15. data/lib/masamune/actions/execute.rb +52 -0
  16. data/lib/masamune/actions/filesystem.rb +37 -0
  17. data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
  18. data/lib/masamune/actions/hadoop_streaming.rb +41 -0
  19. data/lib/masamune/actions/hive.rb +74 -0
  20. data/lib/masamune/actions/postgres.rb +76 -0
  21. data/lib/masamune/actions/postgres_admin.rb +34 -0
  22. data/lib/masamune/actions/s3cmd.rb +44 -0
  23. data/lib/masamune/actions/transform.rb +89 -0
  24. data/lib/masamune/after_initialize_callbacks.rb +55 -0
  25. data/lib/masamune/cached_filesystem.rb +110 -0
  26. data/lib/masamune/commands.rb +37 -0
  27. data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
  28. data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
  29. data/lib/masamune/commands/hadoop_streaming.rb +116 -0
  30. data/lib/masamune/commands/hive.rb +178 -0
  31. data/lib/masamune/commands/interactive.rb +37 -0
  32. data/lib/masamune/commands/postgres.rb +128 -0
  33. data/lib/masamune/commands/postgres_admin.rb +72 -0
  34. data/lib/masamune/commands/postgres_common.rb +33 -0
  35. data/lib/masamune/commands/retry_with_backoff.rb +60 -0
  36. data/lib/masamune/commands/s3cmd.rb +70 -0
  37. data/lib/masamune/commands/shell.rb +202 -0
  38. data/lib/masamune/configuration.rb +195 -0
  39. data/lib/masamune/data_plan.rb +31 -0
  40. data/lib/masamune/data_plan/builder.rb +66 -0
  41. data/lib/masamune/data_plan/elem.rb +190 -0
  42. data/lib/masamune/data_plan/engine.rb +162 -0
  43. data/lib/masamune/data_plan/rule.rb +292 -0
  44. data/lib/masamune/data_plan/set.rb +176 -0
  45. data/lib/masamune/environment.rb +164 -0
  46. data/lib/masamune/filesystem.rb +567 -0
  47. data/lib/masamune/has_environment.rb +40 -0
  48. data/lib/masamune/helpers.rb +27 -0
  49. data/lib/masamune/helpers/postgres.rb +84 -0
  50. data/lib/masamune/io.rb +33 -0
  51. data/lib/masamune/last_element.rb +53 -0
  52. data/lib/masamune/method_logger.rb +41 -0
  53. data/lib/masamune/multi_io.rb +39 -0
  54. data/lib/masamune/schema.rb +36 -0
  55. data/lib/masamune/schema/catalog.rb +233 -0
  56. data/lib/masamune/schema/column.rb +527 -0
  57. data/lib/masamune/schema/dimension.rb +133 -0
  58. data/lib/masamune/schema/event.rb +121 -0
  59. data/lib/masamune/schema/fact.rb +133 -0
  60. data/lib/masamune/schema/map.rb +265 -0
  61. data/lib/masamune/schema/row.rb +133 -0
  62. data/lib/masamune/schema/store.rb +115 -0
  63. data/lib/masamune/schema/table.rb +308 -0
  64. data/lib/masamune/schema/table_reference.rb +76 -0
  65. data/lib/masamune/spec_helper.rb +23 -0
  66. data/lib/masamune/string_format.rb +34 -0
  67. data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
  68. data/lib/masamune/tasks/hive_thor.rb +55 -0
  69. data/lib/masamune/tasks/postgres_thor.rb +47 -0
  70. data/lib/masamune/tasks/shell_thor.rb +63 -0
  71. data/lib/masamune/template.rb +77 -0
  72. data/lib/masamune/thor.rb +186 -0
  73. data/lib/masamune/thor_loader.rb +38 -0
  74. data/lib/masamune/topological_hash.rb +34 -0
  75. data/lib/masamune/transform.rb +47 -0
  76. data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
  77. data/lib/masamune/transform/bulk_upsert.rb +52 -0
  78. data/lib/masamune/transform/consolidate_dimension.rb +54 -0
  79. data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
  80. data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
  81. data/lib/masamune/transform/define_event_view.hql.erb +51 -0
  82. data/lib/masamune/transform/define_event_view.rb +60 -0
  83. data/lib/masamune/transform/define_index.psql.erb +34 -0
  84. data/lib/masamune/transform/define_schema.hql.erb +23 -0
  85. data/lib/masamune/transform/define_schema.psql.erb +79 -0
  86. data/lib/masamune/transform/define_schema.rb +56 -0
  87. data/lib/masamune/transform/define_table.hql.erb +34 -0
  88. data/lib/masamune/transform/define_table.psql.erb +95 -0
  89. data/lib/masamune/transform/define_table.rb +40 -0
  90. data/lib/masamune/transform/define_unique.psql.erb +30 -0
  91. data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
  92. data/lib/masamune/transform/insert_reference_values.rb +64 -0
  93. data/lib/masamune/transform/load_dimension.rb +47 -0
  94. data/lib/masamune/transform/load_fact.rb +45 -0
  95. data/lib/masamune/transform/operator.rb +96 -0
  96. data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
  97. data/lib/masamune/transform/relabel_dimension.rb +39 -0
  98. data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
  99. data/lib/masamune/transform/rollup_fact.rb +149 -0
  100. data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
  101. data/lib/masamune/transform/snapshot_dimension.rb +74 -0
  102. data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
  103. data/lib/masamune/transform/stage_dimension.rb +83 -0
  104. data/lib/masamune/transform/stage_fact.psql.erb +80 -0
  105. data/lib/masamune/transform/stage_fact.rb +111 -0
  106. data/lib/masamune/version.rb +25 -0
  107. data/spec/fixtures/aggregate.sql.erb +25 -0
  108. data/spec/fixtures/comment.sql.erb +27 -0
  109. data/spec/fixtures/invalid.sql.erb +23 -0
  110. data/spec/fixtures/relative.sql.erb +23 -0
  111. data/spec/fixtures/simple.sql.erb +28 -0
  112. data/spec/fixtures/whitespace.sql.erb +30 -0
  113. data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
  114. data/spec/masamune/actions/execute_spec.rb +50 -0
  115. data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
  116. data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
  117. data/spec/masamune/actions/hive_spec.rb +117 -0
  118. data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
  119. data/spec/masamune/actions/postgres_spec.rb +134 -0
  120. data/spec/masamune/actions/s3cmd_spec.rb +44 -0
  121. data/spec/masamune/actions/transform_spec.rb +144 -0
  122. data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
  123. data/spec/masamune/cached_filesystem_spec.rb +167 -0
  124. data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
  125. data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
  126. data/spec/masamune/commands/hive_spec.rb +117 -0
  127. data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
  128. data/spec/masamune/commands/postgres_spec.rb +100 -0
  129. data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
  130. data/spec/masamune/commands/s3cmd_spec.rb +50 -0
  131. data/spec/masamune/commands/shell_spec.rb +101 -0
  132. data/spec/masamune/configuration_spec.rb +102 -0
  133. data/spec/masamune/data_plan/builder_spec.rb +91 -0
  134. data/spec/masamune/data_plan/elem_spec.rb +102 -0
  135. data/spec/masamune/data_plan/engine_spec.rb +356 -0
  136. data/spec/masamune/data_plan/rule_spec.rb +407 -0
  137. data/spec/masamune/data_plan/set_spec.rb +517 -0
  138. data/spec/masamune/environment_spec.rb +65 -0
  139. data/spec/masamune/filesystem_spec.rb +1421 -0
  140. data/spec/masamune/helpers/postgres_spec.rb +95 -0
  141. data/spec/masamune/schema/catalog_spec.rb +613 -0
  142. data/spec/masamune/schema/column_spec.rb +696 -0
  143. data/spec/masamune/schema/dimension_spec.rb +137 -0
  144. data/spec/masamune/schema/event_spec.rb +75 -0
  145. data/spec/masamune/schema/fact_spec.rb +117 -0
  146. data/spec/masamune/schema/map_spec.rb +593 -0
  147. data/spec/masamune/schema/row_spec.rb +28 -0
  148. data/spec/masamune/schema/store_spec.rb +49 -0
  149. data/spec/masamune/schema/table_spec.rb +395 -0
  150. data/spec/masamune/string_format_spec.rb +60 -0
  151. data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
  152. data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
  153. data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
  154. data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
  155. data/spec/masamune/template_spec.rb +77 -0
  156. data/spec/masamune/thor_spec.rb +238 -0
  157. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
  158. data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
  159. data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
  160. data/spec/masamune/transform/define_event_view_spec.rb +84 -0
  161. data/spec/masamune/transform/define_schema_spec.rb +83 -0
  162. data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
  163. data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
  164. data/spec/masamune/transform/define_table.table_spec.rb +525 -0
  165. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
  166. data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
  167. data/spec/masamune/transform/load_dimension_spec.rb +76 -0
  168. data/spec/masamune/transform/load_fact_spec.rb +89 -0
  169. data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
  170. data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
  171. data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
  172. data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
  173. data/spec/masamune/transform/stage_fact_spec.rb +204 -0
  174. data/spec/masamune_spec.rb +32 -0
  175. data/spec/spec_helper.rb +41 -0
  176. data/spec/support/masamune/example_group.rb +36 -0
  177. data/spec/support/masamune/mock_command.rb +99 -0
  178. data/spec/support/masamune/mock_delegate.rb +51 -0
  179. data/spec/support/masamune/mock_filesystem.rb +96 -0
  180. data/spec/support/masamune/thor_mute.rb +35 -0
  181. data/spec/support/rspec/example/action_example_group.rb +34 -0
  182. data/spec/support/rspec/example/task_example_group.rb +80 -0
  183. data/spec/support/rspec/example/transform_example_group.rb +36 -0
  184. data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
  185. metadata +462 -0
@@ -0,0 +1,60 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::StringFormat do
26
+ let(:instance) { Object.new.extend(described_class) }
27
+
28
+ describe '.strip_sql' do
29
+ subject { instance.strip_sql(input) }
30
+
31
+ context 'with quoted sql' do
32
+ let(:input) { %q('SELECT * FROM table;') }
33
+ it { is_expected.to eq(%q(SELECT * FROM table;)) }
34
+ end
35
+
36
+ context 'with ; terminated sql' do
37
+ let(:input) { %q(SELECT * FROM table;;) }
38
+ it { is_expected.to eq(%q(SELECT * FROM table;)) }
39
+ end
40
+
41
+ context 'with multi line sql' do
42
+ let(:input) do
43
+ <<-EOS
44
+ SELECT
45
+ *
46
+ FROM
47
+ table
48
+ ;
49
+
50
+ EOS
51
+ end
52
+ it { is_expected.to eq(%q(SELECT * FROM table;)) }
53
+ end
54
+
55
+ context 'with un-quoted sql' do
56
+ let(:input) { %q(SELECT * FROM table) }
57
+ it { is_expected.to eq(%q(SELECT * FROM table;)) }
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,57 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+ require 'thor'
25
+
26
+ require 'masamune/tasks/elastic_mapreduce_thor'
27
+
28
+ describe Masamune::Tasks::ElasticMapreduceThor do
29
+ context 'with help command ' do
30
+ let(:command) { 'help' }
31
+ it_behaves_like 'command usage'
32
+ end
33
+
34
+ context 'with -j' do
35
+ let(:options) { ['-j', 'j-XYZ'] }
36
+ it do
37
+ expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(jobflow: 'j-XYZ', extra: ['--ssh'])).once.and_return(mock_success)
38
+ cli_invocation
39
+ end
40
+ end
41
+
42
+ context 'with --jobflow' do
43
+ let(:options) { ['--jobflow=j-XYZ'] }
44
+ it do
45
+ expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(jobflow: 'j-XYZ', extra: ['--ssh'])).once.and_return(mock_success)
46
+ cli_invocation
47
+ end
48
+ end
49
+
50
+ context 'with -- --list' do
51
+ let(:options) { ['--', '--list'] }
52
+ it do
53
+ expect_any_instance_of(described_class).to receive(:elastic_mapreduce).with(hash_including(extra: ['--list'])).once.and_return(mock_success)
54
+ cli_invocation
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,75 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+ require 'thor'
25
+
26
+ require 'masamune/tasks/hive_thor'
27
+
28
+ describe Masamune::Tasks::HiveThor do
29
+ context 'with help command ' do
30
+ let(:command) { 'help' }
31
+ before do
32
+ expect_any_instance_of(described_class).to receive(:hive).never
33
+ end
34
+ it_behaves_like 'command usage'
35
+ end
36
+
37
+ context 'with command options' do
38
+ before do
39
+ expect_any_instance_of(described_class).to receive(:hive).with(exec: 'CREATE DATABASE IF NOT EXISTS masamune;', database: nil).and_return(mock_success)
40
+ expect_any_instance_of(described_class).to receive(:hive).with(file: instance_of(String)).and_return(mock_success)
41
+ end
42
+
43
+ context 'with --file' do
44
+ let(:options) { ['--file=zombo.hql'] }
45
+ it do
46
+ expect_any_instance_of(described_class).to receive(:hive).with(hash_including(file: File.expand_path('zombo.hql'))).once.and_return(mock_success)
47
+ cli_invocation
48
+ end
49
+ end
50
+
51
+ context 'with --output' do
52
+ let(:options) { ['--output=report.txt'] }
53
+ it do
54
+ expect_any_instance_of(described_class).to receive(:hive).with(hash_including(output: File.expand_path('report.txt'))).once.and_return(mock_success)
55
+ cli_invocation
56
+ end
57
+ end
58
+
59
+ context 'with --variables=YEAR:2015 MONTH:1' do
60
+ let(:options) { ['--variables=YEAR:2015', 'MONTH:1'] }
61
+ it do
62
+ expect_any_instance_of(described_class).to receive(:hive).with(hash_including(variables: { 'YEAR' => '2015', 'MONTH' => '1'})).once.and_return(mock_success)
63
+ cli_invocation
64
+ end
65
+ end
66
+
67
+ context 'with -X YEAR:2015 MONTH:1' do
68
+ let(:options) { ['-X', 'YEAR:2015', 'MONTH:1'] }
69
+ it do
70
+ expect_any_instance_of(described_class).to receive(:hive).with(hash_including(variables: { 'YEAR' => '2015', 'MONTH' => '1'})).once.and_return(mock_success)
71
+ cli_invocation
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,42 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+ require 'thor'
25
+
26
+ require 'masamune/tasks/postgres_thor'
27
+
28
+ describe Masamune::Tasks::PostgresThor do
29
+ context 'with help command ' do
30
+ let(:command) { 'help' }
31
+ it_behaves_like 'command usage'
32
+ end
33
+
34
+ context 'with --file' do
35
+ let(:options) { ['--file=zombo.hql'] }
36
+ it do
37
+ expect_any_instance_of(described_class).to receive(:postgres).with(file: instance_of(String)).once.and_return(mock_success)
38
+ expect_any_instance_of(described_class).to receive(:postgres).with(hash_including(file: 'zombo.hql')).once.and_return(mock_success)
39
+ cli_invocation
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,51 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+ require 'thor'
25
+
26
+ require 'masamune/tasks/shell_thor'
27
+
28
+ describe Masamune::Tasks::ShellThor do
29
+ context 'with help command ' do
30
+ let(:command) { 'help' }
31
+ it_behaves_like 'command usage'
32
+ end
33
+
34
+ context 'with no arguments' do
35
+ it do
36
+ expect(Pry).to receive(:start)
37
+ cli_invocation
38
+ end
39
+ end
40
+
41
+ context 'with --dump' do
42
+ let(:options) { ['--dump'] }
43
+
44
+ it 'exits with status code 0 and prints catalog' do
45
+ expect { cli_invocation }.to raise_error { |e|
46
+ expect(e).to be_a(SystemExit)
47
+ expect(e.status).to eq(0)
48
+ }
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,77 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Template do
26
+ describe '.render_to_file' do
27
+ let(:parameters) { {} }
28
+
29
+ subject do
30
+ File.read(described_class.render_to_file(template, parameters))
31
+ end
32
+
33
+ context 'with invalid file' do
34
+ let(:template) { 'not_a_file.txt' }
35
+ it { expect { subject }.to raise_error IOError }
36
+ end
37
+
38
+ context 'with invalid template' do
39
+ let(:template) { File.expand_path('../../fixtures/invalid.sql.erb', __FILE__) }
40
+ it { expect { subject }.to raise_error IOError, /not_found.sql.erb/ }
41
+ end
42
+
43
+ context 'with simple template' do
44
+ let(:template) { File.expand_path('../../fixtures/simple.sql.erb', __FILE__) }
45
+ let(:parameters) { {table: 'zombo'} }
46
+
47
+ it { is_expected.to eq("SELECT * FROM zombo;\n") }
48
+ end
49
+
50
+ context 'with template with comments' do
51
+ let(:template) { File.expand_path('../../fixtures/comment.sql.erb', __FILE__) }
52
+ it { is_expected.to eq("SELECT 1;\n") }
53
+ end
54
+
55
+ context 'with template with unnecessary whitespace' do
56
+ let(:template) { File.expand_path('../../fixtures/whitespace.sql.erb', __FILE__) }
57
+
58
+ it { is_expected.to eq("SELECT 1;\n\nSELECT 2;\n") }
59
+ end
60
+
61
+ context 'with aggregate template' do
62
+ let(:template) { File.expand_path('../../fixtures/aggregate.sql.erb', __FILE__) }
63
+
64
+ it do is_expected.to eq <<-EOS.gsub(/^\s*/,'')
65
+ SHOW TABLES;
66
+ SELECT * FROM foo;
67
+ SELECT * FROM bar;
68
+ EOS
69
+ end
70
+ end
71
+
72
+ context 'with aggregate template with relative path' do
73
+ let(:template) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'relative.sql.erb') }
74
+ it { is_expected.to eq("SELECT * FROM relative;\n") }
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,238 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+ require 'thor'
25
+
26
+ describe Masamune::Thor do
27
+ let(:thor_class) do
28
+ Class.new(Thor) do
29
+ include Masamune::Thor
30
+ include Masamune::Actions::DataFlow
31
+
32
+ desc 'command', 'command'
33
+ target path: "target/%Y-%m-%d"
34
+ source path: "source/%Y%m%d*.log"
35
+ method_option :zombo, desc: 'Anything is possible'
36
+ def command
37
+ # NOP
38
+ end
39
+
40
+ desc 'other', 'other'
41
+ skip
42
+ def other
43
+ # NOP
44
+ end
45
+ end
46
+ end
47
+
48
+ before do
49
+ allow_any_instance_of(thor_class).to receive(:top_level?).and_return(true)
50
+ end
51
+
52
+ context 'CLI' do
53
+ context 'ouside of top level' do
54
+ let(:command) { 'command' }
55
+ let(:options) { ['--start', '2013-01-01'] }
56
+
57
+ before do
58
+ allow_any_instance_of(thor_class).to receive(:top_level?).and_return(false)
59
+ end
60
+
61
+ it 'continues execution' do
62
+ expect { cli_invocation }.to_not raise_error
63
+ end
64
+ end
65
+
66
+ context 'without command' do
67
+ it_behaves_like 'general usage'
68
+ end
69
+
70
+ context 'with help command ' do
71
+ let(:command) { 'help' }
72
+ it_behaves_like 'general usage'
73
+ end
74
+
75
+ context 'with --help option' do
76
+ let(:command) { 'command' }
77
+ let(:options) { ['--help'] }
78
+ it_behaves_like 'command usage'
79
+ end
80
+
81
+ context 'with help subcommand ' do
82
+ let(:command) { 'help' }
83
+ let(:options) { ['command'] }
84
+ it_behaves_like 'command usage'
85
+ end
86
+
87
+ context 'with command and --version' do
88
+ let(:command) { 'command' }
89
+ let(:options) { ['--version'] }
90
+ it 'exits with status code 0 and prints version' do
91
+ expect { cli_invocation }.to raise_error { |e|
92
+ expect(e).to be_a(SystemExit)
93
+ expect(e.status).to eq(0)
94
+ }
95
+ expect(stdout.string).to match(/\Amasamune/)
96
+ expect(stderr.string).to be_blank
97
+ end
98
+ end
99
+
100
+ context 'with command and no input options' do
101
+ let(:command) { 'command' }
102
+ it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /No value provided for required options '--start'/ }
103
+ end
104
+
105
+ context 'with command and invalid --start' do
106
+ let(:command) { 'command' }
107
+ let(:options) { ['--start', 'xxx'] }
108
+ it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--start'; got/ }
109
+ end
110
+
111
+ context 'with command and invalid --stop' do
112
+ let(:command) { 'command' }
113
+ let(:options) { ['--start', '2013-01-01', '--stop', 'xxx'] }
114
+ it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected date time value for '--stop'; got/ }
115
+ end
116
+
117
+ context 'with command and invalid --sources' do
118
+ let(:command) { 'command' }
119
+ let(:options) { ['--sources', 'foo'] }
120
+ it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--sources'; got/ }
121
+ end
122
+
123
+ context 'with command and invalid --targets' do
124
+ let(:command) { 'command' }
125
+ let(:options) { ['--targets', 'foo'] }
126
+ it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Expected file value for '--targets'; got/ }
127
+ end
128
+
129
+ context 'with command and both --sources and --targets' do
130
+ let(:command) { 'command' }
131
+ let(:options) { ['--sources', 'sources', '--targets', 'targets'] }
132
+ it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Cannot specify both option '--sources' and option '--targets'/ }
133
+ end
134
+
135
+ context 'with command and --start and bad --config file' do
136
+ let(:command) { 'command' }
137
+ let(:options) { ['--start', '2013-01-01', '--config', 'xxx'] }
138
+ it { expect { cli_invocation }.to raise_error Thor::MalformattedArgumentError, /Could not load file provided for '--config'/ }
139
+ end
140
+
141
+ context 'with command and --start and missing system --config file' do
142
+ let(:command) { 'command' }
143
+ let(:options) { ['--start', '2013-01-01'] }
144
+ before do
145
+ expect_any_instance_of(Masamune::Filesystem).to receive(:resolve_file)
146
+ end
147
+ it { expect { cli_invocation }.to raise_error Thor::RequiredArgumentMissingError, /Option --config or valid system configuration file required/ }
148
+ end
149
+
150
+ context 'with command and -- --extra --args' do
151
+ let(:command) { 'command' }
152
+ let(:options) { ['--start', '2013-01-01', '--', '--extra', '--args'] }
153
+ before do
154
+ expect_any_instance_of(thor_class).to receive(:extra=).with(['--extra', '--args'])
155
+ end
156
+ it do
157
+ expect { cli_invocation }.to raise_error SystemExit
158
+ end
159
+ end
160
+
161
+ context 'with command and --start' do
162
+ let(:command) { 'command' }
163
+ let(:options) { ['--start', '2013-01-01'] }
164
+ it 'exits with status code 0 without error message' do
165
+ expect { cli_invocation }.to raise_error { |e|
166
+ expect(e).to be_a(SystemExit)
167
+ expect(e.status).to eq(0)
168
+ }
169
+ expect(stdout.string).to match(/\AUsing '.*' for --start/)
170
+ expect(stderr.string).to eq('')
171
+ end
172
+ end
173
+
174
+ context 'with command and natural language --start' do
175
+ let(:command) { 'command' }
176
+ let(:options) { ['--start', 'yesterday'] }
177
+ it 'exits with status code 0 without error message' do
178
+ expect { cli_invocation }.to raise_error { |e|
179
+ expect(e).to be_a(SystemExit)
180
+ expect(e.status).to eq(0)
181
+ }
182
+ expect(stdout.string).to match(/\AUsing '.*' for --start/)
183
+ expect(stderr.string).to eq('')
184
+ end
185
+ end
186
+
187
+ context 'with command that raises exception before initialization' do
188
+ let(:command) { 'command' }
189
+ let(:options) { ['--start', '2013-01-01'] }
190
+ before do
191
+ expect_any_instance_of(Logger).to receive(:error).with(/random exception/)
192
+ allow(thor_class).to receive(:dispatch).and_raise('random exception')
193
+ end
194
+ it { expect { cli_invocation }.to raise_error /random exception/ }
195
+ end
196
+
197
+ context 'with command that raises exception after initialization' do
198
+ let(:command) { 'command' }
199
+ let(:options) { ['--start', '2013-01-01'] }
200
+ before do
201
+ expect_any_instance_of(Logger).to receive(:error).with(/random exception/)
202
+ allow(thor_class).to receive(:after_initialize_invoke).and_raise('random exception')
203
+ end
204
+ it { expect { cli_invocation }.to raise_error /random exception/ }
205
+ end
206
+ end
207
+
208
+ context '.parse_extra' do
209
+ subject do
210
+ thor_class.parse_extra(argv)
211
+ end
212
+
213
+ context 'without --' do
214
+ let(:argv) { ['--flag', 'true'] }
215
+ it { is_expected.to eq([['--flag', 'true'],[]]) }
216
+ end
217
+
218
+ context 'with -- and no following arguments' do
219
+ let(:argv) { ['--flag', 'true', '--'] }
220
+ it { is_expected.to eq([['--flag', 'true'],[]]) }
221
+ end
222
+
223
+ context 'with -- and a single extra argument' do
224
+ let(:argv) { ['--flag', 'true', '--', '--more'] }
225
+ it { is_expected.to eq([['--flag', 'true'], ['--more']]) }
226
+ end
227
+
228
+ context 'with -- and multiple extra agruments' do
229
+ let(:argv) { ['--flag', 'true', '--', '--more', 'flag'] }
230
+ it { is_expected.to eq([['--flag', 'true'], ['--more', 'flag']]) }
231
+ end
232
+
233
+ context 'with leading -- and a single extra argument' do
234
+ let(:argv) { ['--', '--more'] }
235
+ it { is_expected.to eq([[], ['--more']]) }
236
+ end
237
+ end
238
+ end