masamune 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/README.md +54 -0
  4. data/Rakefile +15 -0
  5. data/bin/masamune-elastic-mapreduce +4 -0
  6. data/bin/masamune-hive +4 -0
  7. data/bin/masamune-psql +4 -0
  8. data/bin/masamune-shell +4 -0
  9. data/lib/masamune.rb +56 -0
  10. data/lib/masamune/accumulate.rb +60 -0
  11. data/lib/masamune/actions.rb +38 -0
  12. data/lib/masamune/actions/data_flow.rb +131 -0
  13. data/lib/masamune/actions/date_parse.rb +75 -0
  14. data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
  15. data/lib/masamune/actions/execute.rb +52 -0
  16. data/lib/masamune/actions/filesystem.rb +37 -0
  17. data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
  18. data/lib/masamune/actions/hadoop_streaming.rb +41 -0
  19. data/lib/masamune/actions/hive.rb +74 -0
  20. data/lib/masamune/actions/postgres.rb +76 -0
  21. data/lib/masamune/actions/postgres_admin.rb +34 -0
  22. data/lib/masamune/actions/s3cmd.rb +44 -0
  23. data/lib/masamune/actions/transform.rb +89 -0
  24. data/lib/masamune/after_initialize_callbacks.rb +55 -0
  25. data/lib/masamune/cached_filesystem.rb +110 -0
  26. data/lib/masamune/commands.rb +37 -0
  27. data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
  28. data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
  29. data/lib/masamune/commands/hadoop_streaming.rb +116 -0
  30. data/lib/masamune/commands/hive.rb +178 -0
  31. data/lib/masamune/commands/interactive.rb +37 -0
  32. data/lib/masamune/commands/postgres.rb +128 -0
  33. data/lib/masamune/commands/postgres_admin.rb +72 -0
  34. data/lib/masamune/commands/postgres_common.rb +33 -0
  35. data/lib/masamune/commands/retry_with_backoff.rb +60 -0
  36. data/lib/masamune/commands/s3cmd.rb +70 -0
  37. data/lib/masamune/commands/shell.rb +202 -0
  38. data/lib/masamune/configuration.rb +195 -0
  39. data/lib/masamune/data_plan.rb +31 -0
  40. data/lib/masamune/data_plan/builder.rb +66 -0
  41. data/lib/masamune/data_plan/elem.rb +190 -0
  42. data/lib/masamune/data_plan/engine.rb +162 -0
  43. data/lib/masamune/data_plan/rule.rb +292 -0
  44. data/lib/masamune/data_plan/set.rb +176 -0
  45. data/lib/masamune/environment.rb +164 -0
  46. data/lib/masamune/filesystem.rb +567 -0
  47. data/lib/masamune/has_environment.rb +40 -0
  48. data/lib/masamune/helpers.rb +27 -0
  49. data/lib/masamune/helpers/postgres.rb +84 -0
  50. data/lib/masamune/io.rb +33 -0
  51. data/lib/masamune/last_element.rb +53 -0
  52. data/lib/masamune/method_logger.rb +41 -0
  53. data/lib/masamune/multi_io.rb +39 -0
  54. data/lib/masamune/schema.rb +36 -0
  55. data/lib/masamune/schema/catalog.rb +233 -0
  56. data/lib/masamune/schema/column.rb +527 -0
  57. data/lib/masamune/schema/dimension.rb +133 -0
  58. data/lib/masamune/schema/event.rb +121 -0
  59. data/lib/masamune/schema/fact.rb +133 -0
  60. data/lib/masamune/schema/map.rb +265 -0
  61. data/lib/masamune/schema/row.rb +133 -0
  62. data/lib/masamune/schema/store.rb +115 -0
  63. data/lib/masamune/schema/table.rb +308 -0
  64. data/lib/masamune/schema/table_reference.rb +76 -0
  65. data/lib/masamune/spec_helper.rb +23 -0
  66. data/lib/masamune/string_format.rb +34 -0
  67. data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
  68. data/lib/masamune/tasks/hive_thor.rb +55 -0
  69. data/lib/masamune/tasks/postgres_thor.rb +47 -0
  70. data/lib/masamune/tasks/shell_thor.rb +63 -0
  71. data/lib/masamune/template.rb +77 -0
  72. data/lib/masamune/thor.rb +186 -0
  73. data/lib/masamune/thor_loader.rb +38 -0
  74. data/lib/masamune/topological_hash.rb +34 -0
  75. data/lib/masamune/transform.rb +47 -0
  76. data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
  77. data/lib/masamune/transform/bulk_upsert.rb +52 -0
  78. data/lib/masamune/transform/consolidate_dimension.rb +54 -0
  79. data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
  80. data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
  81. data/lib/masamune/transform/define_event_view.hql.erb +51 -0
  82. data/lib/masamune/transform/define_event_view.rb +60 -0
  83. data/lib/masamune/transform/define_index.psql.erb +34 -0
  84. data/lib/masamune/transform/define_schema.hql.erb +23 -0
  85. data/lib/masamune/transform/define_schema.psql.erb +79 -0
  86. data/lib/masamune/transform/define_schema.rb +56 -0
  87. data/lib/masamune/transform/define_table.hql.erb +34 -0
  88. data/lib/masamune/transform/define_table.psql.erb +95 -0
  89. data/lib/masamune/transform/define_table.rb +40 -0
  90. data/lib/masamune/transform/define_unique.psql.erb +30 -0
  91. data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
  92. data/lib/masamune/transform/insert_reference_values.rb +64 -0
  93. data/lib/masamune/transform/load_dimension.rb +47 -0
  94. data/lib/masamune/transform/load_fact.rb +45 -0
  95. data/lib/masamune/transform/operator.rb +96 -0
  96. data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
  97. data/lib/masamune/transform/relabel_dimension.rb +39 -0
  98. data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
  99. data/lib/masamune/transform/rollup_fact.rb +149 -0
  100. data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
  101. data/lib/masamune/transform/snapshot_dimension.rb +74 -0
  102. data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
  103. data/lib/masamune/transform/stage_dimension.rb +83 -0
  104. data/lib/masamune/transform/stage_fact.psql.erb +80 -0
  105. data/lib/masamune/transform/stage_fact.rb +111 -0
  106. data/lib/masamune/version.rb +25 -0
  107. data/spec/fixtures/aggregate.sql.erb +25 -0
  108. data/spec/fixtures/comment.sql.erb +27 -0
  109. data/spec/fixtures/invalid.sql.erb +23 -0
  110. data/spec/fixtures/relative.sql.erb +23 -0
  111. data/spec/fixtures/simple.sql.erb +28 -0
  112. data/spec/fixtures/whitespace.sql.erb +30 -0
  113. data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
  114. data/spec/masamune/actions/execute_spec.rb +50 -0
  115. data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
  116. data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
  117. data/spec/masamune/actions/hive_spec.rb +117 -0
  118. data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
  119. data/spec/masamune/actions/postgres_spec.rb +134 -0
  120. data/spec/masamune/actions/s3cmd_spec.rb +44 -0
  121. data/spec/masamune/actions/transform_spec.rb +144 -0
  122. data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
  123. data/spec/masamune/cached_filesystem_spec.rb +167 -0
  124. data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
  125. data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
  126. data/spec/masamune/commands/hive_spec.rb +117 -0
  127. data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
  128. data/spec/masamune/commands/postgres_spec.rb +100 -0
  129. data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
  130. data/spec/masamune/commands/s3cmd_spec.rb +50 -0
  131. data/spec/masamune/commands/shell_spec.rb +101 -0
  132. data/spec/masamune/configuration_spec.rb +102 -0
  133. data/spec/masamune/data_plan/builder_spec.rb +91 -0
  134. data/spec/masamune/data_plan/elem_spec.rb +102 -0
  135. data/spec/masamune/data_plan/engine_spec.rb +356 -0
  136. data/spec/masamune/data_plan/rule_spec.rb +407 -0
  137. data/spec/masamune/data_plan/set_spec.rb +517 -0
  138. data/spec/masamune/environment_spec.rb +65 -0
  139. data/spec/masamune/filesystem_spec.rb +1421 -0
  140. data/spec/masamune/helpers/postgres_spec.rb +95 -0
  141. data/spec/masamune/schema/catalog_spec.rb +613 -0
  142. data/spec/masamune/schema/column_spec.rb +696 -0
  143. data/spec/masamune/schema/dimension_spec.rb +137 -0
  144. data/spec/masamune/schema/event_spec.rb +75 -0
  145. data/spec/masamune/schema/fact_spec.rb +117 -0
  146. data/spec/masamune/schema/map_spec.rb +593 -0
  147. data/spec/masamune/schema/row_spec.rb +28 -0
  148. data/spec/masamune/schema/store_spec.rb +49 -0
  149. data/spec/masamune/schema/table_spec.rb +395 -0
  150. data/spec/masamune/string_format_spec.rb +60 -0
  151. data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
  152. data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
  153. data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
  154. data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
  155. data/spec/masamune/template_spec.rb +77 -0
  156. data/spec/masamune/thor_spec.rb +238 -0
  157. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
  158. data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
  159. data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
  160. data/spec/masamune/transform/define_event_view_spec.rb +84 -0
  161. data/spec/masamune/transform/define_schema_spec.rb +83 -0
  162. data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
  163. data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
  164. data/spec/masamune/transform/define_table.table_spec.rb +525 -0
  165. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
  166. data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
  167. data/spec/masamune/transform/load_dimension_spec.rb +76 -0
  168. data/spec/masamune/transform/load_fact_spec.rb +89 -0
  169. data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
  170. data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
  171. data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
  172. data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
  173. data/spec/masamune/transform/stage_fact_spec.rb +204 -0
  174. data/spec/masamune_spec.rb +32 -0
  175. data/spec/spec_helper.rb +41 -0
  176. data/spec/support/masamune/example_group.rb +36 -0
  177. data/spec/support/masamune/mock_command.rb +99 -0
  178. data/spec/support/masamune/mock_delegate.rb +51 -0
  179. data/spec/support/masamune/mock_filesystem.rb +96 -0
  180. data/spec/support/masamune/thor_mute.rb +35 -0
  181. data/spec/support/rspec/example/action_example_group.rb +34 -0
  182. data/spec/support/rspec/example/task_example_group.rb +80 -0
  183. data/spec/support/rspec/example/transform_example_group.rb +36 -0
  184. data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
  185. metadata +462 -0
@@ -0,0 +1,58 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::PostgresAdmin do
26
+ let(:klass) do
27
+ Class.new do
28
+ include Masamune::HasEnvironment
29
+ include Masamune::Actions::PostgresAdmin
30
+ end
31
+ end
32
+
33
+ let(:instance) { klass.new }
34
+
35
+ describe '.postgres_admin' do
36
+ subject { instance.postgres_admin(action: action, database: 'zombo') }
37
+
38
+ context 'with :action :create' do
39
+ let(:action) { :create }
40
+
41
+ before do
42
+ mock_command(/\Acreatedb/, mock_success)
43
+ end
44
+
45
+ it { is_expected.to be_success }
46
+ end
47
+
48
+ context 'with :action :drop' do
49
+ let(:action) { :drop }
50
+
51
+ before do
52
+ mock_command(/\Adropdb/, mock_success)
53
+ end
54
+
55
+ it { is_expected.to be_success }
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,134 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::Postgres do
26
+ let(:filesystem) { Masamune::MockFilesystem.new }
27
+
28
+ let(:klass) do
29
+ Class.new do
30
+ include Masamune::HasEnvironment
31
+ include Masamune::AfterInitializeCallbacks
32
+ include Masamune::Actions::Postgres
33
+ end
34
+ end
35
+
36
+ let(:instance) { klass.new }
37
+ let(:configuration) { {database: 'test'} }
38
+ let(:postgres_helper) { double }
39
+ let(:catalog) { double }
40
+
41
+ before do
42
+ allow(instance).to receive(:filesystem) { filesystem }
43
+ allow(instance).to receive(:catalog) { catalog }
44
+ allow(instance).to receive(:postgres_helper) { postgres_helper }
45
+ allow(instance).to receive_message_chain(:configuration, :postgres).and_return(configuration)
46
+ allow(instance).to receive_message_chain(:configuration, :with_quiet).and_yield
47
+ allow(instance).to receive_message_chain(:define_schema, :to_file) { 'catalog.psql' }
48
+ end
49
+
50
+ describe '.postgres' do
51
+ before do
52
+ mock_command(/\Apsql/, mock_success)
53
+ end
54
+
55
+ subject { instance.postgres }
56
+
57
+ it { is_expected.to be_success }
58
+ end
59
+
60
+ describe '.after_initialize' do
61
+ let(:options) { {} }
62
+ let(:setup_files) { [] }
63
+ let(:schema_files) { [] }
64
+ let(:configuration) { {database: 'test', setup_files: setup_files, schema_files: schema_files} }
65
+
66
+ subject(:after_initialize_invoke) do
67
+ instance.after_initialize_invoke(options)
68
+ end
69
+
70
+ context 'when database does not exist' do
71
+ before do
72
+ expect(postgres_helper).to receive(:database_exists?).and_return(false)
73
+ expect(instance).to receive(:postgres_admin).with(action: :create, database: 'test', safe: true).once
74
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
75
+ after_initialize_invoke
76
+ end
77
+ it 'should call posgres_admin once' do; end
78
+ end
79
+
80
+ context 'when database exists' do
81
+ before do
82
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
83
+ expect(instance).to receive(:postgres_admin).never
84
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
85
+ after_initialize_invoke
86
+ end
87
+ it 'should not call postgres_admin' do; end
88
+ end
89
+
90
+ context 'when setup_files are configured' do
91
+ let(:setup_files) { ['setup.psql'] }
92
+ before do
93
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
94
+ expect(instance).to receive(:postgres).with(file: setup_files.first).once
95
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
96
+ after_initialize_invoke
97
+ end
98
+ it 'should call postgres with setup_files' do; end
99
+ end
100
+
101
+ context 'when schema_files are configured' do
102
+ let(:schema_files) { ['schema.psql'] }
103
+ before do
104
+ filesystem.touch!(*schema_files)
105
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
106
+ expect(instance).to receive(:postgres).once
107
+ after_initialize_invoke
108
+ end
109
+ it 'should call postgres with schema_files' do; end
110
+ end
111
+
112
+ context 'when schema_files that are globs are configured' do
113
+ let(:schema_files) { ['schema*.psql'] }
114
+ before do
115
+ filesystem.touch!('schema_1.psql', 'schema_2.psql')
116
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
117
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
118
+ after_initialize_invoke
119
+ end
120
+ it 'should call postgres with schema_files' do; end
121
+ end
122
+
123
+ context 'when ruby schema_files configured' do
124
+ let(:schema_files) { ['schema.rb'] }
125
+ before do
126
+ filesystem.touch!('schema.rb')
127
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
128
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
129
+ after_initialize_invoke
130
+ end
131
+ it 'should call postgres with schema_files' do; end
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,44 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::S3Cmd do
26
+ let(:klass) do
27
+ Class.new do
28
+ include Masamune::HasEnvironment
29
+ include Masamune::Actions::S3Cmd
30
+ end
31
+ end
32
+
33
+ let(:instance) { klass.new }
34
+
35
+ describe '.s3cmd' do
36
+ before do
37
+ mock_command(/\As3cmd/, mock_success)
38
+ end
39
+
40
+ subject { instance.s3cmd 'ls', 's3://fake-bucket' }
41
+
42
+ it { is_expected.to be_success }
43
+ end
44
+ end
@@ -0,0 +1,144 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::Transform do
26
+ let(:environment) { double }
27
+ let(:catalog) { Masamune::Schema::Catalog.new(environment) }
28
+
29
+ before do
30
+ catalog.schema :postgres do
31
+ dimension 'date', type: :date do
32
+ column 'date_id', type: :integer, unique: true, index: true, natural_key: true
33
+ column 'date_epoch', type: :integer
34
+ column 'month_epoch', type: :integer
35
+ column 'year_epoch', type: :integer
36
+ end
37
+
38
+ dimension 'user', type: :four do
39
+ column 'tenant_id', type: :integer, index: true
40
+ column 'user_id', type: :integer, index: true, surrogate_key: true
41
+ end
42
+
43
+ file 'user' do
44
+ column 'id', type: :integer
45
+ column 'tenant_id', type: :integer
46
+ column 'updated_at', type: :timestamp
47
+ end
48
+
49
+ map from: postgres.user_file, to: postgres.user_dimension do |row|
50
+ {
51
+ user_id: row[:id],
52
+ tenant_id: row[:tenant_id],
53
+ source_kind: 'users',
54
+ start_at: row[:updated_at],
55
+ delta: 1
56
+ }
57
+ end
58
+
59
+ fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
60
+ references :date
61
+ references :user
62
+
63
+ measure 'total', type: :integer
64
+ end
65
+
66
+ file 'visits_hourly' do
67
+ column 'user.tenant_id', type: :integer
68
+ column 'user.user_id', type: :integer
69
+ column 'time_key', type: :integer
70
+ column 'total', type: :integer
71
+ end
72
+ end
73
+ end
74
+
75
+ let(:source_file) { Tempfile.new('masamune') }
76
+
77
+ let(:klass) do
78
+ Class.new do
79
+ include Masamune::HasEnvironment
80
+ include Masamune::Actions::Transform
81
+ end
82
+ end
83
+
84
+ let(:instance) { klass.new }
85
+ let(:postgres) { catalog.postgres }
86
+
87
+ before do
88
+ instance.environment = Masamune::ExampleGroup
89
+ end
90
+
91
+ describe '.load_dimension' do
92
+ before do
93
+ mock_command(/\Apsql/, mock_success)
94
+ end
95
+
96
+ subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
97
+
98
+ it { is_expected.to be_success }
99
+ end
100
+
101
+ describe '.relabel_dimension' do
102
+ before do
103
+ mock_command(/\Apsql/, mock_success)
104
+ end
105
+
106
+ subject { instance.relabel_dimension(postgres.user_dimension) }
107
+
108
+ it { is_expected.to be_success }
109
+ end
110
+
111
+ describe '.consolidate_dimension' do
112
+ before do
113
+ mock_command(/\Apsql/, mock_success)
114
+ end
115
+
116
+ subject { instance.consolidate_dimension(postgres.user_dimension) }
117
+
118
+ it { is_expected.to be_success }
119
+ end
120
+
121
+ describe '.load_fact' do
122
+ let(:date) { DateTime.civil(2014, 8) }
123
+
124
+ before do
125
+ mock_command(/\Apsql/, mock_success)
126
+ end
127
+
128
+ subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
129
+
130
+ it { is_expected.to be_success }
131
+ end
132
+
133
+ describe '.rollup_fact' do
134
+ let(:date) { DateTime.civil(2014, 8) }
135
+
136
+ before do
137
+ mock_command(/\Apsql/, mock_success)
138
+ end
139
+
140
+ subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date) }
141
+
142
+ it { is_expected.to be_success }
143
+ end
144
+ end
@@ -0,0 +1,61 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::AfterInitializeCallbacks do
26
+ let(:klass) do
27
+ Class.new do
28
+ include Masamune::AfterInitializeCallbacks
29
+
30
+ def first_callback; end
31
+ def early_callback; end
32
+ def default_callback; end
33
+ def unknown_callback; end
34
+ def later_callback; end
35
+ def final_callback; end
36
+
37
+ after_initialize(:first) { |o| o.first_callback }
38
+ after_initialize(:early) { |o| o.early_callback }
39
+ after_initialize(:default) { |o| o.default_callback }
40
+ after_initialize(:unknown) { |o| o.unknown_callback }
41
+ after_initialize(:later) { |o| o.later_callback }
42
+ after_initialize(:final) { |o| o.final_callback}
43
+ end
44
+ end
45
+
46
+ let(:instance) { klass.new }
47
+
48
+ describe '.after_initialize_invoke' do
49
+ before do
50
+ expect(instance).to receive(:first_callback).once.ordered
51
+ expect(instance).to receive(:early_callback).once.ordered
52
+ expect(instance).to receive(:default_callback).once.ordered
53
+ expect(instance).to receive(:unknown_callback).once.ordered
54
+ expect(instance).to receive(:later_callback).once.ordered
55
+ expect(instance).to receive(:final_callback).once.ordered
56
+ instance.after_initialize_invoke
57
+ end
58
+
59
+ it 'should call callbacks in priority order' do; end
60
+ end
61
+ end
@@ -0,0 +1,167 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::CachedFilesystem do
26
+ let(:filesystem) { Masamune::MockFilesystem.new }
27
+ let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
28
+
29
+ context 'when path is present, top down traversal' do
30
+ before do
31
+ filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
32
+ expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
33
+ expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
34
+ expect(filesystem).to receive(:glob_stat).with('/*').never
35
+ end
36
+
37
+ it 'calls Filesystem#glob_stat once for multiple calls' do
38
+ expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
39
+ expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
40
+ expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
41
+ expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
42
+ expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
43
+ expect(cached_filesystem.glob('/a/b/c/*')).not_to be_empty
44
+ expect(cached_filesystem.glob('/a/b/c/*.txt')).not_to be_empty
45
+ expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
46
+ expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
47
+ expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
48
+ expect(cached_filesystem.stat('/a/b/c/4.txt')).to be_nil
49
+ expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
50
+ expect(cached_filesystem.stat('/a/b')).to_not be_nil
51
+ expect(cached_filesystem.stat('/a')).to_not be_nil
52
+ end
53
+ end
54
+
55
+ context 'when path is present, bottom up traversal' do
56
+ before do
57
+ filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
58
+ expect(filesystem).to receive(:glob_stat).with('/a/*').once.and_call_original
59
+ expect(filesystem).to receive(:glob_stat).with('/*').never
60
+ end
61
+
62
+ it 'calls Filesystem#glob_stat once for multiple calls' do
63
+ expect(cached_filesystem.glob('/a/b/*')).to eq(['/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt'])
64
+ expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
65
+ expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
66
+ expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
67
+ expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
68
+ expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
69
+ expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
70
+ expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
71
+ expect(cached_filesystem.stat('/a/b/c/4.txt')).to be_nil
72
+ expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
73
+ expect(cached_filesystem.stat('/a/b')).to_not be_nil
74
+ expect(cached_filesystem.stat('/a')).to_not be_nil
75
+ end
76
+ end
77
+
78
+ context 'when path is present, checking for similar non existant paths' do
79
+ before do
80
+ filesystem.touch!('/y=2013/m=1/d=22/00000')
81
+ expect(filesystem).to receive(:glob_stat).with('/y=2013/m=1/*').once.and_call_original
82
+ expect(filesystem).to receive(:glob_stat).with('/y=2013/*').once.and_call_original
83
+ expect(filesystem).to receive(:glob_stat).with('/*').never
84
+ end
85
+
86
+ it 'calls Filesystem#glob_stat once for multiple calls' do
87
+ expect(cached_filesystem.exists?('/y=2013/m=1/d=22/00000')).to eq(true)
88
+ expect(cached_filesystem.exists?('/y=2013/m=1/d=22')).to eq(true)
89
+ expect(cached_filesystem.exists?('/y=2013/m=1/d=2')).to eq(false)
90
+ expect(cached_filesystem.glob('/y=2013/m=1/*')).not_to be_empty
91
+ expect(cached_filesystem.glob('/y=2013/m=1/d=22/*')).not_to be_empty
92
+ expect(cached_filesystem.stat('/y=2013/m=1/d=22/00000')).not_to be_nil
93
+ expect(cached_filesystem.stat('/y=2013/m=1/d=22')).not_to be_nil
94
+ expect(cached_filesystem.stat('/y=2013/m=1')).not_to be_nil
95
+ expect(cached_filesystem.stat('/y=2013')).not_to be_nil
96
+ end
97
+ end
98
+
99
+ context 'when path is present, checking for similar existing paths' do
100
+ before do
101
+ filesystem.touch!('/logs/box1_123.txt', '/logs/box2_123.txt', '/logs/box3_123.txt')
102
+ expect(filesystem).to receive(:glob_stat).with('/logs/*').once.and_call_original
103
+ expect(filesystem).to receive(:glob_stat).with('/*').never
104
+ end
105
+
106
+ it 'calls Filesystem#glob_stat once for multiple calls' do
107
+ expect(cached_filesystem.exists?('/logs/box1_123.txt')).to eq(true)
108
+ expect(cached_filesystem.exists?('/logs/box1_456.txt')).to eq(false)
109
+ expect(cached_filesystem.exists?('/logs/box2_123.txt')).to eq(true)
110
+ expect(cached_filesystem.exists?('/logs/box2_456.txt')).to eq(false)
111
+ expect(cached_filesystem.exists?('/logs/box3_123.txt')).to eq(true)
112
+ expect(cached_filesystem.exists?('/logs/box3_456.txt')).to eq(false)
113
+ expect(cached_filesystem.exists?('/logs/box4_123.txt')).to eq(false)
114
+ expect(cached_filesystem.exists?('/logs/box4_456.txt')).to eq(false)
115
+ expect(cached_filesystem.exists?('/logs/box')).to eq(false)
116
+ expect(cached_filesystem.glob('/logs/*')).not_to be_empty
117
+ expect(cached_filesystem.glob('/logs/*.txt')).not_to be_empty
118
+ expect(cached_filesystem.glob('/logs/box1_*.txt')).not_to be_empty
119
+ expect(cached_filesystem.glob('/logs/box2_*.txt')).not_to be_empty
120
+ expect(cached_filesystem.glob('/logs/box3_*.txt')).not_to be_empty
121
+ expect(cached_filesystem.glob('/logs/box*.txt').size).to eq(3)
122
+ expect(cached_filesystem.glob('/logs/box*.csv')).to be_empty
123
+ expect(cached_filesystem.glob('/logs/box')).to be_empty
124
+ expect(cached_filesystem.glob('/logs/box/*')).to be_empty
125
+ expect(cached_filesystem.stat('/logs/box1_123.txt')).to_not be_nil
126
+ expect(cached_filesystem.stat('/logs/box1_456.txt')).to be_nil
127
+ expect(cached_filesystem.stat('/logs/box2_123.txt')).to_not be_nil
128
+ expect(cached_filesystem.stat('/logs/box2_456.txt')).to be_nil
129
+ expect(cached_filesystem.stat('/logs/box3_123.txt')).to_not be_nil
130
+ expect(cached_filesystem.stat('/logs/box3_456.txt')).to be_nil
131
+ expect(cached_filesystem.stat('/logs/box4_123.txt')).to be_nil
132
+ expect(cached_filesystem.stat('/logs/box4_456.txt')).to be_nil
133
+ expect(cached_filesystem.glob('/logs')).not_to be_nil
134
+ end
135
+ end
136
+
137
+ context 'when path is missing' do
138
+ before do
139
+ filesystem.touch!('/a/b/c')
140
+ expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
141
+ expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
142
+ expect(filesystem).to receive(:glob_stat).with('/*').never
143
+ end
144
+
145
+ it 'calls Filesystem#glob_stat once for multiple calls' do
146
+ expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(false)
147
+ expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(false)
148
+ expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(false)
149
+ expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
150
+ expect(cached_filesystem.exists?('/a/b')).to eq(true)
151
+ expect(cached_filesystem.exists?('/a')).to eq(true)
152
+ expect(cached_filesystem.glob('/a')).to be_empty
153
+ expect(cached_filesystem.glob('/a/*')).not_to be_empty
154
+ expect(cached_filesystem.glob('/a/b')).to be_empty
155
+ expect(cached_filesystem.glob('/a/b/*')).not_to be_empty
156
+ expect(cached_filesystem.glob('/a/b/c')).not_to be_empty
157
+ expect(cached_filesystem.glob('/a/b/c/*')).to be_empty
158
+ expect(cached_filesystem.glob('/a/b/c/*.txt')).to be_empty
159
+ expect(cached_filesystem.stat('/a/b/c/1.txt')).to be_nil
160
+ expect(cached_filesystem.stat('/a/b/c/2.txt')).to be_nil
161
+ expect(cached_filesystem.stat('/a/b/c/3.txt')).to be_nil
162
+ expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
163
+ expect(cached_filesystem.stat('/a/b')).to_not be_nil
164
+ expect(cached_filesystem.stat('/a')).to_not be_nil
165
+ end
166
+ end
167
+ end