masamune 0.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/README.md +54 -0
  4. data/Rakefile +15 -0
  5. data/bin/masamune-elastic-mapreduce +4 -0
  6. data/bin/masamune-hive +4 -0
  7. data/bin/masamune-psql +4 -0
  8. data/bin/masamune-shell +4 -0
  9. data/lib/masamune.rb +56 -0
  10. data/lib/masamune/accumulate.rb +60 -0
  11. data/lib/masamune/actions.rb +38 -0
  12. data/lib/masamune/actions/data_flow.rb +131 -0
  13. data/lib/masamune/actions/date_parse.rb +75 -0
  14. data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
  15. data/lib/masamune/actions/execute.rb +52 -0
  16. data/lib/masamune/actions/filesystem.rb +37 -0
  17. data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
  18. data/lib/masamune/actions/hadoop_streaming.rb +41 -0
  19. data/lib/masamune/actions/hive.rb +74 -0
  20. data/lib/masamune/actions/postgres.rb +76 -0
  21. data/lib/masamune/actions/postgres_admin.rb +34 -0
  22. data/lib/masamune/actions/s3cmd.rb +44 -0
  23. data/lib/masamune/actions/transform.rb +89 -0
  24. data/lib/masamune/after_initialize_callbacks.rb +55 -0
  25. data/lib/masamune/cached_filesystem.rb +110 -0
  26. data/lib/masamune/commands.rb +37 -0
  27. data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
  28. data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
  29. data/lib/masamune/commands/hadoop_streaming.rb +116 -0
  30. data/lib/masamune/commands/hive.rb +178 -0
  31. data/lib/masamune/commands/interactive.rb +37 -0
  32. data/lib/masamune/commands/postgres.rb +128 -0
  33. data/lib/masamune/commands/postgres_admin.rb +72 -0
  34. data/lib/masamune/commands/postgres_common.rb +33 -0
  35. data/lib/masamune/commands/retry_with_backoff.rb +60 -0
  36. data/lib/masamune/commands/s3cmd.rb +70 -0
  37. data/lib/masamune/commands/shell.rb +202 -0
  38. data/lib/masamune/configuration.rb +195 -0
  39. data/lib/masamune/data_plan.rb +31 -0
  40. data/lib/masamune/data_plan/builder.rb +66 -0
  41. data/lib/masamune/data_plan/elem.rb +190 -0
  42. data/lib/masamune/data_plan/engine.rb +162 -0
  43. data/lib/masamune/data_plan/rule.rb +292 -0
  44. data/lib/masamune/data_plan/set.rb +176 -0
  45. data/lib/masamune/environment.rb +164 -0
  46. data/lib/masamune/filesystem.rb +567 -0
  47. data/lib/masamune/has_environment.rb +40 -0
  48. data/lib/masamune/helpers.rb +27 -0
  49. data/lib/masamune/helpers/postgres.rb +84 -0
  50. data/lib/masamune/io.rb +33 -0
  51. data/lib/masamune/last_element.rb +53 -0
  52. data/lib/masamune/method_logger.rb +41 -0
  53. data/lib/masamune/multi_io.rb +39 -0
  54. data/lib/masamune/schema.rb +36 -0
  55. data/lib/masamune/schema/catalog.rb +233 -0
  56. data/lib/masamune/schema/column.rb +527 -0
  57. data/lib/masamune/schema/dimension.rb +133 -0
  58. data/lib/masamune/schema/event.rb +121 -0
  59. data/lib/masamune/schema/fact.rb +133 -0
  60. data/lib/masamune/schema/map.rb +265 -0
  61. data/lib/masamune/schema/row.rb +133 -0
  62. data/lib/masamune/schema/store.rb +115 -0
  63. data/lib/masamune/schema/table.rb +308 -0
  64. data/lib/masamune/schema/table_reference.rb +76 -0
  65. data/lib/masamune/spec_helper.rb +23 -0
  66. data/lib/masamune/string_format.rb +34 -0
  67. data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
  68. data/lib/masamune/tasks/hive_thor.rb +55 -0
  69. data/lib/masamune/tasks/postgres_thor.rb +47 -0
  70. data/lib/masamune/tasks/shell_thor.rb +63 -0
  71. data/lib/masamune/template.rb +77 -0
  72. data/lib/masamune/thor.rb +186 -0
  73. data/lib/masamune/thor_loader.rb +38 -0
  74. data/lib/masamune/topological_hash.rb +34 -0
  75. data/lib/masamune/transform.rb +47 -0
  76. data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
  77. data/lib/masamune/transform/bulk_upsert.rb +52 -0
  78. data/lib/masamune/transform/consolidate_dimension.rb +54 -0
  79. data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
  80. data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
  81. data/lib/masamune/transform/define_event_view.hql.erb +51 -0
  82. data/lib/masamune/transform/define_event_view.rb +60 -0
  83. data/lib/masamune/transform/define_index.psql.erb +34 -0
  84. data/lib/masamune/transform/define_schema.hql.erb +23 -0
  85. data/lib/masamune/transform/define_schema.psql.erb +79 -0
  86. data/lib/masamune/transform/define_schema.rb +56 -0
  87. data/lib/masamune/transform/define_table.hql.erb +34 -0
  88. data/lib/masamune/transform/define_table.psql.erb +95 -0
  89. data/lib/masamune/transform/define_table.rb +40 -0
  90. data/lib/masamune/transform/define_unique.psql.erb +30 -0
  91. data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
  92. data/lib/masamune/transform/insert_reference_values.rb +64 -0
  93. data/lib/masamune/transform/load_dimension.rb +47 -0
  94. data/lib/masamune/transform/load_fact.rb +45 -0
  95. data/lib/masamune/transform/operator.rb +96 -0
  96. data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
  97. data/lib/masamune/transform/relabel_dimension.rb +39 -0
  98. data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
  99. data/lib/masamune/transform/rollup_fact.rb +149 -0
  100. data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
  101. data/lib/masamune/transform/snapshot_dimension.rb +74 -0
  102. data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
  103. data/lib/masamune/transform/stage_dimension.rb +83 -0
  104. data/lib/masamune/transform/stage_fact.psql.erb +80 -0
  105. data/lib/masamune/transform/stage_fact.rb +111 -0
  106. data/lib/masamune/version.rb +25 -0
  107. data/spec/fixtures/aggregate.sql.erb +25 -0
  108. data/spec/fixtures/comment.sql.erb +27 -0
  109. data/spec/fixtures/invalid.sql.erb +23 -0
  110. data/spec/fixtures/relative.sql.erb +23 -0
  111. data/spec/fixtures/simple.sql.erb +28 -0
  112. data/spec/fixtures/whitespace.sql.erb +30 -0
  113. data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
  114. data/spec/masamune/actions/execute_spec.rb +50 -0
  115. data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
  116. data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
  117. data/spec/masamune/actions/hive_spec.rb +117 -0
  118. data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
  119. data/spec/masamune/actions/postgres_spec.rb +134 -0
  120. data/spec/masamune/actions/s3cmd_spec.rb +44 -0
  121. data/spec/masamune/actions/transform_spec.rb +144 -0
  122. data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
  123. data/spec/masamune/cached_filesystem_spec.rb +167 -0
  124. data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
  125. data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
  126. data/spec/masamune/commands/hive_spec.rb +117 -0
  127. data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
  128. data/spec/masamune/commands/postgres_spec.rb +100 -0
  129. data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
  130. data/spec/masamune/commands/s3cmd_spec.rb +50 -0
  131. data/spec/masamune/commands/shell_spec.rb +101 -0
  132. data/spec/masamune/configuration_spec.rb +102 -0
  133. data/spec/masamune/data_plan/builder_spec.rb +91 -0
  134. data/spec/masamune/data_plan/elem_spec.rb +102 -0
  135. data/spec/masamune/data_plan/engine_spec.rb +356 -0
  136. data/spec/masamune/data_plan/rule_spec.rb +407 -0
  137. data/spec/masamune/data_plan/set_spec.rb +517 -0
  138. data/spec/masamune/environment_spec.rb +65 -0
  139. data/spec/masamune/filesystem_spec.rb +1421 -0
  140. data/spec/masamune/helpers/postgres_spec.rb +95 -0
  141. data/spec/masamune/schema/catalog_spec.rb +613 -0
  142. data/spec/masamune/schema/column_spec.rb +696 -0
  143. data/spec/masamune/schema/dimension_spec.rb +137 -0
  144. data/spec/masamune/schema/event_spec.rb +75 -0
  145. data/spec/masamune/schema/fact_spec.rb +117 -0
  146. data/spec/masamune/schema/map_spec.rb +593 -0
  147. data/spec/masamune/schema/row_spec.rb +28 -0
  148. data/spec/masamune/schema/store_spec.rb +49 -0
  149. data/spec/masamune/schema/table_spec.rb +395 -0
  150. data/spec/masamune/string_format_spec.rb +60 -0
  151. data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
  152. data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
  153. data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
  154. data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
  155. data/spec/masamune/template_spec.rb +77 -0
  156. data/spec/masamune/thor_spec.rb +238 -0
  157. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
  158. data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
  159. data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
  160. data/spec/masamune/transform/define_event_view_spec.rb +84 -0
  161. data/spec/masamune/transform/define_schema_spec.rb +83 -0
  162. data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
  163. data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
  164. data/spec/masamune/transform/define_table.table_spec.rb +525 -0
  165. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
  166. data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
  167. data/spec/masamune/transform/load_dimension_spec.rb +76 -0
  168. data/spec/masamune/transform/load_fact_spec.rb +89 -0
  169. data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
  170. data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
  171. data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
  172. data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
  173. data/spec/masamune/transform/stage_fact_spec.rb +204 -0
  174. data/spec/masamune_spec.rb +32 -0
  175. data/spec/spec_helper.rb +41 -0
  176. data/spec/support/masamune/example_group.rb +36 -0
  177. data/spec/support/masamune/mock_command.rb +99 -0
  178. data/spec/support/masamune/mock_delegate.rb +51 -0
  179. data/spec/support/masamune/mock_filesystem.rb +96 -0
  180. data/spec/support/masamune/thor_mute.rb +35 -0
  181. data/spec/support/rspec/example/action_example_group.rb +34 -0
  182. data/spec/support/rspec/example/task_example_group.rb +80 -0
  183. data/spec/support/rspec/example/transform_example_group.rb +36 -0
  184. data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
  185. metadata +462 -0
@@ -0,0 +1,58 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::PostgresAdmin do
26
+ let(:klass) do
27
+ Class.new do
28
+ include Masamune::HasEnvironment
29
+ include Masamune::Actions::PostgresAdmin
30
+ end
31
+ end
32
+
33
+ let(:instance) { klass.new }
34
+
35
+ describe '.postgres_admin' do
36
+ subject { instance.postgres_admin(action: action, database: 'zombo') }
37
+
38
+ context 'with :action :create' do
39
+ let(:action) { :create }
40
+
41
+ before do
42
+ mock_command(/\Acreatedb/, mock_success)
43
+ end
44
+
45
+ it { is_expected.to be_success }
46
+ end
47
+
48
+ context 'with :action :drop' do
49
+ let(:action) { :drop }
50
+
51
+ before do
52
+ mock_command(/\Adropdb/, mock_success)
53
+ end
54
+
55
+ it { is_expected.to be_success }
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,134 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::Postgres do
26
+ let(:filesystem) { Masamune::MockFilesystem.new }
27
+
28
+ let(:klass) do
29
+ Class.new do
30
+ include Masamune::HasEnvironment
31
+ include Masamune::AfterInitializeCallbacks
32
+ include Masamune::Actions::Postgres
33
+ end
34
+ end
35
+
36
+ let(:instance) { klass.new }
37
+ let(:configuration) { {database: 'test'} }
38
+ let(:postgres_helper) { double }
39
+ let(:catalog) { double }
40
+
41
+ before do
42
+ allow(instance).to receive(:filesystem) { filesystem }
43
+ allow(instance).to receive(:catalog) { catalog }
44
+ allow(instance).to receive(:postgres_helper) { postgres_helper }
45
+ allow(instance).to receive_message_chain(:configuration, :postgres).and_return(configuration)
46
+ allow(instance).to receive_message_chain(:configuration, :with_quiet).and_yield
47
+ allow(instance).to receive_message_chain(:define_schema, :to_file) { 'catalog.psql' }
48
+ end
49
+
50
+ describe '.postgres' do
51
+ before do
52
+ mock_command(/\Apsql/, mock_success)
53
+ end
54
+
55
+ subject { instance.postgres }
56
+
57
+ it { is_expected.to be_success }
58
+ end
59
+
60
+ describe '.after_initialize' do
61
+ let(:options) { {} }
62
+ let(:setup_files) { [] }
63
+ let(:schema_files) { [] }
64
+ let(:configuration) { {database: 'test', setup_files: setup_files, schema_files: schema_files} }
65
+
66
+ subject(:after_initialize_invoke) do
67
+ instance.after_initialize_invoke(options)
68
+ end
69
+
70
+ context 'when database does not exist' do
71
+ before do
72
+ expect(postgres_helper).to receive(:database_exists?).and_return(false)
73
+ expect(instance).to receive(:postgres_admin).with(action: :create, database: 'test', safe: true).once
74
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
75
+ after_initialize_invoke
76
+ end
77
+ it 'should call posgres_admin once' do; end
78
+ end
79
+
80
+ context 'when database exists' do
81
+ before do
82
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
83
+ expect(instance).to receive(:postgres_admin).never
84
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
85
+ after_initialize_invoke
86
+ end
87
+ it 'should not call postgres_admin' do; end
88
+ end
89
+
90
+ context 'when setup_files are configured' do
91
+ let(:setup_files) { ['setup.psql'] }
92
+ before do
93
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
94
+ expect(instance).to receive(:postgres).with(file: setup_files.first).once
95
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
96
+ after_initialize_invoke
97
+ end
98
+ it 'should call postgres with setup_files' do; end
99
+ end
100
+
101
+ context 'when schema_files are configured' do
102
+ let(:schema_files) { ['schema.psql'] }
103
+ before do
104
+ filesystem.touch!(*schema_files)
105
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
106
+ expect(instance).to receive(:postgres).once
107
+ after_initialize_invoke
108
+ end
109
+ it 'should call postgres with schema_files' do; end
110
+ end
111
+
112
+ context 'when schema_files that are globs are configured' do
113
+ let(:schema_files) { ['schema*.psql'] }
114
+ before do
115
+ filesystem.touch!('schema_1.psql', 'schema_2.psql')
116
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
117
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
118
+ after_initialize_invoke
119
+ end
120
+ it 'should call postgres with schema_files' do; end
121
+ end
122
+
123
+ context 'when ruby schema_files configured' do
124
+ let(:schema_files) { ['schema.rb'] }
125
+ before do
126
+ filesystem.touch!('schema.rb')
127
+ expect(postgres_helper).to receive(:database_exists?).and_return(true)
128
+ expect(instance).to receive(:postgres).with(file: 'catalog.psql').once
129
+ after_initialize_invoke
130
+ end
131
+ it 'should call postgres with schema_files' do; end
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,44 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::S3Cmd do
26
+ let(:klass) do
27
+ Class.new do
28
+ include Masamune::HasEnvironment
29
+ include Masamune::Actions::S3Cmd
30
+ end
31
+ end
32
+
33
+ let(:instance) { klass.new }
34
+
35
+ describe '.s3cmd' do
36
+ before do
37
+ mock_command(/\As3cmd/, mock_success)
38
+ end
39
+
40
+ subject { instance.s3cmd 'ls', 's3://fake-bucket' }
41
+
42
+ it { is_expected.to be_success }
43
+ end
44
+ end
@@ -0,0 +1,144 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Actions::Transform do
26
+ let(:environment) { double }
27
+ let(:catalog) { Masamune::Schema::Catalog.new(environment) }
28
+
29
+ before do
30
+ catalog.schema :postgres do
31
+ dimension 'date', type: :date do
32
+ column 'date_id', type: :integer, unique: true, index: true, natural_key: true
33
+ column 'date_epoch', type: :integer
34
+ column 'month_epoch', type: :integer
35
+ column 'year_epoch', type: :integer
36
+ end
37
+
38
+ dimension 'user', type: :four do
39
+ column 'tenant_id', type: :integer, index: true
40
+ column 'user_id', type: :integer, index: true, surrogate_key: true
41
+ end
42
+
43
+ file 'user' do
44
+ column 'id', type: :integer
45
+ column 'tenant_id', type: :integer
46
+ column 'updated_at', type: :timestamp
47
+ end
48
+
49
+ map from: postgres.user_file, to: postgres.user_dimension do |row|
50
+ {
51
+ user_id: row[:id],
52
+ tenant_id: row[:tenant_id],
53
+ source_kind: 'users',
54
+ start_at: row[:updated_at],
55
+ delta: 1
56
+ }
57
+ end
58
+
59
+ fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
60
+ references :date
61
+ references :user
62
+
63
+ measure 'total', type: :integer
64
+ end
65
+
66
+ file 'visits_hourly' do
67
+ column 'user.tenant_id', type: :integer
68
+ column 'user.user_id', type: :integer
69
+ column 'time_key', type: :integer
70
+ column 'total', type: :integer
71
+ end
72
+ end
73
+ end
74
+
75
+ let(:source_file) { Tempfile.new('masamune') }
76
+
77
+ let(:klass) do
78
+ Class.new do
79
+ include Masamune::HasEnvironment
80
+ include Masamune::Actions::Transform
81
+ end
82
+ end
83
+
84
+ let(:instance) { klass.new }
85
+ let(:postgres) { catalog.postgres }
86
+
87
+ before do
88
+ instance.environment = Masamune::ExampleGroup
89
+ end
90
+
91
+ describe '.load_dimension' do
92
+ before do
93
+ mock_command(/\Apsql/, mock_success)
94
+ end
95
+
96
+ subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
97
+
98
+ it { is_expected.to be_success }
99
+ end
100
+
101
+ describe '.relabel_dimension' do
102
+ before do
103
+ mock_command(/\Apsql/, mock_success)
104
+ end
105
+
106
+ subject { instance.relabel_dimension(postgres.user_dimension) }
107
+
108
+ it { is_expected.to be_success }
109
+ end
110
+
111
+ describe '.consolidate_dimension' do
112
+ before do
113
+ mock_command(/\Apsql/, mock_success)
114
+ end
115
+
116
+ subject { instance.consolidate_dimension(postgres.user_dimension) }
117
+
118
+ it { is_expected.to be_success }
119
+ end
120
+
121
+ describe '.load_fact' do
122
+ let(:date) { DateTime.civil(2014, 8) }
123
+
124
+ before do
125
+ mock_command(/\Apsql/, mock_success)
126
+ end
127
+
128
+ subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
129
+
130
+ it { is_expected.to be_success }
131
+ end
132
+
133
+ describe '.rollup_fact' do
134
+ let(:date) { DateTime.civil(2014, 8) }
135
+
136
+ before do
137
+ mock_command(/\Apsql/, mock_success)
138
+ end
139
+
140
+ subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date) }
141
+
142
+ it { is_expected.to be_success }
143
+ end
144
+ end
@@ -0,0 +1,61 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::AfterInitializeCallbacks do
26
+ let(:klass) do
27
+ Class.new do
28
+ include Masamune::AfterInitializeCallbacks
29
+
30
+ def first_callback; end
31
+ def early_callback; end
32
+ def default_callback; end
33
+ def unknown_callback; end
34
+ def later_callback; end
35
+ def final_callback; end
36
+
37
+ after_initialize(:first) { |o| o.first_callback }
38
+ after_initialize(:early) { |o| o.early_callback }
39
+ after_initialize(:default) { |o| o.default_callback }
40
+ after_initialize(:unknown) { |o| o.unknown_callback }
41
+ after_initialize(:later) { |o| o.later_callback }
42
+ after_initialize(:final) { |o| o.final_callback}
43
+ end
44
+ end
45
+
46
+ let(:instance) { klass.new }
47
+
48
+ describe '.after_initialize_invoke' do
49
+ before do
50
+ expect(instance).to receive(:first_callback).once.ordered
51
+ expect(instance).to receive(:early_callback).once.ordered
52
+ expect(instance).to receive(:default_callback).once.ordered
53
+ expect(instance).to receive(:unknown_callback).once.ordered
54
+ expect(instance).to receive(:later_callback).once.ordered
55
+ expect(instance).to receive(:final_callback).once.ordered
56
+ instance.after_initialize_invoke
57
+ end
58
+
59
+ it 'should call callbacks in priority order' do; end
60
+ end
61
+ end
@@ -0,0 +1,167 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::CachedFilesystem do
26
+ let(:filesystem) { Masamune::MockFilesystem.new }
27
+ let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
28
+
29
+ context 'when path is present, top down traversal' do
30
+ before do
31
+ filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
32
+ expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
33
+ expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
34
+ expect(filesystem).to receive(:glob_stat).with('/*').never
35
+ end
36
+
37
+ it 'calls Filesystem#glob_stat once for multiple calls' do
38
+ expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
39
+ expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
40
+ expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
41
+ expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
42
+ expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
43
+ expect(cached_filesystem.glob('/a/b/c/*')).not_to be_empty
44
+ expect(cached_filesystem.glob('/a/b/c/*.txt')).not_to be_empty
45
+ expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
46
+ expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
47
+ expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
48
+ expect(cached_filesystem.stat('/a/b/c/4.txt')).to be_nil
49
+ expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
50
+ expect(cached_filesystem.stat('/a/b')).to_not be_nil
51
+ expect(cached_filesystem.stat('/a')).to_not be_nil
52
+ end
53
+ end
54
+
55
+ context 'when path is present, bottom up traversal' do
56
+ before do
57
+ filesystem.touch!('/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt')
58
+ expect(filesystem).to receive(:glob_stat).with('/a/*').once.and_call_original
59
+ expect(filesystem).to receive(:glob_stat).with('/*').never
60
+ end
61
+
62
+ it 'calls Filesystem#glob_stat once for multiple calls' do
63
+ expect(cached_filesystem.glob('/a/b/*')).to eq(['/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt'])
64
+ expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
65
+ expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
66
+ expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
67
+ expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
68
+ expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
69
+ expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
70
+ expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
71
+ expect(cached_filesystem.stat('/a/b/c/4.txt')).to be_nil
72
+ expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
73
+ expect(cached_filesystem.stat('/a/b')).to_not be_nil
74
+ expect(cached_filesystem.stat('/a')).to_not be_nil
75
+ end
76
+ end
77
+
78
+ context 'when path is present, checking for similar non existant paths' do
79
+ before do
80
+ filesystem.touch!('/y=2013/m=1/d=22/00000')
81
+ expect(filesystem).to receive(:glob_stat).with('/y=2013/m=1/*').once.and_call_original
82
+ expect(filesystem).to receive(:glob_stat).with('/y=2013/*').once.and_call_original
83
+ expect(filesystem).to receive(:glob_stat).with('/*').never
84
+ end
85
+
86
+ it 'calls Filesystem#glob_stat once for multiple calls' do
87
+ expect(cached_filesystem.exists?('/y=2013/m=1/d=22/00000')).to eq(true)
88
+ expect(cached_filesystem.exists?('/y=2013/m=1/d=22')).to eq(true)
89
+ expect(cached_filesystem.exists?('/y=2013/m=1/d=2')).to eq(false)
90
+ expect(cached_filesystem.glob('/y=2013/m=1/*')).not_to be_empty
91
+ expect(cached_filesystem.glob('/y=2013/m=1/d=22/*')).not_to be_empty
92
+ expect(cached_filesystem.stat('/y=2013/m=1/d=22/00000')).not_to be_nil
93
+ expect(cached_filesystem.stat('/y=2013/m=1/d=22')).not_to be_nil
94
+ expect(cached_filesystem.stat('/y=2013/m=1')).not_to be_nil
95
+ expect(cached_filesystem.stat('/y=2013')).not_to be_nil
96
+ end
97
+ end
98
+
99
+ context 'when path is present, checking for similar existing paths' do
100
+ before do
101
+ filesystem.touch!('/logs/box1_123.txt', '/logs/box2_123.txt', '/logs/box3_123.txt')
102
+ expect(filesystem).to receive(:glob_stat).with('/logs/*').once.and_call_original
103
+ expect(filesystem).to receive(:glob_stat).with('/*').never
104
+ end
105
+
106
+ it 'calls Filesystem#glob_stat once for multiple calls' do
107
+ expect(cached_filesystem.exists?('/logs/box1_123.txt')).to eq(true)
108
+ expect(cached_filesystem.exists?('/logs/box1_456.txt')).to eq(false)
109
+ expect(cached_filesystem.exists?('/logs/box2_123.txt')).to eq(true)
110
+ expect(cached_filesystem.exists?('/logs/box2_456.txt')).to eq(false)
111
+ expect(cached_filesystem.exists?('/logs/box3_123.txt')).to eq(true)
112
+ expect(cached_filesystem.exists?('/logs/box3_456.txt')).to eq(false)
113
+ expect(cached_filesystem.exists?('/logs/box4_123.txt')).to eq(false)
114
+ expect(cached_filesystem.exists?('/logs/box4_456.txt')).to eq(false)
115
+ expect(cached_filesystem.exists?('/logs/box')).to eq(false)
116
+ expect(cached_filesystem.glob('/logs/*')).not_to be_empty
117
+ expect(cached_filesystem.glob('/logs/*.txt')).not_to be_empty
118
+ expect(cached_filesystem.glob('/logs/box1_*.txt')).not_to be_empty
119
+ expect(cached_filesystem.glob('/logs/box2_*.txt')).not_to be_empty
120
+ expect(cached_filesystem.glob('/logs/box3_*.txt')).not_to be_empty
121
+ expect(cached_filesystem.glob('/logs/box*.txt').size).to eq(3)
122
+ expect(cached_filesystem.glob('/logs/box*.csv')).to be_empty
123
+ expect(cached_filesystem.glob('/logs/box')).to be_empty
124
+ expect(cached_filesystem.glob('/logs/box/*')).to be_empty
125
+ expect(cached_filesystem.stat('/logs/box1_123.txt')).to_not be_nil
126
+ expect(cached_filesystem.stat('/logs/box1_456.txt')).to be_nil
127
+ expect(cached_filesystem.stat('/logs/box2_123.txt')).to_not be_nil
128
+ expect(cached_filesystem.stat('/logs/box2_456.txt')).to be_nil
129
+ expect(cached_filesystem.stat('/logs/box3_123.txt')).to_not be_nil
130
+ expect(cached_filesystem.stat('/logs/box3_456.txt')).to be_nil
131
+ expect(cached_filesystem.stat('/logs/box4_123.txt')).to be_nil
132
+ expect(cached_filesystem.stat('/logs/box4_456.txt')).to be_nil
133
+ expect(cached_filesystem.glob('/logs')).not_to be_nil
134
+ end
135
+ end
136
+
137
+ context 'when path is missing' do
138
+ before do
139
+ filesystem.touch!('/a/b/c')
140
+ expect(filesystem).to receive(:glob_stat).with('/a/b/*').once.and_call_original
141
+ expect(filesystem).to receive(:glob_stat).with('/a').once.and_call_original
142
+ expect(filesystem).to receive(:glob_stat).with('/*').never
143
+ end
144
+
145
+ it 'calls Filesystem#glob_stat once for multiple calls' do
146
+ expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(false)
147
+ expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(false)
148
+ expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(false)
149
+ expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
150
+ expect(cached_filesystem.exists?('/a/b')).to eq(true)
151
+ expect(cached_filesystem.exists?('/a')).to eq(true)
152
+ expect(cached_filesystem.glob('/a')).to be_empty
153
+ expect(cached_filesystem.glob('/a/*')).not_to be_empty
154
+ expect(cached_filesystem.glob('/a/b')).to be_empty
155
+ expect(cached_filesystem.glob('/a/b/*')).not_to be_empty
156
+ expect(cached_filesystem.glob('/a/b/c')).not_to be_empty
157
+ expect(cached_filesystem.glob('/a/b/c/*')).to be_empty
158
+ expect(cached_filesystem.glob('/a/b/c/*.txt')).to be_empty
159
+ expect(cached_filesystem.stat('/a/b/c/1.txt')).to be_nil
160
+ expect(cached_filesystem.stat('/a/b/c/2.txt')).to be_nil
161
+ expect(cached_filesystem.stat('/a/b/c/3.txt')).to be_nil
162
+ expect(cached_filesystem.stat('/a/b/c')).to_not be_nil
163
+ expect(cached_filesystem.stat('/a/b')).to_not be_nil
164
+ expect(cached_filesystem.stat('/a')).to_not be_nil
165
+ end
166
+ end
167
+ end