masamune 0.11.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (185) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +21 -0
  3. data/README.md +54 -0
  4. data/Rakefile +15 -0
  5. data/bin/masamune-elastic-mapreduce +4 -0
  6. data/bin/masamune-hive +4 -0
  7. data/bin/masamune-psql +4 -0
  8. data/bin/masamune-shell +4 -0
  9. data/lib/masamune.rb +56 -0
  10. data/lib/masamune/accumulate.rb +60 -0
  11. data/lib/masamune/actions.rb +38 -0
  12. data/lib/masamune/actions/data_flow.rb +131 -0
  13. data/lib/masamune/actions/date_parse.rb +75 -0
  14. data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
  15. data/lib/masamune/actions/execute.rb +52 -0
  16. data/lib/masamune/actions/filesystem.rb +37 -0
  17. data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
  18. data/lib/masamune/actions/hadoop_streaming.rb +41 -0
  19. data/lib/masamune/actions/hive.rb +74 -0
  20. data/lib/masamune/actions/postgres.rb +76 -0
  21. data/lib/masamune/actions/postgres_admin.rb +34 -0
  22. data/lib/masamune/actions/s3cmd.rb +44 -0
  23. data/lib/masamune/actions/transform.rb +89 -0
  24. data/lib/masamune/after_initialize_callbacks.rb +55 -0
  25. data/lib/masamune/cached_filesystem.rb +110 -0
  26. data/lib/masamune/commands.rb +37 -0
  27. data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
  28. data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
  29. data/lib/masamune/commands/hadoop_streaming.rb +116 -0
  30. data/lib/masamune/commands/hive.rb +178 -0
  31. data/lib/masamune/commands/interactive.rb +37 -0
  32. data/lib/masamune/commands/postgres.rb +128 -0
  33. data/lib/masamune/commands/postgres_admin.rb +72 -0
  34. data/lib/masamune/commands/postgres_common.rb +33 -0
  35. data/lib/masamune/commands/retry_with_backoff.rb +60 -0
  36. data/lib/masamune/commands/s3cmd.rb +70 -0
  37. data/lib/masamune/commands/shell.rb +202 -0
  38. data/lib/masamune/configuration.rb +195 -0
  39. data/lib/masamune/data_plan.rb +31 -0
  40. data/lib/masamune/data_plan/builder.rb +66 -0
  41. data/lib/masamune/data_plan/elem.rb +190 -0
  42. data/lib/masamune/data_plan/engine.rb +162 -0
  43. data/lib/masamune/data_plan/rule.rb +292 -0
  44. data/lib/masamune/data_plan/set.rb +176 -0
  45. data/lib/masamune/environment.rb +164 -0
  46. data/lib/masamune/filesystem.rb +567 -0
  47. data/lib/masamune/has_environment.rb +40 -0
  48. data/lib/masamune/helpers.rb +27 -0
  49. data/lib/masamune/helpers/postgres.rb +84 -0
  50. data/lib/masamune/io.rb +33 -0
  51. data/lib/masamune/last_element.rb +53 -0
  52. data/lib/masamune/method_logger.rb +41 -0
  53. data/lib/masamune/multi_io.rb +39 -0
  54. data/lib/masamune/schema.rb +36 -0
  55. data/lib/masamune/schema/catalog.rb +233 -0
  56. data/lib/masamune/schema/column.rb +527 -0
  57. data/lib/masamune/schema/dimension.rb +133 -0
  58. data/lib/masamune/schema/event.rb +121 -0
  59. data/lib/masamune/schema/fact.rb +133 -0
  60. data/lib/masamune/schema/map.rb +265 -0
  61. data/lib/masamune/schema/row.rb +133 -0
  62. data/lib/masamune/schema/store.rb +115 -0
  63. data/lib/masamune/schema/table.rb +308 -0
  64. data/lib/masamune/schema/table_reference.rb +76 -0
  65. data/lib/masamune/spec_helper.rb +23 -0
  66. data/lib/masamune/string_format.rb +34 -0
  67. data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
  68. data/lib/masamune/tasks/hive_thor.rb +55 -0
  69. data/lib/masamune/tasks/postgres_thor.rb +47 -0
  70. data/lib/masamune/tasks/shell_thor.rb +63 -0
  71. data/lib/masamune/template.rb +77 -0
  72. data/lib/masamune/thor.rb +186 -0
  73. data/lib/masamune/thor_loader.rb +38 -0
  74. data/lib/masamune/topological_hash.rb +34 -0
  75. data/lib/masamune/transform.rb +47 -0
  76. data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
  77. data/lib/masamune/transform/bulk_upsert.rb +52 -0
  78. data/lib/masamune/transform/consolidate_dimension.rb +54 -0
  79. data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
  80. data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
  81. data/lib/masamune/transform/define_event_view.hql.erb +51 -0
  82. data/lib/masamune/transform/define_event_view.rb +60 -0
  83. data/lib/masamune/transform/define_index.psql.erb +34 -0
  84. data/lib/masamune/transform/define_schema.hql.erb +23 -0
  85. data/lib/masamune/transform/define_schema.psql.erb +79 -0
  86. data/lib/masamune/transform/define_schema.rb +56 -0
  87. data/lib/masamune/transform/define_table.hql.erb +34 -0
  88. data/lib/masamune/transform/define_table.psql.erb +95 -0
  89. data/lib/masamune/transform/define_table.rb +40 -0
  90. data/lib/masamune/transform/define_unique.psql.erb +30 -0
  91. data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
  92. data/lib/masamune/transform/insert_reference_values.rb +64 -0
  93. data/lib/masamune/transform/load_dimension.rb +47 -0
  94. data/lib/masamune/transform/load_fact.rb +45 -0
  95. data/lib/masamune/transform/operator.rb +96 -0
  96. data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
  97. data/lib/masamune/transform/relabel_dimension.rb +39 -0
  98. data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
  99. data/lib/masamune/transform/rollup_fact.rb +149 -0
  100. data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
  101. data/lib/masamune/transform/snapshot_dimension.rb +74 -0
  102. data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
  103. data/lib/masamune/transform/stage_dimension.rb +83 -0
  104. data/lib/masamune/transform/stage_fact.psql.erb +80 -0
  105. data/lib/masamune/transform/stage_fact.rb +111 -0
  106. data/lib/masamune/version.rb +25 -0
  107. data/spec/fixtures/aggregate.sql.erb +25 -0
  108. data/spec/fixtures/comment.sql.erb +27 -0
  109. data/spec/fixtures/invalid.sql.erb +23 -0
  110. data/spec/fixtures/relative.sql.erb +23 -0
  111. data/spec/fixtures/simple.sql.erb +28 -0
  112. data/spec/fixtures/whitespace.sql.erb +30 -0
  113. data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
  114. data/spec/masamune/actions/execute_spec.rb +50 -0
  115. data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
  116. data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
  117. data/spec/masamune/actions/hive_spec.rb +117 -0
  118. data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
  119. data/spec/masamune/actions/postgres_spec.rb +134 -0
  120. data/spec/masamune/actions/s3cmd_spec.rb +44 -0
  121. data/spec/masamune/actions/transform_spec.rb +144 -0
  122. data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
  123. data/spec/masamune/cached_filesystem_spec.rb +167 -0
  124. data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
  125. data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
  126. data/spec/masamune/commands/hive_spec.rb +117 -0
  127. data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
  128. data/spec/masamune/commands/postgres_spec.rb +100 -0
  129. data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
  130. data/spec/masamune/commands/s3cmd_spec.rb +50 -0
  131. data/spec/masamune/commands/shell_spec.rb +101 -0
  132. data/spec/masamune/configuration_spec.rb +102 -0
  133. data/spec/masamune/data_plan/builder_spec.rb +91 -0
  134. data/spec/masamune/data_plan/elem_spec.rb +102 -0
  135. data/spec/masamune/data_plan/engine_spec.rb +356 -0
  136. data/spec/masamune/data_plan/rule_spec.rb +407 -0
  137. data/spec/masamune/data_plan/set_spec.rb +517 -0
  138. data/spec/masamune/environment_spec.rb +65 -0
  139. data/spec/masamune/filesystem_spec.rb +1421 -0
  140. data/spec/masamune/helpers/postgres_spec.rb +95 -0
  141. data/spec/masamune/schema/catalog_spec.rb +613 -0
  142. data/spec/masamune/schema/column_spec.rb +696 -0
  143. data/spec/masamune/schema/dimension_spec.rb +137 -0
  144. data/spec/masamune/schema/event_spec.rb +75 -0
  145. data/spec/masamune/schema/fact_spec.rb +117 -0
  146. data/spec/masamune/schema/map_spec.rb +593 -0
  147. data/spec/masamune/schema/row_spec.rb +28 -0
  148. data/spec/masamune/schema/store_spec.rb +49 -0
  149. data/spec/masamune/schema/table_spec.rb +395 -0
  150. data/spec/masamune/string_format_spec.rb +60 -0
  151. data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
  152. data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
  153. data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
  154. data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
  155. data/spec/masamune/template_spec.rb +77 -0
  156. data/spec/masamune/thor_spec.rb +238 -0
  157. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
  158. data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
  159. data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
  160. data/spec/masamune/transform/define_event_view_spec.rb +84 -0
  161. data/spec/masamune/transform/define_schema_spec.rb +83 -0
  162. data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
  163. data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
  164. data/spec/masamune/transform/define_table.table_spec.rb +525 -0
  165. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
  166. data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
  167. data/spec/masamune/transform/load_dimension_spec.rb +76 -0
  168. data/spec/masamune/transform/load_fact_spec.rb +89 -0
  169. data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
  170. data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
  171. data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
  172. data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
  173. data/spec/masamune/transform/stage_fact_spec.rb +204 -0
  174. data/spec/masamune_spec.rb +32 -0
  175. data/spec/spec_helper.rb +41 -0
  176. data/spec/support/masamune/example_group.rb +36 -0
  177. data/spec/support/masamune/mock_command.rb +99 -0
  178. data/spec/support/masamune/mock_delegate.rb +51 -0
  179. data/spec/support/masamune/mock_filesystem.rb +96 -0
  180. data/spec/support/masamune/thor_mute.rb +35 -0
  181. data/spec/support/rspec/example/action_example_group.rb +34 -0
  182. data/spec/support/rspec/example/task_example_group.rb +80 -0
  183. data/spec/support/rspec/example/transform_example_group.rb +36 -0
  184. data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
  185. metadata +462 -0
@@ -0,0 +1,200 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Transform::BulkUpsert do
26
+ before do
27
+ catalog.schema :postgres do
28
+ dimension 'cluster', type: :mini do
29
+ column 'id', type: :integer, surrogate_key: true, auto: true
30
+ column 'name', type: :string, unique: true
31
+ row name: 'default', attributes: {default: true}
32
+ end
33
+
34
+ dimension 'user_account_state', type: :mini do
35
+ column 'name', type: :string, unique: true
36
+ column 'description', type: :string
37
+ row name: 'registered', description: 'Registered'
38
+ row name: 'active', description: 'Active', attributes: {default: true}
39
+ row name: 'inactive', description: 'Inactive'
40
+ end
41
+
42
+ dimension 'department', type: :mini do
43
+ references :cluster
44
+ column 'tenant_id', type: :integer, unique: true, natural_key: true
45
+ column 'department_id', type: :integer, unique: true, natural_key: true
46
+ row tenant_id: -1, department_id: -1, attributes: {default: true}
47
+ end
48
+
49
+ dimension 'user', type: :four do
50
+ references :cluster
51
+ references :department, insert: true
52
+ references :user_account_state
53
+ references :user_account_state, label: :hr
54
+ column 'tenant_id', index: true, natural_key: true
55
+ column 'user_id', index: true, natural_key: true
56
+ column 'name', type: :string
57
+ column 'preferences', type: :key_value, null: true
58
+ end
59
+
60
+ file 'user' do
61
+ column 'tenant_id', type: :integer
62
+ column 'user_id', type: :integer
63
+ column 'department.department_id', type: :integer
64
+ column 'user_account_state.name', type: :string
65
+ column 'hr_user_account_state.name', type: :string
66
+ column 'preferences_now', type: :json
67
+ column 'start_at', type: :timestamp
68
+ column 'source_kind', type: :string
69
+ column 'delta', type: :integer
70
+ end
71
+ end
72
+ end
73
+
74
+ let(:target) { catalog.postgres.user_dimension }
75
+
76
+ context 'for postgres dimension' do
77
+ subject(:result) { transform.bulk_upsert(target.stage_table, target).to_s }
78
+
79
+ it 'should render bulk_upsert template' do
80
+ is_expected.to eq <<-EOS.strip_heredoc
81
+ BEGIN;
82
+ LOCK TABLE user_dimension IN EXCLUSIVE MODE;
83
+
84
+ UPDATE
85
+ user_dimension
86
+ SET
87
+ department_type_id = COALESCE(user_dimension_stage.department_type_id, user_dimension.department_type_id),
88
+ user_account_state_type_id = COALESCE(user_dimension_stage.user_account_state_type_id, user_dimension.user_account_state_type_id),
89
+ hr_user_account_state_type_id = COALESCE(user_dimension_stage.hr_user_account_state_type_id, user_dimension.hr_user_account_state_type_id),
90
+ name = COALESCE(user_dimension_stage.name, user_dimension.name),
91
+ preferences = COALESCE(user_dimension_stage.preferences, user_dimension.preferences)
92
+ FROM
93
+ user_dimension_stage
94
+ WHERE
95
+ user_dimension.tenant_id = user_dimension_stage.tenant_id AND
96
+ user_dimension.user_id = user_dimension_stage.user_id AND
97
+ user_dimension.start_at = user_dimension_stage.start_at
98
+ ;
99
+
100
+ INSERT INTO
101
+ user_dimension (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences, parent_id, record_id, start_at, end_at, version, last_modified_at)
102
+ SELECT
103
+ user_dimension_stage.department_type_id,
104
+ user_dimension_stage.user_account_state_type_id,
105
+ user_dimension_stage.hr_user_account_state_type_id,
106
+ user_dimension_stage.tenant_id,
107
+ user_dimension_stage.user_id,
108
+ user_dimension_stage.name,
109
+ user_dimension_stage.preferences,
110
+ user_dimension_stage.parent_id,
111
+ user_dimension_stage.record_id,
112
+ user_dimension_stage.start_at,
113
+ user_dimension_stage.end_at,
114
+ user_dimension_stage.version,
115
+ user_dimension_stage.last_modified_at
116
+ FROM
117
+ user_dimension_stage
118
+ LEFT OUTER JOIN
119
+ user_dimension
120
+ ON
121
+ user_dimension.tenant_id = user_dimension_stage.tenant_id AND
122
+ user_dimension.user_id = user_dimension_stage.user_id AND
123
+ user_dimension.start_at = user_dimension_stage.start_at
124
+ WHERE
125
+ user_dimension.tenant_id IS NULL AND
126
+ user_dimension.user_id IS NULL AND
127
+ user_dimension.start_at IS NULL
128
+ ;
129
+
130
+ COMMIT;
131
+ EOS
132
+ end
133
+ end
134
+
135
+ context 'for postgres dimension ledger' do
136
+ subject(:result) { transform.bulk_upsert(target.ledger_table.stage_table, target.ledger_table).to_s }
137
+
138
+ it 'should render bulk_upsert template' do
139
+ is_expected.to eq <<-EOS.strip_heredoc
140
+ BEGIN;
141
+ LOCK TABLE user_dimension_ledger IN EXCLUSIVE MODE;
142
+
143
+ UPDATE
144
+ user_dimension_ledger
145
+ SET
146
+ department_type_id = COALESCE(user_dimension_ledger_stage.department_type_id, user_dimension_ledger.department_type_id),
147
+ user_account_state_type_id = COALESCE(user_dimension_ledger_stage.user_account_state_type_id, user_dimension_ledger.user_account_state_type_id),
148
+ hr_user_account_state_type_id = COALESCE(user_dimension_ledger_stage.hr_user_account_state_type_id, user_dimension_ledger.hr_user_account_state_type_id),
149
+ name = COALESCE(user_dimension_ledger_stage.name, user_dimension_ledger.name),
150
+ preferences_now = COALESCE(user_dimension_ledger_stage.preferences_now, user_dimension_ledger.preferences_now),
151
+ preferences_was = COALESCE(user_dimension_ledger_stage.preferences_was, user_dimension_ledger.preferences_was)
152
+ FROM
153
+ user_dimension_ledger_stage
154
+ WHERE
155
+ user_dimension_ledger.tenant_id = user_dimension_ledger_stage.tenant_id AND
156
+ user_dimension_ledger.user_id = user_dimension_ledger_stage.user_id AND
157
+ user_dimension_ledger.source_kind = user_dimension_ledger_stage.source_kind AND
158
+ user_dimension_ledger.source_uuid = user_dimension_ledger_stage.source_uuid AND
159
+ user_dimension_ledger.start_at = user_dimension_ledger_stage.start_at
160
+ ;
161
+
162
+ INSERT INTO
163
+ user_dimension_ledger (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences_now, preferences_was, source_kind, source_uuid, start_at, last_modified_at, delta)
164
+ SELECT
165
+ user_dimension_ledger_stage.department_type_id,
166
+ user_dimension_ledger_stage.user_account_state_type_id,
167
+ user_dimension_ledger_stage.hr_user_account_state_type_id,
168
+ user_dimension_ledger_stage.tenant_id,
169
+ user_dimension_ledger_stage.user_id,
170
+ user_dimension_ledger_stage.name,
171
+ user_dimension_ledger_stage.preferences_now,
172
+ user_dimension_ledger_stage.preferences_was,
173
+ user_dimension_ledger_stage.source_kind,
174
+ user_dimension_ledger_stage.source_uuid,
175
+ user_dimension_ledger_stage.start_at,
176
+ user_dimension_ledger_stage.last_modified_at,
177
+ user_dimension_ledger_stage.delta
178
+ FROM
179
+ user_dimension_ledger_stage
180
+ LEFT OUTER JOIN
181
+ user_dimension_ledger
182
+ ON
183
+ user_dimension_ledger.tenant_id = user_dimension_ledger_stage.tenant_id AND
184
+ user_dimension_ledger.user_id = user_dimension_ledger_stage.user_id AND
185
+ user_dimension_ledger.source_kind = user_dimension_ledger_stage.source_kind AND
186
+ user_dimension_ledger.source_uuid = user_dimension_ledger_stage.source_uuid AND
187
+ user_dimension_ledger.start_at = user_dimension_ledger_stage.start_at
188
+ WHERE
189
+ user_dimension_ledger.tenant_id IS NULL AND
190
+ user_dimension_ledger.user_id IS NULL AND
191
+ user_dimension_ledger.source_kind IS NULL AND
192
+ user_dimension_ledger.source_uuid IS NULL AND
193
+ user_dimension_ledger.start_at IS NULL
194
+ ;
195
+
196
+ COMMIT;
197
+ EOS
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,62 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Transform::ConsolidateDimension do
26
+ before do
27
+ catalog.schema :postgres do
28
+ dimension 'user_account_state', type: :mini do
29
+ column 'name', type: :string, unique: true
30
+ column 'description', type: :string
31
+ end
32
+
33
+ dimension 'user', type: :four do
34
+ references :user_account_state
35
+ column 'tenant_id', index: true, natural_key: true
36
+ column 'user_id', index: true, natural_key: true
37
+ column 'preferences', type: :key_value, null: true
38
+ end
39
+ end
40
+ end
41
+
42
+ let(:target) { catalog.postgres.user_dimension }
43
+
44
+ context 'with postgres dimension' do
45
+ subject(:result) { transform.consolidate_dimension(target).to_s }
46
+
47
+ it 'should render combined template' do
48
+ is_expected.to eq Masamune::Template.combine \
49
+ transform.define_table(target.stage_table(suffix: 'consolidated_forward')),
50
+ transform.define_table(target.stage_table(suffix: 'consolidated_reverse')),
51
+ transform.define_table(target.stage_table(suffix: 'consolidated')),
52
+ transform.define_table(target.stage_table(suffix: 'deduplicated')),
53
+ transform.snapshot_dimension(target.ledger_table, target.stage_table(suffix: 'consolidated_forward'), 'ASC'),
54
+ transform.snapshot_dimension(target.ledger_table, target.stage_table(suffix: 'consolidated_reverse'), 'DESC'),
55
+ transform.bulk_upsert(target.stage_table(suffix: 'consolidated_forward'), target.stage_table(suffix: 'consolidated')),
56
+ transform.bulk_upsert(target.stage_table(suffix: 'consolidated_reverse'), target.stage_table(suffix: 'consolidated')),
57
+ transform.deduplicate_dimension(target.stage_table(suffix: 'consolidated'), target.stage_table(suffix: 'deduplicated')),
58
+ transform.bulk_upsert(target.stage_table(suffix: 'deduplicated'), target),
59
+ transform.relabel_dimension(target)
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,84 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Transform::DeduplicateDimension do
26
+ before do
27
+ catalog.schema :postgres do
28
+ dimension 'user_account_state', type: :mini do
29
+ column 'name', type: :string, unique: true
30
+ column 'description', type: :string
31
+ end
32
+
33
+ dimension 'user', type: :four do
34
+ references :user_account_state
35
+ column 'tenant_id', index: true, natural_key: true
36
+ column 'user_id', index: true, natural_key: true
37
+ column 'preferences', type: :key_value, null: true
38
+ end
39
+ end
40
+ end
41
+
42
+ let(:target) { catalog.postgres.user_dimension }
43
+
44
+ context 'with postgres dimension' do
45
+ subject(:result) { transform.deduplicate_dimension(target.stage_table(suffix: 'consolidated'), target.stage_table(suffix: 'deduplicated')).to_s }
46
+
47
+ it 'should render deduplicate_dimension template' do
48
+ is_expected.to eq <<-EOS.strip_heredoc
49
+ INSERT INTO
50
+ user_deduplicated_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, parent_id, record_id, start_at)
51
+ SELECT DISTINCT
52
+ user_account_state_type_id,
53
+ tenant_id,
54
+ user_id,
55
+ preferences,
56
+ parent_id,
57
+ record_id,
58
+ start_at
59
+ FROM (
60
+ SELECT
61
+ user_account_state_type_id,
62
+ tenant_id,
63
+ user_id,
64
+ preferences,
65
+ parent_id,
66
+ record_id,
67
+ start_at,
68
+ CASE
69
+ WHEN (LAG(user_account_state_type_id) OVER w = user_account_state_type_id) AND (LAG(tenant_id) OVER w = tenant_id) AND (LAG(user_id) OVER w = user_id) AND (LAG(preferences) OVER w = preferences) THEN
70
+ 1
71
+ ELSE
72
+ 0
73
+ END AS duplicate
74
+ FROM
75
+ user_consolidated_dimension_stage
76
+ WINDOW w AS (PARTITION BY tenant_id, user_id ORDER BY start_at)
77
+ ) tmp
78
+ WHERE
79
+ duplicate = 0
80
+ ;
81
+ EOS
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,84 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Transform::DefineEventView do
26
+ before do
27
+ catalog.schema :hive do
28
+ event 'tenant' do
29
+ attribute 'tenant_id', type: :integer, immutable: true
30
+ attribute 'account_state', type: :string
31
+ attribute 'premium_type', type: :string
32
+ attribute 'preferences', type: :json
33
+ end
34
+ end
35
+ end
36
+
37
+ let(:target) { catalog.hive.tenant_event }
38
+
39
+ context 'with hive event' do
40
+ subject(:result) { transform.define_event_view(target).to_s }
41
+
42
+ it 'should render define_event_view template' do
43
+ is_expected.to eq <<-EOS.strip_heredoc
44
+ DROP VIEW IF EXISTS tenant_events;
45
+ CREATE VIEW IF NOT EXISTS tenant_events (
46
+ uuid,
47
+ type,
48
+ tenant_id,
49
+ account_state_now,
50
+ account_state_was,
51
+ premium_type_now,
52
+ premium_type_was,
53
+ preferences_now,
54
+ preferences_was,
55
+ delta,
56
+ created_at,
57
+ y, m, d ,h
58
+ ) PARTITIONED ON (y, m, d, h) AS
59
+ SELECT DISTINCT
60
+ uuid,
61
+ type,
62
+ tenant_id,
63
+ account_state_now,
64
+ account_state_was,
65
+ premium_type_now,
66
+ premium_type_was,
67
+ CONCAT('"', REGEXP_REPLACE(preferences_now, '"', '""'), '"') AS preferences_now,
68
+ CONCAT('"', REGEXP_REPLACE(preferences_was, '"', '""'), '"') AS preferences_was,
69
+ IF(type = 'tenant_update', 1, 0) AS delta,
70
+ ctime_iso8601 AS created_at,
71
+ y, m, d ,h
72
+ FROM
73
+ events
74
+ LATERAL VIEW
75
+ json_tuple(events.json, 'tenant_id', 'account_state_now', 'account_state_was', 'premium_type_now', 'premium_type_was', 'preferences_now', 'preferences_was') event_data AS tenant_id, account_state_now, account_state_was, premium_type_now, premium_type_was, preferences_now, preferences_was
76
+ WHERE
77
+ type = 'tenant_create' OR
78
+ type = 'tenant_update' OR
79
+ type = 'tenant_delete'
80
+ ;
81
+ EOS
82
+ end
83
+ end
84
+ end
@@ -0,0 +1,83 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ require 'spec_helper'
24
+
25
+ describe Masamune::Transform::DefineSchema do
26
+ context 'for postgres schema' do
27
+ before do
28
+ catalog.schema :postgres do
29
+ dimension 'user_account_state', type: :mini do
30
+ column 'name', type: :string, unique: true
31
+ column 'description', type: :string
32
+ end
33
+
34
+ dimension 'user', type: :four do
35
+ references :user_account_state
36
+ column 'tenant_id', index: true, natural_key: true
37
+ column 'user_id', index: true, natural_key: true
38
+ column 'preferences', type: :key_value, null: true
39
+ end
40
+
41
+ file 'user' do
42
+ column 'tenant_id', type: :integer
43
+ column 'user_id', type: :integer
44
+ column 'user_account_state.name', type: :string
45
+ column 'preferences_now', type: :json
46
+ column 'start_at', type: :timestamp
47
+ column 'source_kind', type: :string
48
+ column 'delta', type: :integer
49
+ end
50
+ end
51
+ end
52
+
53
+ subject(:result) { transform.define_schema(catalog, :postgres).to_s }
54
+
55
+ it 'should render combined template' do
56
+ is_expected.to eq Masamune::Template.combine \
57
+ Masamune::Transform::Operator.new('define_schema', source: catalog.postgres),
58
+ transform.define_table(catalog.postgres.dimensions['user_account_state']),
59
+ transform.define_table(catalog.postgres.dimensions['user'])
60
+ end
61
+ end
62
+
63
+ context 'for hive schema' do
64
+ before do
65
+ catalog.schema :hive do
66
+ event 'tenant' do
67
+ attribute 'tenant_id', type: :integer, immutable: true
68
+ attribute 'account_state', type: :string
69
+ attribute 'premium_type', type: :string
70
+ attribute 'preferences', type: :json
71
+ end
72
+ end
73
+ end
74
+
75
+ subject(:result) { transform.define_schema(catalog, :hive).to_s }
76
+
77
+ it 'should render combined template' do
78
+ is_expected.to eq Masamune::Template.combine \
79
+ Masamune::Transform::Operator.new('define_schema', source: catalog.hive),
80
+ transform.define_event_view(catalog.hive.events['tenant'])
81
+ end
82
+ end
83
+ end