masamune 0.13.8 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/lib/masamune.rb +8 -5
  3. data/lib/masamune/actions.rb +1 -13
  4. data/lib/masamune/actions/data_flow.rb +2 -1
  5. data/lib/masamune/actions/date_parse.rb +0 -1
  6. data/lib/masamune/actions/elastic_mapreduce.rb +0 -2
  7. data/lib/masamune/actions/filesystem.rb +0 -2
  8. data/lib/masamune/actions/hive.rb +0 -2
  9. data/lib/masamune/actions/invoke_parallel.rb +2 -1
  10. data/lib/masamune/actions/postgres.rb +0 -1
  11. data/lib/masamune/actions/s3cmd.rb +2 -0
  12. data/lib/masamune/actions/transform.rb +0 -2
  13. data/lib/masamune/after_initialize_callbacks.rb +0 -2
  14. data/lib/masamune/commands.rb +1 -11
  15. data/lib/masamune/commands/postgres.rb +1 -0
  16. data/lib/masamune/commands/postgres_admin.rb +2 -0
  17. data/lib/masamune/configuration.rb +2 -0
  18. data/lib/masamune/data_plan/engine.rb +2 -0
  19. data/lib/masamune/filesystem.rb +2 -0
  20. data/lib/masamune/helpers.rb +1 -1
  21. data/lib/masamune/last_element.rb +0 -2
  22. data/lib/masamune/schema/dimension.rb +1 -3
  23. data/lib/masamune/schema/store.rb +2 -0
  24. data/lib/masamune/schema/table.rb +2 -0
  25. data/lib/masamune/template.rb +4 -1
  26. data/lib/masamune/thor.rb +1 -1
  27. data/lib/masamune/transform.rb +1 -21
  28. data/lib/masamune/transform/bulk_upsert.rb +1 -22
  29. data/lib/masamune/transform/common.rb +27 -0
  30. data/lib/masamune/transform/common/denormalize_table.rb +90 -0
  31. data/lib/masamune/transform/deduplicate_dimension.rb +1 -41
  32. data/lib/masamune/transform/define_table.rb +1 -113
  33. data/lib/masamune/transform/denormalize_table.rb +1 -50
  34. data/lib/masamune/transform/hive.rb +27 -0
  35. data/lib/masamune/transform/{define_schema.hql.erb → hive/define_schema.hql.erb} +0 -0
  36. data/lib/masamune/transform/{define_table.hql.erb → hive/define_table.hql.erb} +0 -0
  37. data/lib/masamune/transform/hive/define_table.rb +46 -0
  38. data/lib/masamune/transform/{denormalize_table.hql.erb → hive/denormalize_table.hql.erb} +0 -0
  39. data/lib/masamune/transform/hive/denormalize_table.rb +27 -0
  40. data/lib/masamune/transform/insert_reference_values.rb +1 -30
  41. data/lib/masamune/transform/operator.rb +36 -37
  42. data/lib/masamune/transform/postgres.rb +27 -0
  43. data/lib/masamune/transform/{bulk_upsert.psql.erb → postgres/bulk_upsert.psql.erb} +0 -0
  44. data/lib/masamune/transform/postgres/bulk_upsert.rb +62 -0
  45. data/lib/masamune/transform/{deduplicate_dimension.psql.erb → postgres/deduplicate_dimension.psql.erb} +1 -7
  46. data/lib/masamune/transform/postgres/deduplicate_dimension.rb +79 -0
  47. data/lib/masamune/transform/{define_foreign_key.psql.erb → postgres/define_foreign_key.psql.erb} +0 -0
  48. data/lib/masamune/transform/{define_index.psql.erb → postgres/define_index.psql.erb} +0 -0
  49. data/lib/masamune/transform/{define_inheritance.psql.erb → postgres/define_inheritance.psql.erb} +0 -0
  50. data/lib/masamune/transform/{define_schema.psql.erb → postgres/define_schema.psql.erb} +0 -0
  51. data/lib/masamune/transform/{define_table.psql.erb → postgres/define_table.psql.erb} +0 -0
  52. data/lib/masamune/transform/postgres/define_table.rb +142 -0
  53. data/lib/masamune/transform/{define_unique.psql.erb → postgres/define_unique.psql.erb} +0 -0
  54. data/lib/masamune/transform/{denormalize_table.psql.erb → postgres/denormalize_table.psql.erb} +0 -0
  55. data/lib/masamune/transform/postgres/denormalize_table.rb +27 -0
  56. data/lib/masamune/transform/{insert_reference_values.psql.erb → postgres/insert_reference_values.psql.erb} +1 -1
  57. data/lib/masamune/transform/postgres/insert_reference_values.rb +69 -0
  58. data/lib/masamune/transform/{relabel_dimension.psql.erb → postgres/relabel_dimension.psql.erb} +4 -1
  59. data/lib/masamune/transform/postgres/relabel_dimension.rb +45 -0
  60. data/lib/masamune/transform/{replace_table.psql.erb → postgres/replace_table.psql.erb} +0 -0
  61. data/lib/masamune/transform/{rollup_fact.psql.erb → postgres/rollup_fact.psql.erb} +0 -0
  62. data/lib/masamune/transform/postgres/rollup_fact.rb +123 -0
  63. data/lib/masamune/transform/{snapshot_dimension.psql.erb → postgres/snapshot_dimension.psql.erb} +3 -10
  64. data/lib/masamune/transform/postgres/snapshot_dimension.rb +83 -0
  65. data/lib/masamune/transform/{stage_dimension.psql.erb → postgres/stage_dimension.psql.erb} +0 -0
  66. data/lib/masamune/transform/postgres/stage_dimension.rb +90 -0
  67. data/lib/masamune/transform/{stage_fact.psql.erb → postgres/stage_fact.psql.erb} +0 -0
  68. data/lib/masamune/transform/postgres/stage_fact.rb +134 -0
  69. data/lib/masamune/transform/relabel_dimension.rb +1 -9
  70. data/lib/masamune/transform/rollup_fact.rb +1 -86
  71. data/lib/masamune/transform/snapshot_dimension.rb +1 -44
  72. data/lib/masamune/transform/stage_dimension.rb +1 -53
  73. data/lib/masamune/transform/stage_fact.rb +1 -96
  74. data/lib/masamune/version.rb +1 -1
  75. data/spec/masamune/template_spec.rb +1 -1
  76. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +1 -3
  77. data/spec/masamune/transform/deduplicate_dimension_spec.rb +1 -7
  78. data/spec/masamune/transform/define_table.dimension_spec.rb +0 -14
  79. data/spec/masamune/transform/denormalize_table_spec.rb +34 -0
  80. data/spec/masamune/transform/relabel_dimension_spec.rb +6 -1
  81. data/spec/masamune/transform/snapshot_dimension_spec.rb +3 -10
  82. metadata +37 -21
@@ -24,6 +24,8 @@ require 'spec_helper'
24
24
 
25
25
  describe Masamune::Transform::RelabelDimension do
26
26
  before do
27
+ allow_any_instance_of(Masamune::Schema::Table).to receive(:lock_id).and_return(42)
28
+
27
29
  catalog.schema :postgres do
28
30
  dimension 'user_account_state', type: :mini do
29
31
  column 'name', type: :string, unique: true
@@ -46,8 +48,9 @@ describe Masamune::Transform::RelabelDimension do
46
48
 
47
49
  it 'should eq render relabel_dimension template' do
48
50
  is_expected.to eq <<-EOS.strip_heredoc
51
+ SELECT pg_advisory_lock(42);
52
+
49
53
  BEGIN;
50
- LOCK TABLE user_dimension IN EXCLUSIVE MODE;
51
54
 
52
55
  UPDATE user_dimension SET version = NULL;
53
56
 
@@ -96,6 +99,8 @@ describe Masamune::Transform::RelabelDimension do
96
99
  ;
97
100
 
98
101
  COMMIT;
102
+
103
+ SELECT pg_advisory_unlock(42);
99
104
  EOS
100
105
  end
101
106
  end
@@ -61,14 +61,9 @@ describe Masamune::Transform::SnapshotDimension do
61
61
  consolidated.tenant_id,
62
62
  consolidated.user_id,
63
63
  consolidated.preferences,
64
- consolidated.parent_id,
65
- consolidated.record_id,
66
64
  consolidated.start_at
67
65
  FROM (
68
- SELECT DISTINCT ON (tenant_id, user_id, start_at)
69
- FIRST_VALUE(id) OVER w AS parent_id,
70
- FIRST_VALUE(start_at) OVER w AS parent_start_at,
71
- id AS record_id,
66
+ SELECT DISTINCT ON (tenant_id, user_id, start_at, id)
72
67
  coalesce_merge(user_account_state_type_id) OVER w AS user_account_state_type_id,
73
68
  tenant_id AS tenant_id,
74
69
  user_id AS user_id,
@@ -77,7 +72,7 @@ describe Masamune::Transform::SnapshotDimension do
77
72
  FROM
78
73
  windows
79
74
  WINDOW w AS (PARTITION BY tenant_id, user_id, window_id ORDER BY start_at DESC)
80
- ORDER BY tenant_id, user_id, start_at DESC, window_id
75
+ ORDER BY tenant_id, user_id, start_at DESC, id DESC, window_id
81
76
  ) consolidated
82
77
  WHERE
83
78
  consolidated.user_account_state_type_id IS NOT NULL AND
@@ -85,14 +80,12 @@ describe Masamune::Transform::SnapshotDimension do
85
80
  consolidated.user_id IS NOT NULL
86
81
  )
87
82
  INSERT INTO
88
- user_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, parent_id, record_id, start_at)
83
+ user_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, start_at)
89
84
  SELECT
90
85
  user_account_state_type_id,
91
86
  tenant_id,
92
87
  user_id,
93
88
  preferences,
94
- parent_id,
95
- record_id,
96
89
  start_at
97
90
  FROM
98
91
  snapshot
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: masamune
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.8
4
+ version: 0.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael Andrews
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-20 00:00:00.000000000 Z
11
+ date: 2015-10-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -265,39 +265,55 @@ files:
265
265
  - lib/masamune/thor_loader.rb
266
266
  - lib/masamune/topological_hash.rb
267
267
  - lib/masamune/transform.rb
268
- - lib/masamune/transform/bulk_upsert.psql.erb
269
268
  - lib/masamune/transform/bulk_upsert.rb
269
+ - lib/masamune/transform/common.rb
270
+ - lib/masamune/transform/common/denormalize_table.rb
270
271
  - lib/masamune/transform/consolidate_dimension.rb
271
- - lib/masamune/transform/deduplicate_dimension.psql.erb
272
272
  - lib/masamune/transform/deduplicate_dimension.rb
273
- - lib/masamune/transform/define_foreign_key.psql.erb
274
- - lib/masamune/transform/define_index.psql.erb
275
- - lib/masamune/transform/define_inheritance.psql.erb
276
- - lib/masamune/transform/define_schema.hql.erb
277
- - lib/masamune/transform/define_schema.psql.erb
278
273
  - lib/masamune/transform/define_schema.rb
279
- - lib/masamune/transform/define_table.hql.erb
280
- - lib/masamune/transform/define_table.psql.erb
281
274
  - lib/masamune/transform/define_table.rb
282
- - lib/masamune/transform/define_unique.psql.erb
283
- - lib/masamune/transform/denormalize_table.hql.erb
284
- - lib/masamune/transform/denormalize_table.psql.erb
285
275
  - lib/masamune/transform/denormalize_table.rb
286
- - lib/masamune/transform/insert_reference_values.psql.erb
276
+ - lib/masamune/transform/hive.rb
277
+ - lib/masamune/transform/hive/define_schema.hql.erb
278
+ - lib/masamune/transform/hive/define_table.hql.erb
279
+ - lib/masamune/transform/hive/define_table.rb
280
+ - lib/masamune/transform/hive/denormalize_table.hql.erb
281
+ - lib/masamune/transform/hive/denormalize_table.rb
287
282
  - lib/masamune/transform/insert_reference_values.rb
288
283
  - lib/masamune/transform/load_dimension.rb
289
284
  - lib/masamune/transform/load_fact.rb
290
285
  - lib/masamune/transform/operator.rb
291
- - lib/masamune/transform/relabel_dimension.psql.erb
286
+ - lib/masamune/transform/postgres.rb
287
+ - lib/masamune/transform/postgres/bulk_upsert.psql.erb
288
+ - lib/masamune/transform/postgres/bulk_upsert.rb
289
+ - lib/masamune/transform/postgres/deduplicate_dimension.psql.erb
290
+ - lib/masamune/transform/postgres/deduplicate_dimension.rb
291
+ - lib/masamune/transform/postgres/define_foreign_key.psql.erb
292
+ - lib/masamune/transform/postgres/define_index.psql.erb
293
+ - lib/masamune/transform/postgres/define_inheritance.psql.erb
294
+ - lib/masamune/transform/postgres/define_schema.psql.erb
295
+ - lib/masamune/transform/postgres/define_table.psql.erb
296
+ - lib/masamune/transform/postgres/define_table.rb
297
+ - lib/masamune/transform/postgres/define_unique.psql.erb
298
+ - lib/masamune/transform/postgres/denormalize_table.psql.erb
299
+ - lib/masamune/transform/postgres/denormalize_table.rb
300
+ - lib/masamune/transform/postgres/insert_reference_values.psql.erb
301
+ - lib/masamune/transform/postgres/insert_reference_values.rb
302
+ - lib/masamune/transform/postgres/relabel_dimension.psql.erb
303
+ - lib/masamune/transform/postgres/relabel_dimension.rb
304
+ - lib/masamune/transform/postgres/replace_table.psql.erb
305
+ - lib/masamune/transform/postgres/rollup_fact.psql.erb
306
+ - lib/masamune/transform/postgres/rollup_fact.rb
307
+ - lib/masamune/transform/postgres/snapshot_dimension.psql.erb
308
+ - lib/masamune/transform/postgres/snapshot_dimension.rb
309
+ - lib/masamune/transform/postgres/stage_dimension.psql.erb
310
+ - lib/masamune/transform/postgres/stage_dimension.rb
311
+ - lib/masamune/transform/postgres/stage_fact.psql.erb
312
+ - lib/masamune/transform/postgres/stage_fact.rb
292
313
  - lib/masamune/transform/relabel_dimension.rb
293
- - lib/masamune/transform/replace_table.psql.erb
294
- - lib/masamune/transform/rollup_fact.psql.erb
295
314
  - lib/masamune/transform/rollup_fact.rb
296
- - lib/masamune/transform/snapshot_dimension.psql.erb
297
315
  - lib/masamune/transform/snapshot_dimension.rb
298
- - lib/masamune/transform/stage_dimension.psql.erb
299
316
  - lib/masamune/transform/stage_dimension.rb
300
- - lib/masamune/transform/stage_fact.psql.erb
301
317
  - lib/masamune/transform/stage_fact.rb
302
318
  - lib/masamune/version.rb
303
319
  - spec/fixtures/aggregate.sql.erb