masamune 0.13.8 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/lib/masamune.rb +8 -5
  3. data/lib/masamune/actions.rb +1 -13
  4. data/lib/masamune/actions/data_flow.rb +2 -1
  5. data/lib/masamune/actions/date_parse.rb +0 -1
  6. data/lib/masamune/actions/elastic_mapreduce.rb +0 -2
  7. data/lib/masamune/actions/filesystem.rb +0 -2
  8. data/lib/masamune/actions/hive.rb +0 -2
  9. data/lib/masamune/actions/invoke_parallel.rb +2 -1
  10. data/lib/masamune/actions/postgres.rb +0 -1
  11. data/lib/masamune/actions/s3cmd.rb +2 -0
  12. data/lib/masamune/actions/transform.rb +0 -2
  13. data/lib/masamune/after_initialize_callbacks.rb +0 -2
  14. data/lib/masamune/commands.rb +1 -11
  15. data/lib/masamune/commands/postgres.rb +1 -0
  16. data/lib/masamune/commands/postgres_admin.rb +2 -0
  17. data/lib/masamune/configuration.rb +2 -0
  18. data/lib/masamune/data_plan/engine.rb +2 -0
  19. data/lib/masamune/filesystem.rb +2 -0
  20. data/lib/masamune/helpers.rb +1 -1
  21. data/lib/masamune/last_element.rb +0 -2
  22. data/lib/masamune/schema/dimension.rb +1 -3
  23. data/lib/masamune/schema/store.rb +2 -0
  24. data/lib/masamune/schema/table.rb +2 -0
  25. data/lib/masamune/template.rb +4 -1
  26. data/lib/masamune/thor.rb +1 -1
  27. data/lib/masamune/transform.rb +1 -21
  28. data/lib/masamune/transform/bulk_upsert.rb +1 -22
  29. data/lib/masamune/transform/common.rb +27 -0
  30. data/lib/masamune/transform/common/denormalize_table.rb +90 -0
  31. data/lib/masamune/transform/deduplicate_dimension.rb +1 -41
  32. data/lib/masamune/transform/define_table.rb +1 -113
  33. data/lib/masamune/transform/denormalize_table.rb +1 -50
  34. data/lib/masamune/transform/hive.rb +27 -0
  35. data/lib/masamune/transform/{define_schema.hql.erb → hive/define_schema.hql.erb} +0 -0
  36. data/lib/masamune/transform/{define_table.hql.erb → hive/define_table.hql.erb} +0 -0
  37. data/lib/masamune/transform/hive/define_table.rb +46 -0
  38. data/lib/masamune/transform/{denormalize_table.hql.erb → hive/denormalize_table.hql.erb} +0 -0
  39. data/lib/masamune/transform/hive/denormalize_table.rb +27 -0
  40. data/lib/masamune/transform/insert_reference_values.rb +1 -30
  41. data/lib/masamune/transform/operator.rb +36 -37
  42. data/lib/masamune/transform/postgres.rb +27 -0
  43. data/lib/masamune/transform/{bulk_upsert.psql.erb → postgres/bulk_upsert.psql.erb} +0 -0
  44. data/lib/masamune/transform/postgres/bulk_upsert.rb +62 -0
  45. data/lib/masamune/transform/{deduplicate_dimension.psql.erb → postgres/deduplicate_dimension.psql.erb} +1 -7
  46. data/lib/masamune/transform/postgres/deduplicate_dimension.rb +79 -0
  47. data/lib/masamune/transform/{define_foreign_key.psql.erb → postgres/define_foreign_key.psql.erb} +0 -0
  48. data/lib/masamune/transform/{define_index.psql.erb → postgres/define_index.psql.erb} +0 -0
  49. data/lib/masamune/transform/{define_inheritance.psql.erb → postgres/define_inheritance.psql.erb} +0 -0
  50. data/lib/masamune/transform/{define_schema.psql.erb → postgres/define_schema.psql.erb} +0 -0
  51. data/lib/masamune/transform/{define_table.psql.erb → postgres/define_table.psql.erb} +0 -0
  52. data/lib/masamune/transform/postgres/define_table.rb +142 -0
  53. data/lib/masamune/transform/{define_unique.psql.erb → postgres/define_unique.psql.erb} +0 -0
  54. data/lib/masamune/transform/{denormalize_table.psql.erb → postgres/denormalize_table.psql.erb} +0 -0
  55. data/lib/masamune/transform/postgres/denormalize_table.rb +27 -0
  56. data/lib/masamune/transform/{insert_reference_values.psql.erb → postgres/insert_reference_values.psql.erb} +1 -1
  57. data/lib/masamune/transform/postgres/insert_reference_values.rb +69 -0
  58. data/lib/masamune/transform/{relabel_dimension.psql.erb → postgres/relabel_dimension.psql.erb} +4 -1
  59. data/lib/masamune/transform/postgres/relabel_dimension.rb +45 -0
  60. data/lib/masamune/transform/{replace_table.psql.erb → postgres/replace_table.psql.erb} +0 -0
  61. data/lib/masamune/transform/{rollup_fact.psql.erb → postgres/rollup_fact.psql.erb} +0 -0
  62. data/lib/masamune/transform/postgres/rollup_fact.rb +123 -0
  63. data/lib/masamune/transform/{snapshot_dimension.psql.erb → postgres/snapshot_dimension.psql.erb} +3 -10
  64. data/lib/masamune/transform/postgres/snapshot_dimension.rb +83 -0
  65. data/lib/masamune/transform/{stage_dimension.psql.erb → postgres/stage_dimension.psql.erb} +0 -0
  66. data/lib/masamune/transform/postgres/stage_dimension.rb +90 -0
  67. data/lib/masamune/transform/{stage_fact.psql.erb → postgres/stage_fact.psql.erb} +0 -0
  68. data/lib/masamune/transform/postgres/stage_fact.rb +134 -0
  69. data/lib/masamune/transform/relabel_dimension.rb +1 -9
  70. data/lib/masamune/transform/rollup_fact.rb +1 -86
  71. data/lib/masamune/transform/snapshot_dimension.rb +1 -44
  72. data/lib/masamune/transform/stage_dimension.rb +1 -53
  73. data/lib/masamune/transform/stage_fact.rb +1 -96
  74. data/lib/masamune/version.rb +1 -1
  75. data/spec/masamune/template_spec.rb +1 -1
  76. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +1 -3
  77. data/spec/masamune/transform/deduplicate_dimension_spec.rb +1 -7
  78. data/spec/masamune/transform/define_table.dimension_spec.rb +0 -14
  79. data/spec/masamune/transform/denormalize_table_spec.rb +34 -0
  80. data/spec/masamune/transform/relabel_dimension_spec.rb +6 -1
  81. data/spec/masamune/transform/snapshot_dimension_spec.rb +3 -10
  82. metadata +37 -21
@@ -20,8 +20,9 @@
20
20
  -- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  -- THE SOFTWARE.
22
22
 
23
+ SELECT pg_advisory_lock(<%= target.lock_id %>);
24
+
23
25
  BEGIN;
24
- LOCK TABLE <%= target.name %> IN EXCLUSIVE MODE;
25
26
 
26
27
  <%-# Relabel version column -%>
27
28
  UPDATE <%= target.name %> SET version = NULL;
@@ -74,3 +75,5 @@ WHERE
74
75
  ;
75
76
 
76
77
  COMMIT;
78
+
79
+ SELECT pg_advisory_unlock(<%= target.lock_id %>);
@@ -0,0 +1,45 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Masamune::Transform::Postgres
24
+ class RelabelDimension
25
+ def initialize(options = {})
26
+ @target = options[:target]
27
+ end
28
+
29
+ def locals
30
+ { target: target }
31
+ end
32
+
33
+ def target
34
+ TargetPresenter.new(@target)
35
+ end
36
+
37
+ private
38
+
39
+ class TargetPresenter < SimpleDelegator
40
+ def window(*extra)
41
+ (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,123 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Masamune::Transform::Postgres
24
+ class RollupFact
25
+ def initialize(options = {})
26
+ @target = options[:target]
27
+ @source = options[:source]
28
+ @date = options[:date]
29
+ end
30
+
31
+ def locals
32
+ { target: target, source: @source, date: @date }
33
+ end
34
+
35
+ def target
36
+ TargetPresenter.new(@target)
37
+ end
38
+
39
+ private
40
+
41
+ class TargetPresenter < SimpleDelegator
42
+ include Masamune::LastElement
43
+
44
+ def insert_columns(source)
45
+ values = []
46
+ shared_columns(source).values.map do |columns|
47
+ column = columns.first
48
+ next if column.id == :last_modified_at
49
+ next if column.auto_reference
50
+ values << column.name
51
+ end
52
+ measures.each do |_ ,measure|
53
+ values << measure.name
54
+ end
55
+ values << time_key.name
56
+ values.compact
57
+ end
58
+
59
+ def insert_values(source)
60
+ values = []
61
+ values << calculated_date_key(source)
62
+ shared_columns(source).values.map do |columns|
63
+ column = columns.first
64
+ next unless column.reference
65
+ next if column.reference.type == :date
66
+ next if column.auto_reference
67
+ values << column.qualified_name
68
+ end
69
+ source.measures.each do |_ ,measure|
70
+ values << measure.aggregate_value
71
+ end
72
+ values << calculated_time_key(source)
73
+ values
74
+ end
75
+ method_with_last_element :insert_values
76
+
77
+ def join_conditions(source)
78
+ {
79
+ source.date_column.reference.name => [
80
+ "#{source.date_column.reference.surrogate_key.qualified_name} = #{source.date_column.qualified_name}"
81
+ ]
82
+ }
83
+ end
84
+
85
+ def group_by(source)
86
+ group_by = []
87
+ group_by << calculated_date_key(source)
88
+ shared_columns(source).values.map do |columns|
89
+ column = columns.first
90
+ next unless column.reference
91
+ next if column.reference.type == :date
92
+ next if column.auto_reference
93
+ group_by << column.qualified_name
94
+ end
95
+ group_by << calculated_time_key(source)
96
+ group_by
97
+ end
98
+ method_with_last_element :group_by
99
+
100
+ private
101
+
102
+ def calculated_date_key(source)
103
+ case grain
104
+ when :hourly, :daily
105
+ "#{source.date_column.qualified_name}"
106
+ when :monthly
107
+ "to_char(date_trunc('month',#{source.date_column.qualified_name}::text::date),'YYYYMMDD')::integer"
108
+ end
109
+ end
110
+
111
+ def calculated_time_key(source)
112
+ case grain
113
+ when :hourly
114
+ "(#{source.time_key.qualified_name} - (#{source.time_key.qualified_name} % #{1.hour.seconds}))"
115
+ when :daily
116
+ "extract(EPOCH from #{source.date_column.qualified_name}::text::date)"
117
+ when :monthly
118
+ "extract(EPOCH from date_trunc('month',#{source.date_column.qualified_name}::text::date))"
119
+ end
120
+ end
121
+ end
122
+ end
123
+ end
@@ -39,14 +39,9 @@ WITH ranges AS (
39
39
  <%- target.insert_view_values.each do |value| -%>
40
40
  consolidated.<%= value %><%= ',' %>
41
41
  <%- end -%>
42
- consolidated.parent_id,
43
- consolidated.record_id,
44
42
  consolidated.start_at
45
43
  FROM (
46
- SELECT DISTINCT ON (<%= target.window('start_at').join(', ') %>)
47
- FIRST_VALUE(id) OVER w AS parent_id,
48
- FIRST_VALUE(start_at) OVER w AS parent_start_at,
49
- id AS record_id,
44
+ SELECT DISTINCT ON (<%= target.window('start_at', 'id').join(', ') %>)
50
45
  <%- target.insert_values(window: 'w').each do |value, last| -%>
51
46
  <%= value %><%= ',' %>
52
47
  <%- end -%>
@@ -54,7 +49,7 @@ WITH ranges AS (
54
49
  FROM
55
50
  windows
56
51
  WINDOW w AS (PARTITION BY <%= target.window('window_id').join(', ') %> ORDER BY start_at <%= order %>)
57
- ORDER BY <%= target.window("start_at #{order}", 'window_id').join(', ') %>
52
+ ORDER BY <%= target.window("start_at #{order}", "id DESC", 'window_id').join(', ') %>
58
53
  ) consolidated
59
54
  WHERE
60
55
  <%- target.insert_view_constraints.each do |constraint, last| -%>
@@ -62,13 +57,11 @@ WITH ranges AS (
62
57
  <%- end -%>
63
58
  )
64
59
  INSERT INTO
65
- <%= target.name %> (<%= target.insert_columns.join(', ') %>, parent_id, record_id, start_at)
60
+ <%= target.name %> (<%= target.insert_columns.join(', ') %>, start_at)
66
61
  SELECT
67
62
  <%- target.insert_view_values.each do |value| -%>
68
63
  <%= value %><%= ',' %>
69
64
  <%- end -%>
70
- parent_id,
71
- record_id,
72
65
  start_at
73
66
  FROM
74
67
  snapshot
@@ -0,0 +1,83 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Masamune::Transform::Postgres
24
+ class SnapshotDimension
25
+ def initialize(options = {})
26
+ @target = options[:target]
27
+ @source = options[:source]
28
+ @order = options[:order]
29
+ end
30
+
31
+ def locals
32
+ { target: target, source: @source, order: @order }
33
+ end
34
+
35
+ def target
36
+ TargetPresenter.new(@target)
37
+ end
38
+
39
+ private
40
+
41
+ class TargetPresenter < SimpleDelegator
42
+ include Masamune::LastElement
43
+
44
+ def insert_columns(source = nil)
45
+ consolidated_columns.map { |_, column| column.name }
46
+ end
47
+
48
+ def insert_view_values
49
+ consolidated_columns.map { |_, column| column.name }
50
+ end
51
+
52
+ def insert_view_constraints
53
+ consolidated_columns.reject { |_, column| !column.default.nil? || column.null }.map { |_, column| "#{column.name} IS NOT NULL" }
54
+ end
55
+ method_with_last_element :insert_view_constraints
56
+
57
+ def window(*extra)
58
+ (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
59
+ end
60
+
61
+ def insert_values(opts = {})
62
+ window = opts[:window]
63
+ consolidated_columns.map do |_, column|
64
+ if column.natural_key
65
+ "#{column.name} AS #{column.name}"
66
+ elsif column.type == :key_value
67
+ "hstore_merge(#{column.name}) OVER #{window} AS #{column.name}"
68
+ else
69
+ "coalesce_merge(#{column.name}) OVER #{window} AS #{column.name}"
70
+ end
71
+ end
72
+ end
73
+ method_with_last_element :insert_values
74
+
75
+ private
76
+
77
+ def consolidated_columns
78
+ unreserved_columns.reject { |_, column| column.surrogate_key }
79
+ end
80
+ end
81
+ end
82
+ end
83
+
@@ -0,0 +1,90 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Masamune::Transform::Postgres
24
+ class StageDimension
25
+ def initialize(options = {})
26
+ @target = options[:target]
27
+ @source = options[:source]
28
+ end
29
+
30
+ def locals
31
+ { target: target, source: @source }
32
+ end
33
+
34
+ def target
35
+ TargetPresenter.new(@target)
36
+ end
37
+
38
+ private
39
+
40
+ class TargetPresenter < SimpleDelegator
41
+ include Masamune::LastElement
42
+
43
+ def insert_columns(source)
44
+ shared_columns(source).values.map do |columns|
45
+ column = columns.first
46
+ if reference = column.reference
47
+ reference.foreign_key_name
48
+ else
49
+ column.name
50
+ end
51
+ end.compact
52
+ end
53
+
54
+ def insert_values(source)
55
+ shared_columns(source).values.map do |columns|
56
+ column = columns.first
57
+ if reference = column.reference
58
+ reference.surrogate_key.qualified_name(reference.label)
59
+ elsif column.type == :json || column.type == :yaml || column.type == :key_value
60
+ "json_to_hstore(#{column.qualified_name})"
61
+ else
62
+ column.qualified_name
63
+ end
64
+ end.compact
65
+ end
66
+ method_with_last_element :insert_values
67
+
68
+ def join_conditions(source)
69
+ join_columns = shared_columns(source).values.flatten
70
+ join_columns = join_columns.select { |column| column.reference }
71
+ join_columns = join_columns.group_by { |column| column.reference }
72
+
73
+ conditions = Hash.new { |h,k| h[k] = Set.new }
74
+ join_columns.each do |reference, columns|
75
+ left_uniq = Set.new
76
+ (columns + lateral_references(source, reference)).each do |column|
77
+ left = reference.columns[column.id]
78
+ next unless left_uniq.add?(left.qualified_name(reference.label))
79
+ conditions[[reference.name, reference.alias]] << "#{left.qualified_name(reference.label)} = #{column.qualified_name}"
80
+ end
81
+ end
82
+ conditions
83
+ end
84
+
85
+ def lateral_references(source, reference)
86
+ source.shared_columns(reference).keys.reject { |column| column.auto_reference }
87
+ end
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,134 @@
1
+ # The MIT License (MIT)
2
+ #
3
+ # Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ # of this software and associated documentation files (the "Software"), to deal
7
+ # in the Software without restriction, including without limitation the rights
8
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ # copies of the Software, and to permit persons to whom the Software is
10
+ # furnished to do so, subject to the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be included in
13
+ # all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ # THE SOFTWARE.
22
+
23
+ module Masamune::Transform::Postgres
24
+ class StageFact
25
+ def initialize(options = {})
26
+ @target = options[:target]
27
+ @source = options[:source]
28
+ @date = options[:date]
29
+ end
30
+
31
+ def locals
32
+ { target: target, source: @source, date: @date }
33
+ end
34
+
35
+ def target
36
+ TargetPresenter.new(@target)
37
+ end
38
+
39
+ private
40
+
41
+ class TargetPresenter < SimpleDelegator
42
+ include Masamune::LastElement
43
+
44
+ def insert_columns(source)
45
+ shared_columns(source).values.map do |columns|
46
+ column = columns.first
47
+ if reference = column.reference
48
+ reference.foreign_key_name
49
+ else
50
+ column.name
51
+ end
52
+ end
53
+ end
54
+
55
+ def insert_values(source)
56
+ shared_columns(source).values.map do |columns|
57
+ column = columns.first
58
+ if !column.degenerate? && reference = column.reference
59
+ reference.surrogate_key.qualified_name(column.reference.label)
60
+ else
61
+ column.qualified_name
62
+ end
63
+ end
64
+ end
65
+ method_with_last_element :insert_values
66
+
67
+ def join_alias(reference)
68
+ reference.label ? "#{reference.name} AS #{[reference.label, reference.name].compact.join('_')}" : reference.name
69
+ end
70
+
71
+ def join_conditions(source)
72
+ join_columns = shared_columns(source).values.flatten
73
+ join_columns = join_columns.select { |column| column.reference }
74
+ join_columns = join_columns.group_by { |column| column.reference }
75
+
76
+ dependencies = Masamune::TopologicalHash.new
77
+ conditions = Hash.new { |h,k| h[k] = [] }
78
+ join_columns.each do |reference, columns|
79
+ reference_name = join_alias(reference)
80
+ columns.each do |column|
81
+ next if column.degenerate?
82
+ dependencies[reference_name] ||= []
83
+ cross_references = cross_references(column)
84
+
85
+ coalesce_values = []
86
+
87
+ if cross_references.any?
88
+ dependencies[reference_name] += cross_references.map { |reference, _| join_alias(reference) }
89
+ coalesce_values << cross_references.map { |reference, column| column.qualified_name(reference.label) }
90
+ end
91
+
92
+ column.reference.auto_surrogate_keys.each do |auto_surrogate_key|
93
+ next unless auto_surrogate_key.default
94
+ conditions[reference_name] << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
95
+ end if column.reference
96
+
97
+ if column.reference && !column.reference.default.nil? && column.adjacent.natural_key
98
+ coalesce_values << column.reference.default(column.adjacent)
99
+ elsif column.adjacent && !column.adjacent.default.nil?
100
+ coalesce_values << column.adjacent.sql_value(column.adjacent.default)
101
+ end
102
+
103
+ conditions[reference_name] << (coalesce_values.any? ?
104
+ "#{column.foreign_key_name} = COALESCE(#{column.qualified_name}, #{coalesce_values.join(', ')})" :
105
+ "#{column.foreign_key_name} = #{column.qualified_name}")
106
+ end
107
+
108
+ if reference.type == :two || reference.type == :four
109
+ join_key_a = "TO_TIMESTAMP(#{source.time_key.qualified_name}) BETWEEN #{reference.start_key.qualified_name(reference.label)} AND COALESCE(#{reference.end_key.qualified_name(reference.label)}, 'INFINITY')"
110
+ join_key_b = "TO_TIMESTAMP(#{source.time_key.qualified_name}) < #{reference.start_key.qualified_name(reference.label)} AND #{reference.version_key.qualified_name(reference.label)} = 1"
111
+ conditions[reference_name] << "((#{join_key_a}) OR (#{join_key_b}))"
112
+ end
113
+
114
+ conditions[reference_name].uniq!
115
+ end
116
+ conditions.slice(*dependencies.tsort)
117
+ end
118
+
119
+ private
120
+
121
+ def cross_references(column)
122
+ return {} unless column.natural_key || column.adjacent.try(:natural_key)
123
+ {}.tap do |result|
124
+ column.reference.through.each do |reference_id|
125
+ reference = references[reference_id]
126
+ if reference.columns[column.id]
127
+ result[reference] = reference.columns[column.id]
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end