masamune 0.13.8 → 0.14.0

data/lib/masamune/transform/{relabel_dimension.psql.erb → postgres/relabel_dimension.psql.erb}

@@ -20,8 +20,9 @@
 --  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 --  THE SOFTWARE.
 
+SELECT pg_advisory_lock(<%= target.lock_id %>);
+
 BEGIN;
-LOCK TABLE <%= target.name %> IN EXCLUSIVE MODE;
 
 <%-# Relabel version column -%>
 UPDATE <%= target.name %> SET version = NULL;
@@ -74,3 +75,5 @@ WHERE
 ;
 
 COMMIT;
+
+SELECT pg_advisory_unlock(<%= target.lock_id %>);

data/lib/masamune/transform/postgres/relabel_dimension.rb

@@ -0,0 +1,45 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class RelabelDimension
+    def initialize(options = {})
+      @target   = options[:target]
+    end
+
+    def locals
+      { target: target }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      def window(*extra)
+        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
+      end
+    end
+  end
+end

data/lib/masamune/transform/postgres/rollup_fact.rb

@@ -0,0 +1,123 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class RollupFact
+    def initialize(options = {})
+      @target = options[:target]
+      @source = options[:source]
+      @date   = options[:date]
+    end
+
+    def locals
+      { target: target, source: @source, date: @date }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source)
+        values = []
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          next if column.id == :last_modified_at
+          next if column.auto_reference
+          values << column.name
+        end
+        measures.each do |_ ,measure|
+          values << measure.name
+        end
+        values << time_key.name
+        values.compact
+      end
+
+      def insert_values(source)
+        values = []
+        values << calculated_date_key(source)
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          next unless column.reference
+          next if column.reference.type == :date
+          next if column.auto_reference
+          values << column.qualified_name
+        end
+        source.measures.each do |_ ,measure|
+          values << measure.aggregate_value
+        end
+        values << calculated_time_key(source)
+        values
+      end
+      method_with_last_element :insert_values
+
+      def join_conditions(source)
+        {
+          source.date_column.reference.name => [
+            "#{source.date_column.reference.surrogate_key.qualified_name} = #{source.date_column.qualified_name}"
+          ]
+        }
+      end
+
+      def group_by(source)
+        group_by = []
+        group_by << calculated_date_key(source)
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          next unless column.reference
+          next if column.reference.type == :date
+          next if column.auto_reference
+          group_by << column.qualified_name
+        end
+        group_by << calculated_time_key(source)
+        group_by
+      end
+      method_with_last_element :group_by
+
+      private
+
+      def calculated_date_key(source)
+        case grain
+        when :hourly, :daily
+          "#{source.date_column.qualified_name}"
+        when :monthly
+          "to_char(date_trunc('month',#{source.date_column.qualified_name}::text::date),'YYYYMMDD')::integer"
+        end
+      end
+
+      def calculated_time_key(source)
+        case grain
+        when :hourly
+          "(#{source.time_key.qualified_name} - (#{source.time_key.qualified_name} % #{1.hour.seconds}))"
+        when :daily
+          "extract(EPOCH from #{source.date_column.qualified_name}::text::date)"
+        when :monthly
+          "extract(EPOCH from date_trunc('month',#{source.date_column.qualified_name}::text::date))"
+        end
+      end
+    end
+  end
+end

data/lib/masamune/transform/{snapshot_dimension.psql.erb → postgres/snapshot_dimension.psql.erb}

@@ -39,14 +39,9 @@ WITH ranges AS (
     <%- target.insert_view_values.each do |value| -%>
     consolidated.<%= value %><%= ',' %>
     <%- end -%>
-    consolidated.parent_id,
-    consolidated.record_id,
     consolidated.start_at
   FROM (
-    SELECT DISTINCT ON (<%= target.window('start_at').join(', ') %>)
-      FIRST_VALUE(id) OVER w AS parent_id,
-      FIRST_VALUE(start_at) OVER w AS parent_start_at,
-      id AS record_id,
+    SELECT DISTINCT ON (<%= target.window('start_at', 'id').join(', ') %>)
       <%- target.insert_values(window: 'w').each do |value, last| -%>
       <%= value %><%= ',' %>
       <%- end -%>
@@ -54,7 +49,7 @@ WITH ranges AS (
     FROM
       windows
     WINDOW w AS (PARTITION BY <%= target.window('window_id').join(', ') %> ORDER BY start_at <%= order %>)
-    ORDER BY <%= target.window("start_at #{order}", 'window_id').join(', ') %>
+    ORDER BY <%= target.window("start_at #{order}", "id DESC", 'window_id').join(', ') %>
   ) consolidated
   WHERE
     <%- target.insert_view_constraints.each do |constraint, last| -%>
@@ -62,13 +57,11 @@ WITH ranges AS (
     <%- end -%>
 )
 INSERT INTO
-  <%= target.name %> (<%= target.insert_columns.join(', ') %>, parent_id, record_id, start_at)
+  <%= target.name %> (<%= target.insert_columns.join(', ') %>, start_at)
 SELECT
   <%- target.insert_view_values.each do |value| -%>
   <%= value %><%= ',' %>
   <%- end -%>
-  parent_id,
-  record_id,
   start_at
 FROM
   snapshot

data/lib/masamune/transform/postgres/snapshot_dimension.rb

@@ -0,0 +1,83 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class SnapshotDimension
+    def initialize(options = {})
+      @target   = options[:target]
+      @source   = options[:source]
+      @order    = options[:order]
+    end
+
+    def locals
+      { target: target, source: @source, order: @order }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source = nil)
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def insert_view_values
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def insert_view_constraints
+        consolidated_columns.reject { |_, column| !column.default.nil? || column.null }.map { |_, column| "#{column.name} IS NOT NULL" }
+      end
+      method_with_last_element :insert_view_constraints
+
+      def window(*extra)
+        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
+      end
+
+      def insert_values(opts = {})
+        window = opts[:window]
+        consolidated_columns.map do |_, column|
+          if column.natural_key
+            "#{column.name} AS #{column.name}"
+          elsif column.type == :key_value
+            "hstore_merge(#{column.name}) OVER #{window} AS #{column.name}"
+          else
+            "coalesce_merge(#{column.name}) OVER #{window} AS #{column.name}"
+          end
+        end
+      end
+      method_with_last_element :insert_values
+
+      private
+
+      def consolidated_columns
+        unreserved_columns.reject { |_, column| column.surrogate_key }
+      end
+    end
+  end
+end
+

data/lib/masamune/transform/postgres/stage_dimension.rb

@@ -0,0 +1,90 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class StageDimension
+    def initialize(options = {})
+      @target = options[:target]
+      @source = options[:source]
+    end
+
+    def locals
+      { target: target, source: @source }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source)
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          if reference = column.reference
+            reference.foreign_key_name
+          else
+            column.name
+          end
+        end.compact
+      end
+
+      def insert_values(source)
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          if reference = column.reference
+            reference.surrogate_key.qualified_name(reference.label)
+          elsif column.type == :json || column.type == :yaml || column.type == :key_value
+            "json_to_hstore(#{column.qualified_name})"
+          else
+            column.qualified_name
+          end
+        end.compact
+      end
+      method_with_last_element :insert_values
+
+      def join_conditions(source)
+        join_columns = shared_columns(source).values.flatten
+        join_columns = join_columns.select { |column| column.reference }
+        join_columns = join_columns.group_by { |column| column.reference }
+
+        conditions = Hash.new { |h,k| h[k] = Set.new }
+        join_columns.each do |reference, columns|
+          left_uniq = Set.new
+          (columns + lateral_references(source, reference)).each do |column|
+            left = reference.columns[column.id]
+            next unless left_uniq.add?(left.qualified_name(reference.label))
+            conditions[[reference.name, reference.alias]] << "#{left.qualified_name(reference.label)} = #{column.qualified_name}"
+          end
+        end
+        conditions
+      end
+
+      def lateral_references(source, reference)
+        source.shared_columns(reference).keys.reject { |column| column.auto_reference }
+      end
+    end
+  end
+end

data/lib/masamune/transform/postgres/stage_fact.rb

@@ -0,0 +1,134 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class StageFact
+    def initialize(options = {})
+      @target = options[:target]
+      @source = options[:source]
+      @date   = options[:date]
+    end
+
+    def locals
+      { target: target, source: @source, date: @date }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source)
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          if reference = column.reference
+            reference.foreign_key_name
+          else
+            column.name
+          end
+        end
+      end
+
+      def insert_values(source)
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          if !column.degenerate? && reference = column.reference
+            reference.surrogate_key.qualified_name(column.reference.label)
+          else
+            column.qualified_name
+          end
+        end
+      end
+      method_with_last_element :insert_values
+
+      def join_alias(reference)
+        reference.label ? "#{reference.name} AS #{[reference.label, reference.name].compact.join('_')}" : reference.name
+      end
+
+      def join_conditions(source)
+        join_columns = shared_columns(source).values.flatten
+        join_columns = join_columns.select { |column| column.reference }
+        join_columns = join_columns.group_by { |column| column.reference }
+
+        dependencies = Masamune::TopologicalHash.new
+        conditions = Hash.new { |h,k| h[k] = [] }
+        join_columns.each do |reference, columns|
+          reference_name = join_alias(reference)
+          columns.each do |column|
+            next if column.degenerate?
+            dependencies[reference_name] ||= []
+            cross_references = cross_references(column)
+
+            coalesce_values = []
+
+            if cross_references.any?
+              dependencies[reference_name] += cross_references.map { |reference, _| join_alias(reference) }
+              coalesce_values << cross_references.map { |reference, column| column.qualified_name(reference.label) }
+            end
+
+            column.reference.auto_surrogate_keys.each do |auto_surrogate_key|
+              next unless auto_surrogate_key.default
+              conditions[reference_name] << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
+            end if column.reference
+
+            if column.reference && !column.reference.default.nil? && column.adjacent.natural_key
+              coalesce_values << column.reference.default(column.adjacent)
+            elsif column.adjacent && !column.adjacent.default.nil?
+              coalesce_values << column.adjacent.sql_value(column.adjacent.default)
+            end
+
+            conditions[reference_name] << (coalesce_values.any? ?
+              "#{column.foreign_key_name} = COALESCE(#{column.qualified_name}, #{coalesce_values.join(', ')})" :
+              "#{column.foreign_key_name} = #{column.qualified_name}")
+          end
+
+          if reference.type == :two || reference.type == :four
+            join_key_a = "TO_TIMESTAMP(#{source.time_key.qualified_name}) BETWEEN #{reference.start_key.qualified_name(reference.label)} AND COALESCE(#{reference.end_key.qualified_name(reference.label)}, 'INFINITY')"
+            join_key_b = "TO_TIMESTAMP(#{source.time_key.qualified_name}) < #{reference.start_key.qualified_name(reference.label)} AND #{reference.version_key.qualified_name(reference.label)} = 1"
+            conditions[reference_name] << "((#{join_key_a}) OR (#{join_key_b}))"
+          end
+
+          conditions[reference_name].uniq!
+        end
+        conditions.slice(*dependencies.tsort)
+      end
+
+      private
+
+      def cross_references(column)
+        return {} unless column.natural_key || column.adjacent.try(:natural_key)
+        {}.tap do |result|
+          column.reference.through.each do |reference_id|
+            reference = references[reference_id]
+            if reference.columns[column.id]
+              result[reference] = reference.columns[column.id]
+            end
+          end
+        end
+      end
+    end
+  end
+end