masamune 0.11.0

data/lib/masamune/transform/relabel_dimension.psql.erb

@@ -0,0 +1,76 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+BEGIN;
+LOCK TABLE <%= target.name %> IN EXCLUSIVE MODE;
+
+<%-# Relabel version column -%>
+UPDATE <%= target.name %> SET version = NULL;
+
+UPDATE
+  <%= target.name %>
+SET
+  version = tmp.version
+FROM
+  (
+    SELECT
+      id,
+      <%- target.window.each do |name| -%>
+      <%= name %>,
+      <%- end -%>
+      start_at,
+      rank() OVER (PARTITION BY <%= target.window.join(', ') %> ORDER BY start_at) AS version
+    FROM
+      <%= target.name %>
+    GROUP BY
+      <%= target.window(:id, :start_at).join(', ') %>
+   ) AS tmp
+WHERE
+  <%= target.name %>.id = tmp.id
+;
+
+<%-# Relabel end_at column -%>
+UPDATE <%= target.name %> SET end_at = NULL;
+
+UPDATE
+  <%= target.name %>
+SET
+  end_at = tmp.end_at
+FROM
+  (
+    SELECT
+      id,
+      start_at,
+      <%- target.window.each do |name| -%>
+      <%= name %>,
+      <%- end -%>
+      LEAD(start_at, 1) OVER (PARTITION BY <%= target.window.join(', ') %> ORDER BY start_at) AS end_at
+    FROM
+      <%= target.name %>
+    GROUP BY
+      <%= target.window(:id, :start_at).join(', ') %>
+   ) AS tmp
+WHERE
+  <%= target.name %>.id = tmp.id
+;
+
+COMMIT;

data/lib/masamune/transform/relabel_dimension.rb

@@ -0,0 +1,39 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module RelabelDimension
+    extend ActiveSupport::Concern
+
+    def relabel_dimension(target)
+      Operator.new(__method__, target: target, presenters: { postgres: Postgres })
+    end
+
+    private
+
+    class Postgres < SimpleDelegator
+      def window(*extra)
+        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
+      end
+    end
+  end
+end

data/lib/masamune/transform/rollup_fact.psql.erb

@@ -0,0 +1,79 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<%- target_stage = target.stage_table %>
+
+BEGIN;
+
+DROP TABLE IF EXISTS <%= target_stage.name %> CASCADE;
+CREATE TABLE IF NOT EXISTS <%= target_stage.name %> (LIKE <%= target.parent.name %> INCLUDING ALL);
+
+ALTER TABLE <%= target_stage.name %> ADD CONSTRAINT <%= target_stage.name %>_time_key_check <%= target_stage.constraints %>;
+<%- target.foreign_key_columns.each do |column| -%>
+ALTER TABLE <%= target_stage.name %> ADD CONSTRAINT <%= target_stage.name %>_<%= column.name %>_fkey FOREIGN KEY (<%= column.name %>) <%= column.reference_constraint %>;
+<%- end -%>
+
+INSERT INTO
+  <%= target_stage.name %> (<%= target.insert_columns(source).join(', ') %>)
+SELECT
+  <%- target.insert_values(source).each do |value, last| -%>
+  <%= value %><%= ',' unless last %>
+  <%- end -%>
+FROM
+  <%= source.name %>
+<%- target.join_conditions(source).each do |table, conditions| -%>
+JOIN
+  <%= table %>
+ON
+  <%= conditions.join(" AND\n  ") %>
+<%- end -%>
+GROUP BY
+  <%- target.group_by(source).each do |value, last| -%>
+  <%= value %><%= ',' unless last %>
+  <%- end -%>
+;
+
+<%- target.index_columns.each do |column_names, unique| -%>
+<%- index_name = "#{target_stage.name}_#{column_names.join('_')}_index" -%>
+CREATE <%= unique ? 'UNIQUE INDEX' : 'INDEX' %> <%= index_name %> ON <%= target_stage.name %> (<%= column_names.join(', ') %>);
+<%- end -%>
+
+COMMIT;
+
+BEGIN;
+
+DROP TABLE IF EXISTS <%= target.name %>;
+ALTER TABLE <%= target_stage.name %> RENAME TO <%= target.name %>;
+
+ALTER TABLE <%= target.name %> INHERIT <%= target.parent.name %>;
+ALTER TABLE <%= target.name %> ADD CONSTRAINT <%= target.name %>_time_key_check <%= target.constraints %> NOT VALID;
+<%- target.foreign_key_columns.each do |column| -%>
+ALTER TABLE <%= target.name %> ADD CONSTRAINT <%= target.name %>_<%= column.name %>_fkey FOREIGN KEY (<%= column.name %>) <%= column.reference_constraint %> NOT VALID;
+<%- end -%>
+
+<%- target.index_columns.each do |column_names, unique| -%>
+<%- old_index_name = "#{target_stage.name}_#{column_names.join('_')}_index" -%>
+<%- new_index_name = "#{target.name}_#{column_names.join('_')}_index" -%>
+ALTER INDEX <%= old_index_name %> RENAME TO <%= new_index_name %>;
+<%- end -%>
+
+COMMIT;

data/lib/masamune/transform/rollup_fact.rb

@@ -0,0 +1,149 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module RollupFact
+    extend ActiveSupport::Concern
+
+    def rollup_fact(source, target, date)
+      raise ArgumentError, "#{source.name} must have date_column to rollup" unless source.date_column
+      raise ArgumentError, "#{target.name} must have date_column to rollup" unless target.date_column
+      Operator.new __method__, source: source.partition_table(date), target: target.partition_table(date), presenters: { postgres: Postgres }
+    end
+
+    private
+
+    class Postgres < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source)
+        values = []
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          next if column.id == :last_modified_at
+          next if column.auto_reference
+          values << column.name
+        end
+        measures.each do |_ ,measure|
+          values << measure.name
+        end
+        values << time_key.name
+        values.compact
+      end
+
+      def insert_values(source)
+        values = []
+        values << "(#{first_date_surrogate_key})"
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          next unless column.reference
+          next if column.reference.type == :date
+          next if column.auto_reference
+          values << column.qualified_name
+        end
+        source.measures.each do |_ ,measure|
+          values << measure.aggregate_value
+        end
+        values << "(#{floor_time_key(source)})"
+        values
+      end
+      method_with_last_element :insert_values
+
+      def join_conditions(source)
+        {
+          source.date_column.reference.name => [
+            "#{source.date_column.reference.surrogate_key.qualified_name} = #{source.date_column.qualified_name}"
+          ]
+        }
+      end
+
+      def group_by(source)
+        group_by = []
+        group_by << date_column.reference.columns[rollup_key].qualified_name
+        shared_columns(source).values.map do |columns|
+          column = columns.first
+          next unless column.reference
+          next if column.reference.type == :date
+          next if column.auto_reference
+          group_by << column.qualified_name
+        end
+        group_by << "(#{floor_time_key(source)})" if grain == :hourly
+        group_by
+      end
+      method_with_last_element :group_by
+
+      private
+
+      def rollup_key
+        case grain
+        when :hourly
+          :date_epoch
+        when :daily
+          :date_epoch
+        when :monthly
+          :month_epoch
+        end
+      end
+
+      def date_key
+        :date_id
+      end
+
+      def first_date_surrogate_key
+        <<-EOS.gsub(/\s+/, ' ').strip
+          SELECT
+            #{date_column.reference.surrogate_key.name}
+          FROM
+            #{date_column.reference.name} d
+          WHERE
+            d.#{rollup_key} = #{date_column.reference.columns[rollup_key].qualified_name}
+          ORDER BY
+            d.#{date_key}
+          LIMIT 1
+        EOS
+      end
+
+      def floor_time_key(source)
+        case grain
+        when :hourly
+          "#{source.time_key.qualified_name} - (#{source.time_key.qualified_name} % #{1.hour.seconds})"
+        when :daily, :monthly
+          first_date_time_key
+        end
+      end
+
+      def first_date_time_key
+        <<-EOS.gsub(/\s+/, ' ').strip
+          SELECT
+            #{rollup_key}
+          FROM
+            #{date_column.reference.name} d
+          WHERE
+            d.#{rollup_key} = #{date_column.reference.columns[rollup_key].qualified_name}
+          ORDER BY
+            d.#{date_key}
+          LIMIT 1
+        EOS
+      end
+    end
+  end
+end

data/lib/masamune/transform/snapshot_dimension.psql.erb

@@ -0,0 +1,75 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<%#
+Consolidate complete events from partial events
+For discussion on range and group constructions, see:
+http://www.postgresql.org/message-id/CAGnEbohhKmW55oB0FpQd3naXBkwi74E=8DRZBGFjS-MeGpXLaA@mail.gmail.com
+-%>
+WITH ranges AS (
+  SELECT *,
+  CASE WHEN delta = 0
+  THEN 1 ELSE NULL END r
+  FROM <%= source.name %>
+), windows AS (
+  SELECT *,
+  SUM(r) OVER (ORDER BY <%= target.window("start_at #{order}", 'delta', 'source_uuid').join(', ') %>) window_id
+  FROM ranges
+), snapshot AS (
+  SELECT
+    <%- target.insert_view_values.each do |value| -%>
+    consolidated.<%= value %><%= ',' %>
+    <%- end -%>
+    consolidated.parent_id,
+    consolidated.record_id,
+    consolidated.start_at
+  FROM (
+    SELECT DISTINCT ON (<%= target.window('start_at').join(', ') %>)
+      FIRST_VALUE(id) OVER w AS parent_id,
+      FIRST_VALUE(start_at) OVER w AS parent_start_at,
+      id AS record_id,
+      <%- target.insert_values(window: 'w').each do |value, last| -%>
+      <%= value %><%= ',' %>
+      <%- end -%>
+      start_at AS start_at
+    FROM
+      windows
+    WINDOW w AS (PARTITION BY <%= target.window('window_id').join(', ') %> ORDER BY start_at <%= order %>)
+    ORDER BY <%= target.window("start_at #{order}", 'window_id').join(', ') %>
+  ) consolidated
+  WHERE
+    <%- target.insert_view_constraints.each do |constraint, last| -%>
+    consolidated.<%= constraint %><%= ' AND' unless last %>
+    <%- end -%>
+)
+INSERT INTO
+  <%= target.name %> (<%= target.insert_columns.join(', ') %>, parent_id, record_id, start_at)
+SELECT
+  <%- target.insert_view_values.each do |value| -%>
+  <%= value %><%= ',' %>
+  <%- end -%>
+  parent_id,
+  record_id,
+  start_at
+FROM
+  snapshot
+;

data/lib/masamune/transform/snapshot_dimension.rb

@@ -0,0 +1,74 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module SnapshotDimension
+    extend ActiveSupport::Concern
+
+    def snapshot_dimension(source, target, order = 'DESC')
+      Operator.new(__method__, source: source, target: target, order: order, presenters: { postgres: Postgres })
+    end
+
+    private
+
+    class Postgres < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source = nil)
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def insert_view_values
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def insert_view_constraints
+        consolidated_columns.reject { |_, column| column.null }.map { |_, column| "#{column.name} IS NOT NULL" }
+      end
+      method_with_last_element :insert_view_constraints
+
+      def window(*extra)
+        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
+      end
+
+      def insert_values(opts = {})
+        window = opts[:window]
+        consolidated_columns.map do |_, column|
+          if column.natural_key
+            "#{column.name} AS #{column.name}"
+          elsif column.type == :key_value
+            "hstore_merge(#{column.name}_now) OVER #{window} - hstore_merge(#{column.name}_was) OVER #{window} AS #{column.name}"
+          else
+            "coalesce_merge(#{column.name}) OVER #{window} AS #{column.name}"
+          end
+        end
+      end
+      method_with_last_element :insert_values
+
+      private
+
+      def consolidated_columns
+        unreserved_columns.reject { |_, column| column.surrogate_key }
+      end
+    end
+  end
+end