masamune 0.13.8 → 0.14.0

data/lib/masamune/transform/postgres.rb

@@ -0,0 +1,27 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module Postgres
+    Dir["#{File.dirname(__FILE__)}/postgres/*.rb"].each { |f| require f }
+  end
+end

data/lib/masamune/transform/postgres/bulk_upsert.rb

@@ -0,0 +1,62 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/last_element'
+
+module Masamune::Transform::Postgres
+  class BulkUpsert
+    def initialize(options = {})
+      @target   = options[:target]
+      @source   = options[:source]
+    end
+
+    def locals
+      { target: target, source: @source }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def update_columns
+        columns.values.reject { |column| reserved_column_ids.include?(column.id) || column.surrogate_key || column.natural_key || column.unique.any? || column.auto_reference || column.ignore }
+      end
+      method_with_last_element :update_columns
+
+      def insert_columns
+        columns.values.reject { |column| column.surrogate_key || column.auto_reference || column.ignore }
+      end
+      method_with_last_element :insert_columns
+
+      def unique_columns
+        columns.values.select { |column| column.unique.any? && !column.null }
+      end
+      method_with_last_element :unique_columns
+    end
+  end
+end
+

data/lib/masamune/transform/{deduplicate_dimension.psql.erb → postgres/deduplicate_dimension.psql.erb}

@@ -25,28 +25,22 @@ WITH consolidated AS (
     <%- target.insert_view_values(coalesce: true).each do |value| -%>
     <%= value %><%= ',' %>
     <%- end -%>
-    parent_id,
-    record_id,
     start_at
   FROM
     <%= source.name %>
 )
 INSERT INTO
-  <%= target.name %> (<%= target.insert_columns.join(', ') %>, parent_id, record_id, start_at)
+  <%= target.name %> (<%= target.insert_columns.join(', ') %>, start_at)
 SELECT DISTINCT
   <%- target.insert_view_values.each do |value| -%>
   <%= value %><%= ',' %>
   <%- end -%>
-  parent_id,
-  record_id,
   start_at
 FROM (
   SELECT
     <%- target.insert_view_values.each do |value| -%>
     <%= value %><%= ',' %>
     <%- end -%>
-    parent_id,
-    record_id,
     start_at,
     CASE
     WHEN <%= target.duplicate_value_conditions('w').join(' AND ') %> THEN

data/lib/masamune/transform/postgres/deduplicate_dimension.rb

@@ -0,0 +1,79 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class DeduplicateDimension
+    def initialize(options = {})
+      @target   = options[:target]
+      @source   = options[:source]
+    end
+
+    def locals
+      { target: target, source: @source }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      def insert_columns(source = nil)
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def insert_view_values(coalesce: false)
+        consolidated_columns.map do |_, column|
+          if !column.default.nil? && coalesce
+            "COALESCE(#{column.name}, #{column.sql_value(column.default)}) AS #{column.name}"
+          else
+            column.name
+          end
+        end
+      end
+
+      def duplicate_value_conditions(window)
+        [].tap do |result|
+          consolidated_columns.map do |_, column|
+            if column.null
+              result << "((LAG(#{column.name}) OVER #{window} = #{column.name}) OR (LAG(#{column.name}) OVER #{window} IS NULL AND #{column.name} IS NULL))"
+            else
+              result << "(LAG(#{column.name}) OVER #{window} = #{column.name})"
+            end
+          end
+        end
+      end
+
+      def window(*extra)
+        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
+      end
+
+      private
+
+      def consolidated_columns
+        unreserved_columns.reject { |_, column| column.surrogate_key }
+      end
+    end
+  end
+end
+

data/lib/masamune/transform/postgres/define_table.rb

@@ -0,0 +1,142 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Postgres
+  class DefineTable 
+    def initialize(options = {})
+      @target   = options[:target]
+      @files    = options[:files] || []
+      @section  = options[:section]
+    end
+
+    def locals
+      { target: target, files: files, helper: self }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    def files
+      Masamune::Schema::Map.convert_files(@files)
+    end
+
+    def section
+      @section || :all
+    end
+
+    def define_types?
+      !post_section?
+    end
+
+    def define_tables?
+      !post_section?
+    end
+
+    def define_functions?
+      !post_section?
+    end
+
+    def define_sequences?
+      !post_section?
+    end
+
+    def define_primary_keys?
+      !pre_section? && !(target.temporary? || target.primary_keys.empty?)
+    end
+
+    def define_inheritance?
+      return false unless target.inherited?
+      return false if pre_section?
+      return true if post_section?
+      !target.delay_indexes?
+    end
+
+    def define_indexes?
+      return false if pre_section?
+      return true if post_section?
+      !target.delay_indexes?
+    end
+
+    def define_foreign_keys?
+      return false if pre_section?
+      return true if post_section?
+      !target.delay_foreign_keys?
+    end
+
+    def define_unique_constraints?
+      return false if pre_section?
+      return true if post_section?
+      !target.delay_unique_constraints?
+    end
+
+    def insert_rows?
+      !post_section?
+    end
+
+    def load_files?
+      all_section?
+    end
+
+    def perform_analyze?
+      return false if pre_section?
+      return true if post_section?
+      files.any? || target.insert_rows.any?
+    end
+
+    private
+
+    def all_section?
+      section == :all
+    end
+
+    def pre_section?
+      section == :pre
+    end
+
+    def post_section?
+      section == :post
+    end
+
+    class TargetPresenter < SimpleDelegator
+      def children
+        super.map { |child| self.class.new(child) }
+      end
+
+      def inherited?
+        type == :fact && inheritance_constraints
+      end
+
+      def delay_indexes?
+        type == :fact
+      end
+
+      def delay_foreign_keys?
+        type == :fact
+      end
+
+      def delay_unique_constraints?
+        type == :fact
+      end
+    end
+  end
+end

data/lib/masamune/transform/postgres/denormalize_table.rb

@@ -0,0 +1,27 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/common/denormalize_table'
+
+module Masamune::Transform::Postgres
+  DenormalizeTable = Masamune::Transform::Common::DenormalizeTable
+end

data/lib/masamune/transform/{insert_reference_values.psql.erb → postgres/insert_reference_values.psql.erb}

@@ -41,5 +41,5 @@ WHERE
 
 ANALYZE <%= target.stage_table.name %>;
 
-<%= target.bulk_upsert(target.stage_table, target) %>
+<%= helper.bulk_upsert(target.stage_table, target) %>
 <%- end -%>

data/lib/masamune/transform/postgres/insert_reference_values.rb

@@ -0,0 +1,69 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/bulk_upsert'
+
+module Masamune::Transform::Postgres
+  class InsertReferenceValues
+    include Masamune::Transform::BulkUpsert
+
+    def initialize(options = {})
+      @target   = options[:target]
+      @source   = options[:source]
+    end
+
+    def locals
+      { target: target, source: @source, helper: self }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def insert_columns(source)
+        source.shared_columns(stage_table).map { |_, columns| columns.first.name }
+      end
+
+      def insert_values(source)
+        source.shared_columns(stage_table).map do |column, _|
+          if column.adjacent.try(:default)
+            "COALESCE(#{column.name}, #{column.adjacent.sql_value(column.adjacent.default)})"
+          else
+            column.name
+          end
+        end
+      end
+      method_with_last_element :insert_values
+
+      def insert_constraints(source)
+        source.shared_columns(stage_table).reject { |column, _| column.null || column.default || column.adjacent.try(:default) }.map { |column, _| "#{column.name} IS NOT NULL"}
+      end
+      method_with_last_element :insert_constraints
+    end
+  end
+end
+