masamune 0.13.8 → 0.14.0

data/lib/masamune/transform/deduplicate_dimension.rb

@@ -25,47 +25,7 @@ module Masamune::Transform
     extend ActiveSupport::Concern
 
     def deduplicate_dimension(source, target)
-      Operator.new(__method__, source: source, target: target, presenters: { postgres: Postgres })
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      def insert_columns(source = nil)
-        consolidated_columns.map { |_, column| column.name }
-      end
-
-      def insert_view_values(coalesce: false)
-        consolidated_columns.map do |_, column|
-          if !column.default.nil? && coalesce
-            "COALESCE(#{column.name}, #{column.sql_value(column.default)}) AS #{column.name}"
-          else
-            column.name
-          end
-        end
-      end
-
-      def duplicate_value_conditions(window)
-        [].tap do |result|
-          consolidated_columns.map do |_, column|
-            if column.null
-              result << "((LAG(#{column.name}) OVER #{window} = #{column.name}) OR (LAG(#{column.name}) OVER #{window} IS NULL AND #{column.name} IS NULL))"
-            else
-              result << "(LAG(#{column.name}) OVER #{window} = #{column.name})"
-            end
-          end
-        end
-      end
-
-      def window(*extra)
-        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
-      end
-
-      private
-
-      def consolidated_columns
-        unreserved_columns.reject { |_, column| column.surrogate_key }
-      end
+      Operator.new(__method__, source: source, target: target)
     end
   end
 end

data/lib/masamune/transform/define_table.rb

@@ -26,121 +26,9 @@ module Masamune::Transform
 
     def define_table(target, files = [], section = nil)
       return if target.implicit
-      Operator.new(__method__, target: target, files: Masamune::Schema::Map.convert_files(files), section: section, helper: Helper, presenters: { postgres: Postgres, hive: Hive }).tap do |operator|
+      Operator.new(__method__, target: target, files: files, section: section).tap do |operator|
         logger.debug("#{target.id}\n" + operator.to_s) if target.debug
       end
     end
-
-    class Helper < SimpleDelegator
-      def files
-        locals[:files]
-      end
-
-      def section
-        locals[:section] || :all
-      end
-
-      def define_types?
-        !post_section?
-      end
-
-      def define_tables?
-        !post_section?
-      end
-
-      def define_functions?
-        !post_section?
-      end
-
-      def define_sequences?
-        !post_section?
-      end
-
-      def define_primary_keys?
-        !pre_section? && !(target.temporary? || target.primary_keys.empty?)
-      end
-
-      def define_inheritance?
-        return false unless target.inherited?
-        return false if pre_section?
-        return true if post_section?
-        !target.delay_indexes?
-      end
-
-      def define_indexes?
-        return false if pre_section?
-        return true if post_section?
-        !target.delay_indexes?
-      end
-
-      def define_foreign_keys?
-        return false if pre_section?
-        return true if post_section?
-        !target.delay_foreign_keys?
-      end
-
-      def define_unique_constraints?
-        return false if pre_section?
-        return true if post_section?
-        !target.delay_unique_constraints?
-      end
-
-      def insert_rows?
-        !post_section?
-      end
-
-      def load_files?
-        all_section?
-      end
-
-      def perform_analyze?
-        return false if pre_section?
-        return true if post_section?
-        files.any? || target.insert_rows.any?
-      end
-
-      private
-
-      def all_section?
-        section == :all
-      end
-
-      def pre_section?
-        section == :pre
-      end
-
-      def post_section?
-        section == :post
-      end
-    end
-
-    class Postgres < SimpleDelegator
-      def children
-        super.map { |child| self.class.new(child) }
-      end
-
-      def inherited?
-        type == :fact && inheritance_constraints
-      end
-
-      def delay_indexes?
-        type == :fact
-      end
-
-      def delay_foreign_keys?
-        type == :fact
-      end
-
-      def delay_unique_constraints?
-        type == :fact
-      end
-    end
-
-    class Hive < SimpleDelegator
-      def partition_by
-        return unless partitions.any?
-        partitions.map { |_, column| "#{column.name} #{column.hql_type}" }.join(', ')
-      end
-    end
   end
 end

data/lib/masamune/transform/denormalize_table.rb

@@ -32,56 +32,7 @@ module Masamune::Transform
       columns -= ['last_modified_at']
       columns.uniq!
       order_by = options[:order] || columns
-      Operator.new(__method__, target: target, columns: columns, order_by: order_by, presenters: { postgres: Common, hive: Common })
-    end
-
-    private
-
-    class Common < SimpleDelegator
-      include Masamune::LastElement
-
-      def select_columns(column_names)
-        column_names.map do |column_name|
-          next unless column = dereference_column_name(column_name)
-          if column.reference
-            if column.reference.implicit || column.reference.degenerate
-              "#{column.name} AS #{column.name}"
-            else
-              "#{column.foreign_key_name} AS #{column.name}"
-            end
-          else
-            column.qualified_name
-          end
-        end.compact
-      end
-      method_with_last_element :select_columns
-
-      def join_alias(reference)
-        reference.label ? "#{reference.name} AS #{[reference.label, reference.name].compact.join('_')}" : reference.name
-      end
-
-      def join_conditions(column_names)
-        {}.tap do |conditions|
-          column_names.each do |column_name|
-            next unless column = dereference_column_name(column_name)
-            next unless column.reference
-            next if column.reference.degenerate
-            adjacent_reference = references[column.reference.id]
-            next unless adjacent_reference
-            adjacent_column = columns[adjacent_reference.foreign_key_name]
-            next unless adjacent_column
-            conditions[join_alias(column.reference)] = "#{column.reference.surrogate_key.qualified_name(column.reference.label)} = #{adjacent_column.qualified_name}"
-          end
-        end
-      end
-
-      def order_by_columns(column_names)
-        column_names.map do |column_name|
-          next unless column = dereference_column_name(column_name)
-          column.name
-        end.compact
-      end
-      method_with_last_element :order_by_columns
+      Operator.new(__method__, target: target, columns: columns, order_by: order_by)
     end
   end
 end

data/lib/masamune/transform/hive.rb

@@ -0,0 +1,27 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module Hive
+    Dir["#{File.dirname(__FILE__)}/hive/*.rb"].each { |f| require f }
+  end
+end

data/lib/masamune/transform/hive/define_table.rb

@@ -0,0 +1,46 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform::Hive
+  class DefineTable
+    def initialize(options = {})
+      @target = options[:target]
+    end
+
+    def locals
+      { target: target }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      def partition_by
+        return unless partitions.any?
+        partitions.map { |_, column| "#{column.name} #{column.hql_type}" }.join(', ')
+      end
+    end
+  end
+end

data/lib/masamune/transform/hive/denormalize_table.rb

@@ -0,0 +1,27 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/common/denormalize_table'
+
+module Masamune::Transform::Hive
+  DenormalizeTable = Masamune::Transform::Common::DenormalizeTable
+end

data/lib/masamune/transform/insert_reference_values.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'masamune/transform/bulk_upsert'
-
 module Masamune::Transform
   module InsertReferenceValues
     extend ActiveSupport::Concern
@@ -29,36 +27,9 @@ module Masamune::Transform
     def insert_reference_values(source, target)
       operators = []
       target.insert_references.each do |_, reference|
-        operators << Operator.new(__method__, source: source, target: reference, presenters: { postgres: Postgres })
+        operators << Operator.new(__method__, source: source, target: reference)
       end
       Operator.new *operators
     end
-
-    private
-
-    class Postgres < SimpleDelegator
-      include BulkUpsert
-      include Masamune::LastElement
-
-      def insert_columns(source)
-        source.shared_columns(stage_table).map { |_, columns| columns.first.name }
-      end
-
-      def insert_values(source)
-        source.shared_columns(stage_table).map do |column, _|
-          if column.adjacent.try(:default)
-            "COALESCE(#{column.name}, #{column.adjacent.sql_value(column.adjacent.default)})"
-          else
-            column.name
-          end
-        end
-      end
-      method_with_last_element :insert_values
-
-      def insert_constraints(source)
-        source.shared_columns(stage_table).reject { |column, _| column.null || column.default || column.adjacent.try(:default) }.map { |column, _| "#{column.name} IS NOT NULL"}
-      end
-      method_with_last_element :insert_constraints
-    end
   end
 end

data/lib/masamune/transform/operator.rb

@@ -25,29 +25,7 @@ module Masamune::Transform
     def initialize(*args)
       options     = args.last.is_a?(Hash) ? args.pop : {}
       @templates  = args
-      @source     = options.delete(:source)
-      @target     = options.delete(:target)
-      @presenters = options.delete(:presenters) || {}
-      @helper     = options.delete(:helper)
-      @locals     = options
-    end
-
-    def source
-      return unless @source
-      @presenters.key?(source_store.try(:type)) ? @presenters[source_store.try(:type)].new(@source) : @source
-    end
-
-    def target
-      return unless @target
-      @presenters.key?(target_store.try(:type)) ? @presenters[target_store.try(:type)].new(@target) : @target
-    end
-
-    def helper
-      (@helper || SimpleDelegator).new(self)
-    end
-
-    def locals
-      @locals
+      @options    = options
     end
 
     def to_s
@@ -72,34 +50,55 @@ module Masamune::Transform
 
     private
 
-    def source_store
-      return @source if @source.is_a?(Masamune::Schema::Store)
-      @source.try(:store)
+    def template_eval(template)
+      return File.read(template) if File.exists?(template.to_s) && template.to_s !~ /erb\Z/
+      template_file = File.exists?(template.to_s) ? template : template_file(template)
+      if template_helper(template)
+        Masamune::Template.render_to_string(template_file, template_helper(template).new(@options).locals)
+      else
+        Masamune::Template.render_to_string(template_file, @options)
+      end
     end
 
-    def target_store
-      return @target if @target.is_a?(Masamune::Schema::Store)
-      @target.try(:store)
+    def template_helper(template_name)
+      "Masamune::Transform::#{template_type.to_s.camelize}::#{template_name.to_s.camelize}".constantize
+    rescue NameError
     end
 
-    def template_eval(template)
-      return File.read(template) if File.exists?(template.to_s) && template.to_s !~ /erb\Z/
-      template_file = File.exists?(template.to_s) ? template : template_file(template)
-      Masamune::Template.render_to_string(template_file, @locals.merge(source: source, target: target, helper: helper))
+    def template_file(template_name)
+      File.expand_path(File.join(__FILE__, '..', template_dir, "#{template_name}.#{template_suffix}.erb"))
     end
 
-    def template_file(template_prefix)
-      File.expand_path(File.join(__FILE__, '..', "#{template_prefix}.#{template_suffix}.erb"))
+    def template_type
+      @options.values.map do |value|
+        case value
+        when Masamune::Schema::Store
+          value.type
+        when Masamune::Schema::Table
+          value.store.type
+        end
+      end.first
+    end
+
+    def template_dir
+      case template_type
+      when :postgres
+        'postgres'
+      when :hive
+        'hive'
+      else
+        raise ArgumentError, "Unknown template_dir for #{template_type}"
+      end
     end
 
     def template_suffix
-      case (target_store || source_store).try(:type)
+      case template_type
       when :postgres
         'psql'
       when :hive
         'hql'
       else
-        'txt'
+        raise ArgumentError, "Unknown template_suffix for #{template_type}"
       end
     end
   end