masamune 0.13.8 → 0.14.0

data/lib/masamune/transform/relabel_dimension.rb

@@ -25,15 +25,7 @@ module Masamune::Transform
     extend ActiveSupport::Concern
 
     def relabel_dimension(target)
-      Operator.new(__method__, target: target, presenters: { postgres: Postgres })
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      def window(*extra)
-        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
-      end
+      Operator.new(__method__, target: target)
     end
   end
 end

data/lib/masamune/transform/rollup_fact.rb

@@ -27,92 +27,7 @@ module Masamune::Transform
     def rollup_fact(source, target, date)
       raise ArgumentError, "#{source.name} must have date_column to rollup" unless source.date_column
       raise ArgumentError, "#{target.name} must have date_column to rollup" unless target.date_column
-      Operator.new __method__, source: source.partition_table(date), target: target.partition_table(date), presenters: { postgres: Postgres }
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      include Masamune::LastElement
-
-      def insert_columns(source)
-        values = []
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          next if column.id == :last_modified_at
-          next if column.auto_reference
-          values << column.name
-        end
-        measures.each do |_ ,measure|
-          values << measure.name
-        end
-        values << time_key.name
-        values.compact
-      end
-
-      def insert_values(source)
-        values = []
-        values << calculated_date_key(source)
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          next unless column.reference
-          next if column.reference.type == :date
-          next if column.auto_reference
-          values << column.qualified_name
-        end
-        source.measures.each do |_ ,measure|
-          values << measure.aggregate_value
-        end
-        values << calculated_time_key(source)
-        values
-      end
-      method_with_last_element :insert_values
-
-      def join_conditions(source)
-        {
-          source.date_column.reference.name => [
-            "#{source.date_column.reference.surrogate_key.qualified_name} = #{source.date_column.qualified_name}"
-          ]
-        }
-      end
-
-      def group_by(source)
-        group_by = []
-        group_by << calculated_date_key(source)
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          next unless column.reference
-          next if column.reference.type == :date
-          next if column.auto_reference
-          group_by << column.qualified_name
-        end
-        group_by << calculated_time_key(source)
-        group_by
-      end
-      method_with_last_element :group_by
-
-      private
-
-      def calculated_date_key(source)
-        case grain
-        when :hourly, :daily
-          "#{source.date_column.qualified_name}"
-        when :monthly
-          "to_char(date_trunc('month',#{source.date_column.qualified_name}::text::date),'YYYYMMDD')::integer"
-        end
-      end
-
-      def calculated_time_key(source)
-        case grain
-        when :hourly
-          "(#{source.time_key.qualified_name} - (#{source.time_key.qualified_name} % #{1.hour.seconds}))"
-        when :daily
-          "extract(EPOCH from #{source.date_column.qualified_name}::text::date)"
-        when :monthly
-          "extract(EPOCH from date_trunc('month',#{source.date_column.qualified_name}::text::date))"
-        end
-      end
-
+      Operator.new __method__, source: source.partition_table(date), target: target.partition_table(date)
     end
   end
 end

data/lib/masamune/transform/snapshot_dimension.rb

@@ -25,50 +25,7 @@ module Masamune::Transform
     extend ActiveSupport::Concern
 
     def snapshot_dimension(source, target, order = 'DESC')
-      Operator.new(__method__, source: source, target: target, order: order, presenters: { postgres: Postgres })
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      include Masamune::LastElement
-
-      def insert_columns(source = nil)
-        consolidated_columns.map { |_, column| column.name }
-      end
-
-      def insert_view_values
-        consolidated_columns.map { |_, column| column.name }
-      end
-
-      def insert_view_constraints
-        consolidated_columns.reject { |_, column| !column.default.nil? || column.null }.map { |_, column| "#{column.name} IS NOT NULL" }
-      end
-      method_with_last_element :insert_view_constraints
-
-      def window(*extra)
-        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
-      end
-
-      def insert_values(opts = {})
-        window = opts[:window]
-        consolidated_columns.map do |_, column|
-          if column.natural_key
-            "#{column.name} AS #{column.name}"
-          elsif column.type == :key_value
-            "hstore_merge(#{column.name}) OVER #{window} AS #{column.name}"
-          else
-            "coalesce_merge(#{column.name}) OVER #{window} AS #{column.name}"
-          end
-        end
-      end
-      method_with_last_element :insert_values
-
-      private
-
-      def consolidated_columns
-        unreserved_columns.reject { |_, column| column.surrogate_key }
-      end
+      Operator.new(__method__, source: source, target: target, order: order)
     end
   end
 end

data/lib/masamune/transform/stage_dimension.rb

@@ -25,59 +25,7 @@ module Masamune::Transform
     extend ActiveSupport::Concern
 
     def stage_dimension(source, target)
-      Operator.new(__method__, source: source, target: target, presenters: { postgres: Postgres })
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      include Masamune::LastElement
-
-      def insert_columns(source)
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          if reference = column.reference
-            reference.foreign_key_name
-          else
-            column.name
-          end
-        end.compact
-      end
-
-      def insert_values(source)
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          if reference = column.reference
-            reference.surrogate_key.qualified_name(reference.label)
-          elsif column.type == :json || column.type == :yaml || column.type == :key_value
-            "json_to_hstore(#{column.qualified_name})"
-          else
-            column.qualified_name
-          end
-        end.compact
-      end
-      method_with_last_element :insert_values
-
-      def join_conditions(source)
-        join_columns = shared_columns(source).values.flatten
-        join_columns = join_columns.select { |column| column.reference }
-        join_columns = join_columns.group_by { |column| column.reference }
-
-        conditions = Hash.new { |h,k| h[k] = Set.new }
-        join_columns.each do |reference, columns|
-          left_uniq = Set.new
-          (columns + lateral_references(source, reference)).each do |column|
-            left = reference.columns[column.id]
-            next unless left_uniq.add?(left.qualified_name(reference.label))
-            conditions[[reference.name, reference.alias]] << "#{left.qualified_name(reference.label)} = #{column.qualified_name}"
-          end
-        end
-        conditions
-      end
-
-      def lateral_references(source, reference)
-        source.shared_columns(reference).keys.reject { |column| column.auto_reference }
-      end
+      Operator.new(__method__, source: source, target: target)
     end
   end
 end

data/lib/masamune/transform/stage_fact.rb

@@ -25,102 +25,7 @@ module Masamune::Transform
     extend ActiveSupport::Concern
 
     def stage_fact(source, target, date)
-      Operator.new(__method__, source: source, target: target, date: date, presenters: { postgres: Postgres })
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      include Masamune::LastElement
-
-      def insert_columns(source)
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          if reference = column.reference
-            reference.foreign_key_name
-          else
-            column.name
-          end
-        end
-      end
-
-      def insert_values(source)
-        shared_columns(source).values.map do |columns|
-          column = columns.first
-          if !column.degenerate? && reference = column.reference
-            reference.surrogate_key.qualified_name(column.reference.label)
-          else
-            column.qualified_name
-          end
-        end
-      end
-      method_with_last_element :insert_values
-
-      def join_alias(reference)
-        reference.label ? "#{reference.name} AS #{[reference.label, reference.name].compact.join('_')}" : reference.name
-      end
-
-      def join_conditions(source)
-        join_columns = shared_columns(source).values.flatten
-        join_columns = join_columns.select { |column| column.reference }
-        join_columns = join_columns.group_by { |column| column.reference }
-
-        dependencies = Masamune::TopologicalHash.new
-        conditions = Hash.new { |h,k| h[k] = [] }
-        join_columns.each do |reference, columns|
-          reference_name = join_alias(reference)
-          columns.each do |column|
-            next if column.degenerate?
-            dependencies[reference_name] ||= []
-            cross_references = cross_references(column)
-
-            coalesce_values = []
-
-            if cross_references.any?
-              dependencies[reference_name] += cross_references.map { |reference, _| join_alias(reference) }
-              coalesce_values << cross_references.map { |reference, column| column.qualified_name(reference.label) }
-            end
-
-            column.reference.auto_surrogate_keys.each do |auto_surrogate_key|
-              next unless auto_surrogate_key.default
-              conditions[reference_name] << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
-            end if column.reference
-
-            if column.reference && !column.reference.default.nil? && column.adjacent.natural_key
-              coalesce_values << column.reference.default(column.adjacent)
-            elsif column.adjacent && !column.adjacent.default.nil?
-              coalesce_values << column.adjacent.sql_value(column.adjacent.default)
-            end
-
-            conditions[reference_name] << (coalesce_values.any? ?
-              "#{column.foreign_key_name} = COALESCE(#{column.qualified_name}, #{coalesce_values.join(', ')})" :
-              "#{column.foreign_key_name} = #{column.qualified_name}")
-          end
-
-          if reference.type == :two || reference.type == :four
-            join_key_a = "TO_TIMESTAMP(#{source.time_key.qualified_name}) BETWEEN #{reference.start_key.qualified_name(reference.label)} AND COALESCE(#{reference.end_key.qualified_name(reference.label)}, 'INFINITY')"
-            join_key_b = "TO_TIMESTAMP(#{source.time_key.qualified_name}) < #{reference.start_key.qualified_name(reference.label)} AND #{reference.version_key.qualified_name(reference.label)} = 1"
-            conditions[reference_name] << "((#{join_key_a}) OR (#{join_key_b}))"
-          end
-
-          conditions[reference_name].uniq!
-        end
-        conditions.slice(*dependencies.tsort)
-      end
-
-      private
-
-      def cross_references(column)
-        return {} unless column.natural_key || column.adjacent.try(:natural_key)
-        {}.tap do |result|
-          column.reference.through.each do |reference_id|
-            reference = references[reference_id]
-            if reference.columns[column.id]
-              result[reference] = reference.columns[column.id]
-            end
-          end
-        end
-      end
+      Operator.new(__method__, source: source, target: target, date: date)
     end
   end
 end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.13.8'
+  VERSION = '0.14.0'
 end

data/spec/masamune/template_spec.rb

@@ -75,7 +75,7 @@ describe Masamune::Template do
     end
 
     context 'with packaged template' do
-      let(:template) { 'define_schema.hql.erb' }
+      let(:template) { 'hive/define_schema.hql.erb' }
       it { is_expected.to_not be_nil }
     end
   end

data/spec/masamune/transform/bulk_upsert.dimension_spec.rb

@@ -101,7 +101,7 @@ describe Masamune::Transform::BulkUpsert do
         ;
 
         INSERT INTO
-          user_dimension (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences, parent_id, record_id, start_at, end_at, version, last_modified_at)
+          user_dimension (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences, start_at, end_at, version, last_modified_at)
         SELECT
           user_dimension_stage.department_type_id,
           user_dimension_stage.user_account_state_type_id,
@@ -110,8 +110,6 @@ describe Masamune::Transform::BulkUpsert do
           user_dimension_stage.user_id,
           user_dimension_stage.name,
           user_dimension_stage.preferences,
-          user_dimension_stage.parent_id,
-          user_dimension_stage.record_id,
           user_dimension_stage.start_at,
           user_dimension_stage.end_at,
           user_dimension_stage.version,

data/spec/masamune/transform/deduplicate_dimension_spec.rb

@@ -52,21 +52,17 @@ describe Masamune::Transform::DeduplicateDimension do
             tenant_id,
             user_id,
             preferences,
-            parent_id,
-            record_id,
             start_at
           FROM
             user_consolidated_dimension_stage
         )
         INSERT INTO
-          user_deduplicated_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, parent_id, record_id, start_at)
+          user_deduplicated_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, start_at)
         SELECT DISTINCT
           user_account_state_type_id,
           tenant_id,
           user_id,
           preferences,
-          parent_id,
-          record_id,
           start_at
         FROM (
           SELECT
@@ -74,8 +70,6 @@ describe Masamune::Transform::DeduplicateDimension do
             tenant_id,
             user_id,
             preferences,
-            parent_id,
-            record_id,
             start_at,
             CASE
             WHEN (LAG(user_account_state_type_id) OVER w = user_account_state_type_id) AND (LAG(tenant_id) OVER w = tenant_id) AND (LAG(user_id) OVER w = user_id) AND ((LAG(preferences) OVER w = preferences) OR (LAG(preferences) OVER w IS NULL AND preferences IS NULL)) THEN

data/spec/masamune/transform/define_table.dimension_spec.rb

@@ -321,8 +321,6 @@ describe Masamune::Transform::DefineTable do
           tenant_id INTEGER NOT NULL,
           user_id INTEGER NOT NULL,
           preferences HSTORE,
-          parent_id INTEGER,
-          record_id INTEGER,
           start_at TIMESTAMP NOT NULL DEFAULT TO_TIMESTAMP(0),
           end_at TIMESTAMP,
           version INTEGER DEFAULT 1,
@@ -344,16 +342,6 @@ describe Masamune::Transform::DefineTable do
         ALTER TABLE user_dimension ADD CONSTRAINT user_dimension_7988187_fkey FOREIGN KEY (user_account_state_type_id) REFERENCES user_account_state_type(id);
         END IF; END $$;
 
-        DO $$ BEGIN
-        IF NOT EXISTS (SELECT 1 FROM pg_constraint c WHERE c.conname = 'user_dimension_e0538bc_fkey') THEN
-        ALTER TABLE user_dimension ADD CONSTRAINT user_dimension_e0538bc_fkey FOREIGN KEY (cluster_type_id, parent_id) REFERENCES user_dimension_ledger(cluster_type_id, id);
-        END IF; END $$;
-
-        DO $$ BEGIN
-        IF NOT EXISTS (SELECT 1 FROM pg_constraint c WHERE c.conname = 'user_dimension_824002d_fkey') THEN
-        ALTER TABLE user_dimension ADD CONSTRAINT user_dimension_824002d_fkey FOREIGN KEY (cluster_type_id, record_id) REFERENCES user_dimension_ledger(cluster_type_id, id);
-        END IF; END $$;
-
         DO $$ BEGIN
         IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = 'user_dimension_3fcebfa_key') THEN
         ALTER TABLE user_dimension ADD CONSTRAINT user_dimension_3fcebfa_key UNIQUE(cluster_type_id, tenant_id, user_id, start_at);
@@ -415,8 +403,6 @@ describe Masamune::Transform::DefineTable do
           tenant_id INTEGER,
           user_id INTEGER,
           preferences HSTORE,
-          parent_id INTEGER,
-          record_id INTEGER,
           start_at TIMESTAMP DEFAULT TO_TIMESTAMP(0),
           end_at TIMESTAMP,
           version INTEGER DEFAULT 1,

data/spec/masamune/transform/denormalize_table_spec.rb

@@ -73,6 +73,39 @@ describe Masamune::Transform::DenormalizeTable do
 
   subject(:result) { transform.denormalize_table(target, options).to_s }
 
+  context 'with postgres dimension' do
+    let(:target) { catalog.postgres.user_dimension }
+    let(:options) { { } }
+
+    it 'should eq render denormalize_table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+       SELECT
+         cluster_type.name AS cluster_type_name,
+         user_dimension.tenant_id,
+         user_dimension.user_id,
+         user_dimension.name,
+         user_dimension.start_at,
+         user_dimension.end_at,
+         user_dimension.version
+       FROM
+         user_dimension
+       LEFT JOIN
+         cluster_type
+       ON
+         cluster_type.id = user_dimension.cluster_type_id
+       ORDER BY
+         cluster_type_name,
+         tenant_id,
+         user_id,
+         name,
+         start_at,
+         end_at,
+         version
+       ;
+      EOS
+    end
+  end
+
   context 'with postgres fact without :columns' do
     let(:target) { catalog.postgres.visits_fact }
     let(:options) { { } }
@@ -137,6 +170,7 @@ describe Masamune::Transform::DenormalizeTable do
       EOS
     end
   end
+
   context 'with postgres fact with :columns' do
     let(:target) { catalog.postgres.visits_fact }
     let(:options) do