masamune 0.17.10 → 0.17.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f638075056a6d6b8ef776e42e1cd54b0dd8751ca
-  data.tar.gz: f9bac35901816b810e83da6e7ecbdcbc70e4a110
+  metadata.gz: bedd64be2e2081da0dcd22d860eb4f502c83268f
+  data.tar.gz: 8ebcbaa4c7c8011b7db7179ab1671e381d35edb4
 SHA512:
-  metadata.gz: 293482ddeba0fe77d3e78f04ec9e50f4b26ca3245fec87c32338b0a74946b3e06e8949fd47c256bad14c0af8f4a2faca89d703581c71fbecb6447f1c6fa8d9ec
-  data.tar.gz: d48aa19c8c17612c4eb044d802bacfd1dd60511be11e5fce33cb6fefd76ea4ae98e05f3c727da8fb87bc4ae287b6a93f64d92a3325f2e1e1375b8100567810a2
+  metadata.gz: a952af9940ed6d29d08623df5809b1100b813e3ac602fe4f206c88fc2cc52a9beaedfe4516b40a1ee20af9d665f32716a3e44e5b956a9c442380bdf583ca9e15
+  data.tar.gz: cd3d3bebf7a7e09380244120fb25d20cb019c5544a21a52375e937810b052c1f57d5e34c882cb1e523b132d0f8bfae2183a315e24ed34b93cfd41b101c3aeedd

data/lib/masamune/schema/table_reference.rb

@@ -28,6 +28,7 @@ module Masamune::Schema
       insert:          false,
       null:            false,
       default:         nil,
+      unknown:         nil,
       natural_key:     false,
       denormalize:     false,
       multiple:        false,
@@ -74,6 +75,14 @@ module Masamune::Schema
       end
     end
 
+    def unknown(column = nil)
+      return unless @unknown
+      return if @unknown == :null
+      if unknown_row = @table.rows.detect { |row| row.id == @unknown }
+        unknown_row.name(column)
+      end
+    end
+
     def through=(columns)
       @through = Array.wrap(columns)
     end

data/lib/masamune/transform/postgres/stage_fact.psql.erb

@@ -40,11 +40,11 @@ SELECT
   <%- end -%>
 FROM
   <%= source.name %>
-<%- target.join_conditions(source).each do |table, conditions| -%>
-JOIN
+<%- target.join_conditions(source).each do |table, clause| -%>
+<%= clause.type %> JOIN
   <%= table %>
 ON
-  <%= conditions.join(" AND\n  ") %>
+  <%= clause.conditions.join(" AND\n  ") %>
 <%- end -%>
 ;
 

data/lib/masamune/transform/postgres/stage_fact.rb

@@ -56,7 +56,8 @@ module Masamune::Transform::Postgres
         shared_columns(source).values.map do |columns|
           column = columns.first
           if !column.degenerate? && reference = column.reference
-            reference.surrogate_key.qualified_name(column.reference.label)
+            value = reference.surrogate_key.qualified_name(column.reference.label)
+            column.reference.unknown ? "COALESCE(#{value}, #{column.reference.unknown})" : value
           else
             column.qualified_name
           end
@@ -74,7 +75,7 @@ module Masamune::Transform::Postgres
         join_columns = join_columns.group_by { |column| column.reference }
 
         dependencies = Masamune::TopologicalHash.new
-        conditions = Hash.new { |h,k| h[k] = [] }
+        conditions = Hash.new { |h,k| h[k] = OpenStruct.new(type: 'INNER', conditions: []) }
         join_columns.each do |reference, columns|
           reference_name = join_alias(reference)
           columns.each do |column|
@@ -91,7 +92,7 @@ module Masamune::Transform::Postgres
 
             column.reference.auto_surrogate_keys.each do |auto_surrogate_key|
               next unless auto_surrogate_key.default
-              conditions[reference_name] << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
+              conditions[reference_name].conditions << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
             end if column.reference
 
             if column.reference && !column.reference.default.nil? && column.adjacent.natural_key
@@ -100,7 +101,7 @@ module Masamune::Transform::Postgres
               coalesce_values << column.adjacent.sql_value(column.adjacent.default)
             end
 
-            conditions[reference_name] << (coalesce_values.any? ?
+            conditions[reference_name].conditions << (coalesce_values.any? ?
               "#{column.foreign_key_name} = COALESCE(#{column.qualified_name}, #{coalesce_values.join(', ')})" :
               "#{column.foreign_key_name} = #{column.qualified_name}")
           end
@@ -108,10 +109,11 @@ module Masamune::Transform::Postgres
           if reference.type == :two || reference.type == :four
             join_key_a = "TO_TIMESTAMP(#{source.time_key.qualified_name}) BETWEEN #{reference.start_key.qualified_name(reference.label)} AND COALESCE(#{reference.end_key.qualified_name(reference.label)}, 'INFINITY')"
             join_key_b = "TO_TIMESTAMP(#{source.time_key.qualified_name}) < #{reference.start_key.qualified_name(reference.label)} AND #{reference.version_key.qualified_name(reference.label)} = 1"
-            conditions[reference_name] << "((#{join_key_a}) OR (#{join_key_b}))"
+            conditions[reference_name].conditions << "((#{join_key_a}) OR (#{join_key_b}))"
           end
 
-          conditions[reference_name].uniq!
+          conditions[reference_name].type = 'LEFT' if reference.unknown
+          conditions[reference_name].conditions.uniq!
         end
         conditions.slice(*dependencies.tsort)
       end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.17.10'
+  VERSION = '0.17.11'
 end

data/spec/masamune/transform/stage_fact_spec.rb

@@ -67,8 +67,9 @@ describe Masamune::Transform::StageFact do
 
         column 'tenant_id', type: :integer, natural_key: true
         column 'group_id', type: :integer, natural_key: true
-        column 'group_mode', type: :enum, sub_type: 'group_mode', values: %(missing public private), index: true, natural_key: true, default: 'missing'
+        column 'group_mode', type: :enum, sub_type: 'group_mode', values: %(missing unknown public private), index: true, natural_key: true, default: 'missing'
         row group_id: -1, group_mode: 'missing', attributes: {id: :missing}
+        row group_id: -2, group_mode: 'unknown', attributes: {id: :unknown}
       end
 
       fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
@@ -76,8 +77,8 @@ describe Masamune::Transform::StageFact do
         references :date
         references :tenant, through: [:user, :from_group, :group]
         references :user
-        references :group, label: 'from', default: :missing
-        references :group, default: :missing
+        references :group, label: 'from', default: :missing, unknown: :unknown
+        references :group, default: :missing, unknown: :unknown
         references :user_agent, insert: true
         references :feature, insert: true
         references :session, degenerate: true
@@ -127,8 +128,8 @@ describe Masamune::Transform::StageFact do
           date_dimension.id,
           tenant_dimension.id,
           user_dimension.id,
-          from_group_dimension.id,
-          group_dimension.id,
+          COALESCE(from_group_dimension.id, unknown_group_dimension_id()),
+          COALESCE(group_dimension.id, unknown_group_dimension_id()),
           user_agent_type.id,
           feature_type.id,
           visits_hourly_file_fact_stage.session_type_id,
@@ -136,44 +137,44 @@ describe Masamune::Transform::StageFact do
           visits_hourly_file_fact_stage.time_key
         FROM
           visits_hourly_file_fact_stage
-        JOIN
+        INNER JOIN
           date_dimension
         ON
           date_dimension.date_id = visits_hourly_file_fact_stage.date_dimension_date_id
-        JOIN
+        INNER JOIN
           user_dimension
         ON
           user_dimension.cluster_type_id = default_cluster_type_id() AND
           user_dimension.user_id = visits_hourly_file_fact_stage.user_dimension_user_id AND
           ((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN user_dimension.start_at AND COALESCE(user_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < user_dimension.start_at AND user_dimension.version = 1))
-        JOIN
+        LEFT JOIN
           group_dimension AS from_group_dimension
         ON
           from_group_dimension.cluster_type_id = default_cluster_type_id() AND
           from_group_dimension.group_id = COALESCE(visits_hourly_file_fact_stage.from_group_dimension_group_id, missing_group_dimension_group_id()) AND
           from_group_dimension.group_mode = COALESCE(visits_hourly_file_fact_stage.from_group_dimension_group_mode, missing_group_dimension_group_mode()) AND
           ((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN from_group_dimension.start_at AND COALESCE(from_group_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < from_group_dimension.start_at AND from_group_dimension.version = 1))
-        JOIN
+        LEFT JOIN
           group_dimension
         ON
           group_dimension.cluster_type_id = default_cluster_type_id() AND
           group_dimension.group_id = COALESCE(visits_hourly_file_fact_stage.group_dimension_group_id, missing_group_dimension_group_id()) AND
           group_dimension.group_mode = COALESCE(visits_hourly_file_fact_stage.group_dimension_group_mode, missing_group_dimension_group_mode()) AND
           ((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN group_dimension.start_at AND COALESCE(group_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < group_dimension.start_at AND group_dimension.version = 1))
-        JOIN
+        INNER JOIN
           tenant_dimension
         ON
           tenant_dimension.cluster_type_id = default_cluster_type_id() AND
           tenant_dimension.tenant_id = COALESCE(visits_hourly_file_fact_stage.tenant_dimension_tenant_id, user_dimension.tenant_id, from_group_dimension.tenant_id, group_dimension.tenant_id) AND
           ((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN tenant_dimension.start_at AND COALESCE(tenant_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < tenant_dimension.start_at AND tenant_dimension.version = 1))
-        JOIN
+        INNER JOIN
           user_agent_type
         ON
           user_agent_type.cluster_type_id = default_cluster_type_id() AND
           user_agent_type.name = visits_hourly_file_fact_stage.user_agent_type_name AND
           user_agent_type.version = COALESCE(visits_hourly_file_fact_stage.user_agent_type_version, 'Unknown') AND
           user_agent_type.mobile = COALESCE(visits_hourly_file_fact_stage.user_agent_type_mobile, FALSE)
-        JOIN
+        INNER JOIN
           feature_type
         ON
           feature_type.name = visits_hourly_file_fact_stage.feature_type_name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.17.10
+  version: 0.17.11
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-29 00:00:00.000000000 Z
+date: 2016-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor