masamune 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8bcd874b8f53a052225b41043cfda18de823cec
-  data.tar.gz: 9e4c71e47097a5035be079b92ce9d2698d74c7b4
+  metadata.gz: fed58d83f573640ea93aaee3226ba39e4b504302
+  data.tar.gz: a40fb42e8f9dd313c7606266424df457b61b5cef
 SHA512:
-  metadata.gz: 8e4b65c5424fdf38915854a32299cc7825418f96798c63719619a8cc4bfd7a69d81e286006974b5660a03cb3b020a677b7db838cf5efa0f03d79ff4cac209053
-  data.tar.gz: 8099dca87827675a761f9882025452d0c8bfd24773b09538006747e5b155d1ef9ec2f82a16bb16d5dc7b313aec748081d07541c332be5e633299c0dfeb88000c
+  metadata.gz: 1eef004f660f92fdb0664e65df4f402d0cf68059af223140fb3f33d5ad2a4c9a060543555d1cab548b46f4971e0fddc9ec4f1f2f48ee964ad596cc99dd6386c8
+  data.tar.gz: c07cd4e970b1208efcff02c052538590c9f8d27dbd53efaeedcd9845e7a19e37505954c2360d5835fb7588505e089378c7538549c4ada9cfe6fdcbc82a8bdd4d

data/lib/masamune/schema/dimension.rb

@@ -22,11 +22,31 @@
 
 module Masamune::Schema
   class Dimension < Table
+    SUPPORTED_GRAINS = [:hourly, :daily, :monthly]
+
     def initialize(opts = {})
-      super
+      opts.symbolize_keys!
+      self.grain = opts.delete(:grain)
+      super opts
       initialize_dimension_columns!
     end
 
+    def grain
+      return @grain if @grain
+      case type
+      when :date
+        :daily
+      else
+        :hourly
+      end
+    end
+
+    def grain=(grain = nil)
+      return unless grain
+      raise ArgumentError, "unknown grain '#{grain}'" unless SUPPORTED_GRAINS.include?(grain.to_sym)
+      @grain = grain.to_sym
+    end
+
     def suffix
       suffix = case type
       when :mini

data/lib/masamune/transform/postgres/deduplicate_dimension.psql.erb

@@ -22,12 +22,13 @@
 
 WITH consolidated AS (
   SELECT
-    <%- target.insert_view_values(coalesce: true).each do |value| -%>
+    <%- target.insert_view_last_values('w').each do |value| -%>
     <%= value %><%= ',' %>
     <%- end -%>
-    start_at
+    <%= target.start_at_with_grain %> AS start_at
   FROM
     <%= source.name %>
+  WINDOW w AS (PARTITION BY <%= target.window(target.start_at_with_grain).join(', ') %> ORDER BY start_at ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
 )
 INSERT INTO
   <%= target.name %> (<%= target.insert_columns.join(', ') %>, start_at)

data/lib/masamune/transform/postgres/deduplicate_dimension.rb

@@ -42,12 +42,18 @@ module Masamune::Transform::Postgres
         consolidated_columns.map { |_, column| column.name }
       end
 
-      def insert_view_values(coalesce: false)
+      def insert_view_values
         consolidated_columns.map do |_, column|
-          if !column.default.nil? && coalesce
-            "COALESCE(#{column.name}, #{column.sql_value(column.default)}) AS #{column.name}"
+          column.name
+        end
+      end
+
+      def insert_view_last_values(window)
+        consolidated_columns.map do |_, column|
+          if column.default.nil?
+            "LAST_VALUE(#{column.name}) OVER #{window} AS #{column.name}"
           else
-            column.name
+            "COALESCE(LAST_VALUE(#{column.name}) OVER #{window}, #{column.sql_value(column.default)}) AS #{column.name}"
           end
         end
       end
@@ -68,6 +74,17 @@ module Masamune::Transform::Postgres
         (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
       end
 
+      def start_at_with_grain
+        case grain
+        when :hourly
+          "date_trunc('hour', start_at)"
+        when :daily
+          "date_trunc('day', start_at)"
+        when :monthly
+          "date_trunc('month', start_at)"
+        end
+      end
+
       private
 
       def consolidated_columns

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.16.0'
+  VERSION = '0.17.0'
 end

data/spec/masamune/schema/dimension_spec.rb

@@ -33,6 +33,7 @@ describe Masamune::Schema::Dimension do
 
     it { expect(dimension.name).to eq('date_dimension') }
     it { expect(dimension.type).to eq(:date) }
+    it { expect(dimension.grain).to eq(:daily) }
   end
 
   context 'for type :one' do
@@ -46,6 +47,7 @@ describe Masamune::Schema::Dimension do
 
     it { expect(dimension.name).to eq('user_dimension') }
     it { expect(dimension.type).to eq(:one) }
+    it { expect(dimension.grain).to eq(:hourly) }
   end
 
   context 'for type :two' do
@@ -59,6 +61,7 @@ describe Masamune::Schema::Dimension do
 
     it { expect(dimension.name).to eq('user_dimension') }
     it { expect(dimension.type).to eq(:two) }
+    it { expect(dimension.grain).to eq(:hourly) }
   end
 
   context 'with invalid values' do
@@ -106,6 +109,7 @@ describe Masamune::Schema::Dimension do
 
     it { expect(dimension.name).to eq('user_dimension') }
     it { expect(dimension.type).to eq(:four) }
+    it { expect(dimension.grain).to eq(:hourly) }
 
     describe '#stage_table' do
       let!(:stage_table) { dimension.stage_table }
@@ -132,4 +136,20 @@ describe Masamune::Schema::Dimension do
       end
     end
   end
+
+  context 'dimension with daily grain' do
+    let(:dimension) { described_class.new id: 'users', type: :one, grain: :daily }
+
+    it { expect(dimension.name).to eq('users_dimension') }
+    it { expect(dimension.type).to eq(:one) }
+    it { expect(dimension.grain).to eq(:daily) }
+  end
+
+  context 'dimension with unknown grain' do
+    subject(:dimension) do
+      described_class.new id: 'users', grain: :quarterly
+    end
+
+    it { expect { dimension }.to raise_error ArgumentError, "unknown grain 'quarterly'" }
+  end
 end

data/spec/masamune/transform/deduplicate_dimension_spec.rb

@@ -46,13 +46,14 @@ describe Masamune::Transform::DeduplicateDimension do
       is_expected.to eq <<-EOS.strip_heredoc
         WITH consolidated AS (
           SELECT
-            user_account_state_type_id,
-            tenant_id,
-            user_id,
-            preferences,
-            start_at
+            LAST_VALUE(user_account_state_type_id) OVER w AS user_account_state_type_id,
+            LAST_VALUE(tenant_id) OVER w AS tenant_id,
+            LAST_VALUE(user_id) OVER w AS user_id,
+            LAST_VALUE(preferences) OVER w AS preferences,
+            date_trunc('hour', start_at) AS start_at
           FROM
             user_consolidated_dimension_stage
+          WINDOW w AS (PARTITION BY tenant_id, user_id, date_trunc('hour', start_at) ORDER BY start_at ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
         )
         INSERT INTO
           user_deduplicated_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, start_at)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-21 00:00:00.000000000 Z
+date: 2016-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor