masamune 0.11.9 → 0.12.0

data/spec/masamune/schema/table_spec.rb

@@ -36,6 +36,14 @@ describe Masamune::Schema::Table do
     it { expect(table.name).to eq('account_table') }
   end
 
+  context 'with format' do
+    let(:table) do
+      described_class.new id: 'user', properties: { format: :tsv }
+    end
+
+    it { expect(table.properties[:format]).to eq(:tsv) }
+  end
+
   context 'with columns' do
     let(:table) do
       described_class.new id: 'user',

data/spec/masamune/transform/bulk_upsert.dimension_spec.rb

@@ -63,7 +63,7 @@ describe Masamune::Transform::BulkUpsert do
         column 'department.department_id', type: :integer
         column 'user_account_state.name', type: :string
         column 'hr_user_account_state.name', type: :string
-        column 'preferences_now', type: :json
+        column 'preferences', type: :json
         column 'start_at', type: :timestamp
         column 'source_kind', type: :string
         column 'delta', type: :integer
@@ -147,8 +147,7 @@ describe Masamune::Transform::BulkUpsert do
           user_account_state_type_id = COALESCE(user_dimension_ledger_stage.user_account_state_type_id, user_dimension_ledger.user_account_state_type_id),
           hr_user_account_state_type_id = COALESCE(user_dimension_ledger_stage.hr_user_account_state_type_id, user_dimension_ledger.hr_user_account_state_type_id),
           name = COALESCE(user_dimension_ledger_stage.name, user_dimension_ledger.name),
-          preferences_now = COALESCE(user_dimension_ledger_stage.preferences_now, user_dimension_ledger.preferences_now),
-          preferences_was = COALESCE(user_dimension_ledger_stage.preferences_was, user_dimension_ledger.preferences_was)
+          preferences = COALESCE(user_dimension_ledger_stage.preferences, user_dimension_ledger.preferences)
         FROM
           user_dimension_ledger_stage
         WHERE
@@ -160,7 +159,7 @@ describe Masamune::Transform::BulkUpsert do
         ;
 
         INSERT INTO
-          user_dimension_ledger (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences_now, preferences_was, source_kind, source_uuid, start_at, last_modified_at, delta)
+          user_dimension_ledger (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences, source_kind, source_uuid, start_at, last_modified_at, delta)
         SELECT
           user_dimension_ledger_stage.department_type_id,
           user_dimension_ledger_stage.user_account_state_type_id,
@@ -168,8 +167,7 @@ describe Masamune::Transform::BulkUpsert do
           user_dimension_ledger_stage.tenant_id,
           user_dimension_ledger_stage.user_id,
           user_dimension_ledger_stage.name,
-          user_dimension_ledger_stage.preferences_now,
-          user_dimension_ledger_stage.preferences_was,
+          user_dimension_ledger_stage.preferences,
           user_dimension_ledger_stage.source_kind,
           user_dimension_ledger_stage.source_uuid,
           user_dimension_ledger_stage.start_at,

data/spec/masamune/transform/define_schema_spec.rb

@@ -63,11 +63,10 @@ describe Masamune::Transform::DefineSchema do
   context 'for hive schema' do
     before do
       catalog.schema :hive do
-        event 'tenant' do
-          attribute 'tenant_id', type: :integer, immutable: true
-          attribute 'account_state', type: :string
-          attribute 'premium_type', type: :string
-          attribute 'preferences', type: :json
+        dimension 'user', type: :ledger do
+          column 'tenant_id', index: true, natural_key: true
+          column 'user_id', index: true, natural_key: true
+          column 'preferences', type: :key_value, null: true
         end
       end
     end
@@ -77,7 +76,7 @@ describe Masamune::Transform::DefineSchema do
     it 'should render combined template' do
       is_expected.to eq Masamune::Template.combine \
         Masamune::Transform::Operator.new('define_schema', source: catalog.hive),
-        transform.define_event_view(catalog.hive.events['tenant'])
+        transform.define_table(catalog.hive.dimensions['user'])
     end
   end
 end

data/spec/masamune/transform/define_table.dimension_spec.rb

@@ -41,6 +41,106 @@ describe Masamune::Transform::DefineTable do
     end
   end
 
+  context 'for hive ledger dimension' do
+    before do
+      catalog.schema :hive do
+        dimension 'tenant', type: :ledger do
+          column 'tenant_id', type: :integer, natural_key: true
+          column 'tenant_account_state', type: :enum, values: %w(missing unknown active inactive)
+          column 'tenant_premium_state', type: :enum, values: %w(missing unkown goodwill pilot sandbox premium internal free vmware)
+          column 'preferences', type: :key_value, null: true
+        end
+      end
+    end
+
+    let(:table) { catalog.hive.tenant_dimension }
+
+    it 'should render table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TABLE IF NOT EXISTS tenant_ledger
+        (
+          id STRING,
+          tenant_id INT,
+          tenant_account_state STRING,
+          tenant_premium_state STRING,
+          preferences STRING,
+          source_kind STRING,
+          source_uuid STRING,
+          start_at STRING,
+          last_modified_at STRING,
+          delta INT
+        )
+        TBLPROPERTIES ('serialization.null.format' = '');
+      EOS
+    end
+  end
+
+  context 'for hive ledger dimension with partitions' do
+    before do
+      catalog.schema :hive do
+        dimension 'tenant', type: :ledger do
+          partition :y
+          partition :m
+          column 'tenant_id', type: :integer, natural_key: true
+          column 'tenant_account_state', type: :enum, values: %w(missing unknown active inactive)
+          column 'tenant_premium_state', type: :enum, values: %w(missing unkown goodwill pilot sandbox premium internal free vmware)
+          column 'preferences', type: :key_value, null: true
+        end
+      end
+    end
+
+    let(:table) { catalog.hive.tenant_dimension }
+
+    it 'should render table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TABLE IF NOT EXISTS tenant_ledger
+        (
+          id STRING,
+          tenant_id INT,
+          tenant_account_state STRING,
+          tenant_premium_state STRING,
+          preferences STRING,
+          source_kind STRING,
+          source_uuid STRING,
+          start_at STRING,
+          last_modified_at STRING,
+          delta INT
+        )
+        PARTITIONED BY (y INT, m INT)
+        TBLPROPERTIES ('serialization.null.format' = '');
+      EOS
+    end
+  end
+
+  context 'for hive ledger dimension with :tsv format' do
+    before do
+      catalog.schema :hive do
+        dimension 'tenant', type: :ledger, properties: { format: :tsv } do
+          column 'tenant_id', type: :integer, natural_key: true
+        end
+      end
+    end
+
+    let(:table) { catalog.hive.tenant_dimension }
+
+    it 'should render table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TABLE IF NOT EXISTS tenant_ledger
+        (
+          id STRING,
+          tenant_id INT,
+          source_kind STRING,
+          source_uuid STRING,
+          start_at STRING,
+          last_modified_at STRING,
+          delta INT
+        )
+        ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t'
+        TBLPROPERTIES ('serialization.null.format' = '');
+      EOS
+    end
+  end
+
   context 'for postgres dimension type: one' do
     before do
       catalog.schema :postgres do
@@ -160,8 +260,7 @@ describe Masamune::Transform::DefineTable do
           user_account_state_type_id INTEGER REFERENCES user_account_state_type(id),
           tenant_id INTEGER NOT NULL,
           user_id INTEGER NOT NULL,
-          preferences_now HSTORE,
-          preferences_was HSTORE,
+          preferences HSTORE,
           source_kind VARCHAR NOT NULL,
           source_uuid VARCHAR NOT NULL,
           start_at TIMESTAMP NOT NULL,

data/spec/masamune/transform/define_table.fact_spec.rb

@@ -60,6 +60,8 @@ describe Masamune::Transform::DefineTable do
       end
 
       fact 'visits', partition: 'y%Ym%m' do
+        partition :y
+        partition :m
         references :cluster
         references :date
         references :tenant

data/spec/masamune/transform/denormalize_table_spec.rb

@@ -63,22 +63,80 @@ describe Masamune::Transform::DenormalizeTable do
     end
   end
 
-  let(:target) { catalog.postgres.visits_fact }
-  let(:columns) do
-    [
-      'date.date_id',
-      'tenant.tenant_id',
-      'user.tenant_id',
-      'user.user_id',
-      'user_agent.name',
-      'user_agent.version',
-      'total',
-      'time_key'
-    ]
+  let(:options) { {} }
+
+  subject(:result) { transform.denormalize_table(target, options).to_s }
+
+  context 'with postgres fact with :columns' do
+    let(:target) { catalog.postgres.visits_fact }
+    let(:options) do
+      {
+        columns: [
+          'date.date_id',
+          'tenant.tenant_id',
+          'user.tenant_id',
+          'user.user_id',
+          'user_agent.name',
+          'user_agent.version',
+          'total',
+          'time_key'
+        ]
+      }
+    end
+
+    it 'should eq render denormalize_table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+      SELECT
+        date_dimension.date_id AS date_dimension_date_id,
+        tenant_dimension.tenant_id AS tenant_dimension_tenant_id,
+        user_dimension.tenant_id AS user_dimension_tenant_id,
+        user_dimension.user_id AS user_dimension_user_id,
+        user_agent_type.name AS user_agent_type_name,
+        user_agent_type.version AS user_agent_type_version,
+        visits_fact.total,
+        visits_fact.time_key
+      FROM
+        visits_fact
+      JOIN
+        date_dimension
+      ON
+        date_dimension.id = visits_fact.date_dimension_id
+      JOIN
+        tenant_dimension
+      ON
+        tenant_dimension.id = visits_fact.tenant_dimension_id
+      JOIN
+        user_dimension
+      ON
+        user_dimension.id = visits_fact.user_dimension_id
+      JOIN
+        user_agent_type
+      ON
+        user_agent_type.id = visits_fact.user_agent_type_id
+      ORDER BY
+        date_dimension_date_id,
+        tenant_dimension_tenant_id,
+        user_dimension_tenant_id,
+        user_dimension_user_id,
+        user_agent_type_name,
+        user_agent_type_version,
+        total,
+        time_key
+      ;
+      EOS
+    end
   end
 
-  context 'with postgres fact' do
-    subject(:result) { transform.denormalize_table(target, columns).to_s }
+  context 'with postgres fact with :except' do
+    let(:target) { catalog.postgres.visits_fact }
+    let(:options) do
+      {
+        except: [
+          'cluster.name',
+          'last_modified_at'
+        ]
+      }
+    end
 
     it 'should eq render denormalize_table template' do
       is_expected.to eq <<-EOS.strip_heredoc
@@ -89,6 +147,7 @@ describe Masamune::Transform::DenormalizeTable do
         user_dimension.user_id AS user_dimension_user_id,
         user_agent_type.name AS user_agent_type_name,
         user_agent_type.version AS user_agent_type_version,
+        user_agent_type.mobile AS user_agent_type_mobile,
         visits_fact.total,
         visits_fact.time_key
       FROM
@@ -116,10 +175,107 @@ describe Masamune::Transform::DenormalizeTable do
         user_dimension_user_id,
         user_agent_type_name,
         user_agent_type_version,
+        user_agent_type_mobile,
         total,
         time_key
       ;
       EOS
     end
   end
+
+  context 'with hive table' do
+    before do
+      catalog.schema :hive do
+        dimension 'tenant', type: :ledger do
+          partition :y
+          partition :m
+          column 'tenant_id', type: :integer, natural_key: true
+          column 'tenant_account_state', type: :enum, values: %w(missing unknown active inactive)
+          column 'tenant_premium_state', type: :enum, values: %w(missing unkown goodwill pilot sandbox premium internal free vmware)
+          column 'preferences', type: :key_value, null: true
+        end
+      end
+    end
+
+    let(:target) { catalog.hive.tenant_dimension }
+
+    let(:options) do
+      {
+        columns: [
+          'tenant_id',
+          'tenant_account_state',
+          'tenant_premium_state',
+          'preferences',
+          'y',
+          'm'
+        ],
+        order: [
+          'tenant_id',
+          'start_at'
+        ]
+      }
+    end
+
+    it 'should eq render denormalize_table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        SELECT
+          tenant_ledger.tenant_id,
+          tenant_ledger.tenant_account_state,
+          tenant_ledger.tenant_premium_state,
+          tenant_ledger.preferences,
+          tenant_ledger.y,
+          tenant_ledger.m
+        FROM
+          tenant_ledger
+        ORDER BY
+          tenant_id,
+          start_at
+        ;
+      EOS
+    end
+  end
+
+  context 'with hive table with implicit references' do
+    before do
+      catalog.schema :hive do
+        dimension 'date', type: :date, implicit: true do
+          column 'date_id', type: :integer, natural_key: true
+        end
+
+        fact 'visits' do
+          references :date
+          references :user, degenerate: true
+          measure 'total'
+        end
+      end
+    end
+
+    let(:target) { catalog.hive.visits_fact }
+
+    let(:options) do
+      {
+        columns: [
+          'date.date_id',
+          'user.id',
+          'total'
+        ]
+      }
+    end
+
+    it 'should eq render denormalize_table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        SELECT
+          date_dimension_date_id AS date_dimension_date_id,
+          user_type_id AS user_type_id,
+          visits_fact.total
+        FROM
+          visits_fact
+        ORDER BY
+          date_dimension_date_id,
+          user_type_id,
+          total
+        ;
+      EOS
+    end
+  end
 end

data/spec/masamune/transform/snapshot_dimension_spec.rb

@@ -72,7 +72,7 @@ describe Masamune::Transform::SnapshotDimension do
               coalesce_merge(user_account_state_type_id) OVER w AS user_account_state_type_id,
               tenant_id AS tenant_id,
               user_id AS user_id,
-              hstore_merge(preferences_now) OVER w - hstore_merge(preferences_was) OVER w AS preferences,
+              hstore_merge(preferences) OVER w AS preferences,
               start_at AS start_at
             FROM
               windows

data/spec/masamune/transform/stage_dimension_spec.rb

@@ -63,7 +63,7 @@ describe Masamune::Transform::StageDimension do
         column 'department.department_id', type: :integer
         column 'user_account_state.name', type: :string
         column 'hr_user_account_state.name', type: :string
-        column 'preferences_now', type: :json
+        column 'preferences', type: :json
         column 'start_at', type: :timestamp
         column 'source_kind', type: :string
         column 'delta', type: :integer
@@ -82,14 +82,14 @@ describe Masamune::Transform::StageDimension do
         CREATE TEMPORARY TABLE IF NOT EXISTS user_dimension_ledger_stage (LIKE user_dimension_ledger INCLUDING ALL);
 
         INSERT INTO
-          user_dimension_ledger_stage (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, preferences_now, source_kind, start_at, delta)
+          user_dimension_ledger_stage (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, preferences, source_kind, start_at, delta)
         SELECT
           department_type.id,
           user_account_state_type.id,
           hr_user_account_state_type.id,
           user_file_dimension_ledger_stage.tenant_id,
           user_file_dimension_ledger_stage.user_id,
-          json_to_hstore(user_file_dimension_ledger_stage.preferences_now),
+          json_to_hstore(user_file_dimension_ledger_stage.preferences),
           user_file_dimension_ledger_stage.source_kind,
           user_file_dimension_ledger_stage.start_at,
           user_file_dimension_ledger_stage.delta