masamune 0.11.0

data/spec/masamune/transform/insert_reference_values.dimension_spec.rb

@@ -0,0 +1,111 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::InsertReferenceValues do
+  before do
+    catalog.schema :postgres do
+      dimension 'department', type: :mini do
+        column 'tenant_id', type: :integer, unique: true, natural_key: true
+        column 'department_id', type: :integer, unique: true, natural_key: true
+        row tenant_id: -1, department_id: -1, attributes: {default: true}
+      end
+
+      dimension 'user', type: :four do
+        references :department, insert: true
+        column 'tenant_id', index: true, natural_key: true
+        column 'user_id', index: true, natural_key: true
+        column 'name', type: :string
+      end
+
+      file 'user' do
+        column 'tenant_id', type: :integer
+        column 'user_id', type: :integer
+        column 'department.department_id', type: :integer
+        column 'start_at', type: :timestamp
+        column 'source_kind', type: :string
+        column 'delta', type: :integer
+      end
+
+      file 'misc' do
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.user_dimension.ledger_table }
+
+  context 'for postgres dimension with file containing references' do
+    let(:source) { catalog.postgres.user_file.stage_table(table: target) }
+    subject(:result) { transform.insert_reference_values(source, target).to_s }
+
+    it 'should render insert_reference_values template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TEMPORARY TABLE IF NOT EXISTS department_type_stage (LIKE department_type INCLUDING ALL);
+
+        INSERT INTO
+          department_type_stage (tenant_id, department_id)
+        SELECT DISTINCT
+          tenant_id,
+          department_type_department_id
+        FROM
+          user_dimension_ledger_stage
+        WHERE
+          tenant_id IS NOT NULL AND
+          department_type_department_id IS NOT NULL
+        ;
+
+        BEGIN;
+        LOCK TABLE department_type IN EXCLUSIVE MODE;
+
+        INSERT INTO
+          department_type (tenant_id, department_id)
+        SELECT
+          department_type_stage.tenant_id,
+          department_type_stage.department_id
+        FROM
+          department_type_stage
+        LEFT OUTER JOIN
+          department_type
+        ON
+          department_type.tenant_id = department_type_stage.tenant_id AND
+          department_type.department_id = department_type_stage.department_id
+        WHERE
+          department_type.tenant_id IS NULL AND
+          department_type.department_id IS NULL
+        ;
+
+        COMMIT;
+      EOS
+    end
+  end
+
+  context 'for postgres dimension with file missing references' do
+    let(:source) { catalog.postgres.misc_file.stage_table(table: target) }
+
+    subject(:result) { transform.insert_reference_values(source, target).to_s }
+
+    it 'should not render insert_reference_values template' do
+      is_expected.to eq("\n")
+    end
+  end
+end

data/spec/masamune/transform/insert_reference_values.fact_spec.rb

@@ -0,0 +1,149 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::InsertReferenceValues do
+  before do
+    catalog.schema :postgres do
+      dimension 'date', type: :one do
+        column 'date_id', type: :integer, unique: true, index: true, natural_key: true
+      end
+
+      dimension 'user_agent', type: :mini do
+        column 'name', type: :string, unique: true, index: 'shared'
+        column 'version', type: :string, unique: true, index: 'shared', default: 'Unknown'
+        column 'description', type: :string, null: true, ignore: true
+      end
+
+      dimension 'feature', type: :mini do
+        column 'name', type: :string, unique: true, index: true
+      end
+
+      dimension 'tenant', type: :two do
+        column 'tenant_id', type: :integer, index: true, natural_key: true
+      end
+
+      dimension 'user', type: :two do
+        column 'tenant_id', type: :integer, index: true, natural_key: true
+        column 'user_id', type: :integer, index: true, natural_key: true
+      end
+
+      fact 'visits', partition: 'y%Ym%m' do
+        references :date
+        references :tenant
+        references :user
+        references :user_agent, insert: true
+        references :feature, insert: true
+        measure 'total', type: :integer
+      end
+
+      file 'visits' do
+        column 'date.date_id', type: :integer
+        column 'tenant.tenant_id', type: :integer
+        column 'user.user_id', type: :integer
+        column 'user_agent.name', type: :string
+        column 'user_agent.version', type: :string
+        column 'feature.name', type: :string
+        column 'time_key', type: :integer
+        column 'total', type: :integer
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.visits_fact }
+  let(:source) { catalog.postgres.visits_file.stage_table(table: target) }
+
+  context 'with postgres fact' do
+    subject(:result) { transform.insert_reference_values(source, target).to_s }
+
+    it 'should eq render insert_reference_values template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        CREATE TEMPORARY TABLE IF NOT EXISTS user_agent_type_stage (LIKE user_agent_type INCLUDING ALL);
+
+        INSERT INTO
+          user_agent_type_stage (name, version)
+        SELECT DISTINCT
+          user_agent_type_name,
+          COALESCE(user_agent_type_version, 'Unknown')
+        FROM
+          visits_fact_stage
+        WHERE
+          user_agent_type_name IS NOT NULL
+        ;
+
+        BEGIN;
+        LOCK TABLE user_agent_type IN EXCLUSIVE MODE;
+
+        INSERT INTO
+          user_agent_type (name, version)
+        SELECT
+          user_agent_type_stage.name,
+          user_agent_type_stage.version
+        FROM
+          user_agent_type_stage
+        LEFT OUTER JOIN
+          user_agent_type
+        ON
+          user_agent_type.name = user_agent_type_stage.name AND
+          user_agent_type.version = user_agent_type_stage.version
+        WHERE
+          user_agent_type.name IS NULL AND
+          user_agent_type.version IS NULL
+        ;
+
+        COMMIT;
+
+        CREATE TEMPORARY TABLE IF NOT EXISTS feature_type_stage (LIKE feature_type INCLUDING ALL);
+
+        INSERT INTO
+          feature_type_stage (name)
+        SELECT DISTINCT
+          feature_type_name
+        FROM
+          visits_fact_stage
+        WHERE
+          feature_type_name IS NOT NULL
+        ;
+
+        BEGIN;
+        LOCK TABLE feature_type IN EXCLUSIVE MODE;
+
+        INSERT INTO
+          feature_type (name)
+        SELECT
+          feature_type_stage.name
+        FROM
+          feature_type_stage
+        LEFT OUTER JOIN
+          feature_type
+        ON
+          feature_type.name = feature_type_stage.name
+        WHERE
+          feature_type.name IS NULL
+        ;
+
+        COMMIT;
+      EOS
+    end
+  end
+end

data/spec/masamune/transform/load_dimension_spec.rb

@@ -0,0 +1,76 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::LoadDimension do
+  before do
+    catalog.schema :postgres do
+      dimension 'user_account_state', type: :mini do
+        column 'name', type: :string, unique: true
+        column 'description', type: :string
+      end
+
+      dimension 'department', type: :mini do
+        column 'tenant_id', type: :integer, unique: true, natural_key: true
+        column 'department_id', type: :integer, unique: true, natural_key: true
+        row tenant_id: -1, department_id: -1, attributes: {default: true}
+      end
+
+      dimension 'user', type: :four do
+        references :department, insert: true
+        references :user_account_state
+        column 'tenant_id', index: true, natural_key: true
+        column 'user_id', index: true, natural_key: true
+        column 'preferences', type: :key_value, null: true
+      end
+
+      file 'user' do
+        column 'tenant_id', type: :integer
+        column 'user_id', type: :integer
+        column 'user_account_state.name', type: :string
+        column 'preferences_now', type: :json
+        column 'start_at', type: :timestamp
+        column 'source_kind', type: :string
+        column 'delta', type: :integer
+      end
+    end
+  end
+
+  let(:data) { double(path: 'output.csv') }
+  let(:target) { catalog.postgres.user_dimension }
+  let(:source) { catalog.postgres.user_file }
+  let(:target_ledger) { target.ledger_table }
+  let(:source_table) { source.stage_table(suffix: 'file', table: target_ledger, inherit: false) }
+
+  context 'with postgres dimension' do
+    subject(:result) { transform.load_dimension(data, source, target).to_s }
+
+    it 'should render combined template' do
+      is_expected.to eq Masamune::Template.combine \
+        transform.define_table(source_table, data),
+        transform.insert_reference_values(source_table, target_ledger),
+        transform.stage_dimension(source_table, target_ledger),
+        transform.bulk_upsert(target_ledger.stage_table, target_ledger)
+    end
+  end
+end

data/spec/masamune/transform/load_fact_spec.rb

@@ -0,0 +1,89 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::LoadFact do
+  before do
+    catalog.schema :postgres do
+      dimension 'date', type: :date do
+        column 'date_id', type: :integer, unique: true, index: true, natural_key: true
+      end
+
+      dimension 'user_agent', type: :mini do
+        column 'name', type: :string, unique: true, index: 'shared'
+        column 'version', type: :string, unique: true, index: 'shared', default: 'Unknown'
+        column 'description', type: :string, null: true, ignore: true
+      end
+
+      dimension 'feature', type: :mini do
+        column 'name', type: :string, unique: true, index: true
+      end
+
+      dimension 'tenant', type: :two do
+        column 'tenant_id', type: :integer, index: true, natural_key: true
+      end
+
+      dimension 'user', type: :two do
+        column 'tenant_id', type: :integer, index: true, natural_key: true
+        column 'user_id', type: :integer, index: true, natural_key: true
+      end
+
+      fact 'visits', partition: 'y%Ym%m' do
+        references :date
+        references :tenant
+        references :user
+        references :user_agent, insert: true
+        references :feature, insert: true
+        measure 'total', type: :integer
+      end
+
+      file 'visits' do
+        column 'date.date_id', type: :integer
+        column 'tenant.tenant_id', type: :integer
+        column 'user.user_id', type: :integer
+        column 'user_agent.name', type: :string
+        column 'user_agent.version', type: :string
+        column 'feature.name', type: :string
+        column 'time_key', type: :integer
+        column 'total', type: :integer
+      end
+    end
+  end
+
+  let(:files) { (1..3).map { |i| double(path: "output_#{i}.csv") } }
+  let(:date) { DateTime.civil(2014,8) }
+  let(:target) { catalog.postgres.visits_fact }
+  let(:source) { catalog.postgres.visits_file }
+  let(:source_table) { source.stage_table(suffix: 'file', table: target, inherit: false) }
+
+  context 'for postgres fact' do
+    subject(:result) { transform.load_fact(files, source, target, date).to_s }
+
+    it 'should render combined template' do
+      is_expected.to eq Masamune::Template.combine \
+        transform.define_table(source_table, files),
+        transform.insert_reference_values(source_table, target),
+        transform.stage_fact(source_table, target, date)
+    end
+  end
+end

data/spec/masamune/transform/relabel_dimension_spec.rb

@@ -0,0 +1,102 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::RelabelDimension do
+  before do
+    catalog.schema :postgres do
+      dimension 'user_account_state', type: :mini do
+        column 'name', type: :string, unique: true
+        column 'description', type: :string
+      end
+
+      dimension 'user', type: :four do
+        references :user_account_state
+        column 'tenant_id', index: true, natural_key: true
+        column 'user_id', index: true, natural_key: true
+        column 'preferences', type: :key_value, null: true
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.user_dimension }
+
+  context 'for postgres dimension' do
+    subject(:result) { transform.relabel_dimension(target).to_s }
+
+    it 'should eq render relabel_dimension template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        BEGIN;
+        LOCK TABLE user_dimension IN EXCLUSIVE MODE;
+
+        UPDATE user_dimension SET version = NULL;
+
+        UPDATE
+          user_dimension
+        SET
+          version = tmp.version
+        FROM
+          (
+            SELECT
+              id,
+              tenant_id,
+              user_id,
+              start_at,
+              rank() OVER (PARTITION BY tenant_id, user_id ORDER BY start_at) AS version
+            FROM
+              user_dimension
+            GROUP BY
+              id, tenant_id, user_id, start_at
+           ) AS tmp
+        WHERE
+          user_dimension.id = tmp.id
+        ;
+
+        UPDATE user_dimension SET end_at = NULL;
+
+        UPDATE
+          user_dimension
+        SET
+          end_at = tmp.end_at
+        FROM
+          (
+            SELECT
+              id,
+              start_at,
+              tenant_id,
+              user_id,
+              LEAD(start_at, 1) OVER (PARTITION BY tenant_id, user_id ORDER BY start_at) AS end_at
+            FROM
+              user_dimension
+            GROUP BY
+              id, tenant_id, user_id, start_at
+           ) AS tmp
+        WHERE
+          user_dimension.id = tmp.id
+        ;
+
+        COMMIT;
+      EOS
+    end
+  end
+end