masamune 0.11.0

data/spec/masamune/transform/bulk_upsert.dimension_spec.rb

@@ -0,0 +1,200 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::BulkUpsert do
+  before do
+    catalog.schema :postgres do
+      dimension 'cluster', type: :mini do
+        column 'id', type: :integer, surrogate_key: true, auto: true
+        column 'name', type: :string, unique: true
+        row name: 'default', attributes: {default: true}
+      end
+
+      dimension 'user_account_state', type: :mini do
+        column 'name', type: :string, unique: true
+        column 'description', type: :string
+        row name: 'registered', description: 'Registered'
+        row name: 'active', description: 'Active', attributes: {default: true}
+        row name: 'inactive', description: 'Inactive'
+      end
+
+      dimension 'department', type: :mini do
+        references :cluster
+        column 'tenant_id', type: :integer, unique: true, natural_key: true
+        column 'department_id', type: :integer, unique: true, natural_key: true
+        row tenant_id: -1, department_id: -1, attributes: {default: true}
+      end
+
+      dimension 'user', type: :four do
+        references :cluster
+        references :department, insert: true
+        references :user_account_state
+        references :user_account_state, label: :hr
+        column 'tenant_id', index: true, natural_key: true
+        column 'user_id', index: true, natural_key: true
+        column 'name', type: :string
+        column 'preferences', type: :key_value, null: true
+      end
+
+      file 'user' do
+        column 'tenant_id', type: :integer
+        column 'user_id', type: :integer
+        column 'department.department_id', type: :integer
+        column 'user_account_state.name', type: :string
+        column 'hr_user_account_state.name', type: :string
+        column 'preferences_now', type: :json
+        column 'start_at', type: :timestamp
+        column 'source_kind', type: :string
+        column 'delta', type: :integer
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.user_dimension }
+
+  context 'for postgres dimension' do
+    subject(:result) { transform.bulk_upsert(target.stage_table, target).to_s }
+
+    it 'should render bulk_upsert template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        BEGIN;
+        LOCK TABLE user_dimension IN EXCLUSIVE MODE;
+
+        UPDATE
+          user_dimension
+        SET
+          department_type_id = COALESCE(user_dimension_stage.department_type_id, user_dimension.department_type_id),
+          user_account_state_type_id = COALESCE(user_dimension_stage.user_account_state_type_id, user_dimension.user_account_state_type_id),
+          hr_user_account_state_type_id = COALESCE(user_dimension_stage.hr_user_account_state_type_id, user_dimension.hr_user_account_state_type_id),
+          name = COALESCE(user_dimension_stage.name, user_dimension.name),
+          preferences = COALESCE(user_dimension_stage.preferences, user_dimension.preferences)
+        FROM
+          user_dimension_stage
+        WHERE
+          user_dimension.tenant_id = user_dimension_stage.tenant_id AND
+          user_dimension.user_id = user_dimension_stage.user_id AND
+          user_dimension.start_at = user_dimension_stage.start_at
+        ;
+
+        INSERT INTO
+          user_dimension (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences, parent_id, record_id, start_at, end_at, version, last_modified_at)
+        SELECT
+          user_dimension_stage.department_type_id,
+          user_dimension_stage.user_account_state_type_id,
+          user_dimension_stage.hr_user_account_state_type_id,
+          user_dimension_stage.tenant_id,
+          user_dimension_stage.user_id,
+          user_dimension_stage.name,
+          user_dimension_stage.preferences,
+          user_dimension_stage.parent_id,
+          user_dimension_stage.record_id,
+          user_dimension_stage.start_at,
+          user_dimension_stage.end_at,
+          user_dimension_stage.version,
+          user_dimension_stage.last_modified_at
+        FROM
+          user_dimension_stage
+        LEFT OUTER JOIN
+          user_dimension
+        ON
+          user_dimension.tenant_id = user_dimension_stage.tenant_id AND
+          user_dimension.user_id = user_dimension_stage.user_id AND
+          user_dimension.start_at = user_dimension_stage.start_at
+        WHERE
+          user_dimension.tenant_id IS NULL AND
+          user_dimension.user_id IS NULL AND
+          user_dimension.start_at IS NULL
+        ;
+
+        COMMIT;
+      EOS
+    end
+  end
+
+  context 'for postgres dimension ledger' do
+    subject(:result) { transform.bulk_upsert(target.ledger_table.stage_table, target.ledger_table).to_s }
+
+    it 'should render bulk_upsert template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        BEGIN;
+        LOCK TABLE user_dimension_ledger IN EXCLUSIVE MODE;
+
+        UPDATE
+          user_dimension_ledger
+        SET
+          department_type_id = COALESCE(user_dimension_ledger_stage.department_type_id, user_dimension_ledger.department_type_id),
+          user_account_state_type_id = COALESCE(user_dimension_ledger_stage.user_account_state_type_id, user_dimension_ledger.user_account_state_type_id),
+          hr_user_account_state_type_id = COALESCE(user_dimension_ledger_stage.hr_user_account_state_type_id, user_dimension_ledger.hr_user_account_state_type_id),
+          name = COALESCE(user_dimension_ledger_stage.name, user_dimension_ledger.name),
+          preferences_now = COALESCE(user_dimension_ledger_stage.preferences_now, user_dimension_ledger.preferences_now),
+          preferences_was = COALESCE(user_dimension_ledger_stage.preferences_was, user_dimension_ledger.preferences_was)
+        FROM
+          user_dimension_ledger_stage
+        WHERE
+          user_dimension_ledger.tenant_id = user_dimension_ledger_stage.tenant_id AND
+          user_dimension_ledger.user_id = user_dimension_ledger_stage.user_id AND
+          user_dimension_ledger.source_kind = user_dimension_ledger_stage.source_kind AND
+          user_dimension_ledger.source_uuid = user_dimension_ledger_stage.source_uuid AND
+          user_dimension_ledger.start_at = user_dimension_ledger_stage.start_at
+        ;
+
+        INSERT INTO
+          user_dimension_ledger (department_type_id, user_account_state_type_id, hr_user_account_state_type_id, tenant_id, user_id, name, preferences_now, preferences_was, source_kind, source_uuid, start_at, last_modified_at, delta)
+        SELECT
+          user_dimension_ledger_stage.department_type_id,
+          user_dimension_ledger_stage.user_account_state_type_id,
+          user_dimension_ledger_stage.hr_user_account_state_type_id,
+          user_dimension_ledger_stage.tenant_id,
+          user_dimension_ledger_stage.user_id,
+          user_dimension_ledger_stage.name,
+          user_dimension_ledger_stage.preferences_now,
+          user_dimension_ledger_stage.preferences_was,
+          user_dimension_ledger_stage.source_kind,
+          user_dimension_ledger_stage.source_uuid,
+          user_dimension_ledger_stage.start_at,
+          user_dimension_ledger_stage.last_modified_at,
+          user_dimension_ledger_stage.delta
+        FROM
+          user_dimension_ledger_stage
+        LEFT OUTER JOIN
+          user_dimension_ledger
+        ON
+          user_dimension_ledger.tenant_id = user_dimension_ledger_stage.tenant_id AND
+          user_dimension_ledger.user_id = user_dimension_ledger_stage.user_id AND
+          user_dimension_ledger.source_kind = user_dimension_ledger_stage.source_kind AND
+          user_dimension_ledger.source_uuid = user_dimension_ledger_stage.source_uuid AND
+          user_dimension_ledger.start_at = user_dimension_ledger_stage.start_at
+        WHERE
+          user_dimension_ledger.tenant_id IS NULL AND
+          user_dimension_ledger.user_id IS NULL AND
+          user_dimension_ledger.source_kind IS NULL AND
+          user_dimension_ledger.source_uuid IS NULL AND
+          user_dimension_ledger.start_at IS NULL
+        ;
+
+        COMMIT;
+      EOS
+    end
+  end
+end

data/spec/masamune/transform/consolidate_dimension_spec.rb

@@ -0,0 +1,62 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::ConsolidateDimension do
+  before do
+    catalog.schema :postgres do
+      dimension 'user_account_state', type: :mini do
+        column 'name', type: :string, unique: true
+        column 'description', type: :string
+      end
+
+      dimension 'user', type: :four do
+        references :user_account_state
+        column 'tenant_id', index: true, natural_key: true
+        column 'user_id', index: true, natural_key: true
+        column 'preferences', type: :key_value, null: true
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.user_dimension }
+
+  context 'with postgres dimension' do
+    subject(:result) { transform.consolidate_dimension(target).to_s }
+
+    it 'should render combined template' do
+      is_expected.to eq Masamune::Template.combine \
+        transform.define_table(target.stage_table(suffix: 'consolidated_forward')),
+        transform.define_table(target.stage_table(suffix: 'consolidated_reverse')),
+        transform.define_table(target.stage_table(suffix: 'consolidated')),
+        transform.define_table(target.stage_table(suffix: 'deduplicated')),
+        transform.snapshot_dimension(target.ledger_table, target.stage_table(suffix: 'consolidated_forward'), 'ASC'),
+        transform.snapshot_dimension(target.ledger_table, target.stage_table(suffix: 'consolidated_reverse'), 'DESC'),
+        transform.bulk_upsert(target.stage_table(suffix: 'consolidated_forward'), target.stage_table(suffix: 'consolidated')),
+        transform.bulk_upsert(target.stage_table(suffix: 'consolidated_reverse'), target.stage_table(suffix: 'consolidated')),
+        transform.deduplicate_dimension(target.stage_table(suffix: 'consolidated'), target.stage_table(suffix: 'deduplicated')),
+        transform.bulk_upsert(target.stage_table(suffix: 'deduplicated'), target),
+        transform.relabel_dimension(target)
+    end
+  end
+end

data/spec/masamune/transform/deduplicate_dimension_spec.rb

@@ -0,0 +1,84 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::DeduplicateDimension do
+  before do
+    catalog.schema :postgres do
+      dimension 'user_account_state', type: :mini do
+        column 'name', type: :string, unique: true
+        column 'description', type: :string
+      end
+
+      dimension 'user', type: :four do
+        references :user_account_state
+        column 'tenant_id', index: true, natural_key: true
+        column 'user_id', index: true, natural_key: true
+        column 'preferences', type: :key_value, null: true
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.user_dimension }
+
+  context 'with postgres dimension' do
+    subject(:result) { transform.deduplicate_dimension(target.stage_table(suffix: 'consolidated'), target.stage_table(suffix: 'deduplicated')).to_s }
+
+    it 'should render deduplicate_dimension template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        INSERT INTO
+          user_deduplicated_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, parent_id, record_id, start_at)
+        SELECT DISTINCT
+          user_account_state_type_id,
+          tenant_id,
+          user_id,
+          preferences,
+          parent_id,
+          record_id,
+          start_at
+        FROM (
+          SELECT
+            user_account_state_type_id,
+            tenant_id,
+            user_id,
+            preferences,
+            parent_id,
+            record_id,
+            start_at,
+            CASE
+            WHEN (LAG(user_account_state_type_id) OVER w = user_account_state_type_id) AND (LAG(tenant_id) OVER w = tenant_id) AND (LAG(user_id) OVER w = user_id) AND (LAG(preferences) OVER w = preferences) THEN
+              1
+            ELSE
+              0
+            END AS duplicate
+          FROM
+            user_consolidated_dimension_stage
+          WINDOW w AS (PARTITION BY tenant_id, user_id ORDER BY start_at)
+        ) tmp
+        WHERE
+          duplicate = 0
+        ;
+      EOS
+    end
+  end
+end

data/spec/masamune/transform/define_event_view_spec.rb

@@ -0,0 +1,84 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::DefineEventView do
+  before do
+    catalog.schema :hive do
+      event 'tenant' do
+        attribute 'tenant_id', type: :integer, immutable: true
+        attribute 'account_state', type: :string
+        attribute 'premium_type', type: :string
+        attribute 'preferences', type: :json
+      end
+    end
+  end
+
+  let(:target) { catalog.hive.tenant_event }
+
+  context 'with hive event' do
+    subject(:result) { transform.define_event_view(target).to_s }
+
+    it 'should render define_event_view template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+        DROP VIEW IF EXISTS tenant_events;
+        CREATE VIEW IF NOT EXISTS tenant_events (
+          uuid,
+          type,
+          tenant_id,
+          account_state_now,
+          account_state_was,
+          premium_type_now,
+          premium_type_was,
+          preferences_now,
+          preferences_was,
+          delta,
+          created_at,
+          y, m, d ,h
+        ) PARTITIONED ON (y, m, d, h) AS
+        SELECT DISTINCT
+          uuid,
+          type,
+          tenant_id,
+          account_state_now,
+          account_state_was,
+          premium_type_now,
+          premium_type_was,
+          CONCAT('"', REGEXP_REPLACE(preferences_now, '"', '""'),  '"') AS preferences_now,
+          CONCAT('"', REGEXP_REPLACE(preferences_was, '"', '""'),  '"') AS preferences_was,
+          IF(type = 'tenant_update', 1, 0) AS delta,
+          ctime_iso8601 AS created_at,
+          y, m, d ,h
+        FROM
+          events
+        LATERAL VIEW
+          json_tuple(events.json, 'tenant_id', 'account_state_now', 'account_state_was', 'premium_type_now', 'premium_type_was', 'preferences_now', 'preferences_was') event_data AS tenant_id, account_state_now, account_state_was, premium_type_now, premium_type_was, preferences_now, preferences_was
+        WHERE
+          type = 'tenant_create' OR
+          type = 'tenant_update' OR
+          type = 'tenant_delete'
+        ;
+      EOS
+    end
+  end
+end

data/spec/masamune/transform/define_schema_spec.rb

@@ -0,0 +1,83 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::DefineSchema do
+  context 'for postgres schema' do
+    before do
+      catalog.schema :postgres do
+        dimension 'user_account_state', type: :mini do
+          column 'name', type: :string, unique: true
+          column 'description', type: :string
+        end
+
+        dimension 'user', type: :four do
+          references :user_account_state
+          column 'tenant_id', index: true, natural_key: true
+          column 'user_id', index: true, natural_key: true
+          column 'preferences', type: :key_value, null: true
+        end
+
+        file 'user' do
+          column 'tenant_id', type: :integer
+          column 'user_id', type: :integer
+          column 'user_account_state.name', type: :string
+          column 'preferences_now', type: :json
+          column 'start_at', type: :timestamp
+          column 'source_kind', type: :string
+          column 'delta', type: :integer
+        end
+      end
+    end
+
+    subject(:result) { transform.define_schema(catalog, :postgres).to_s }
+
+    it 'should render combined template' do
+      is_expected.to eq Masamune::Template.combine \
+        Masamune::Transform::Operator.new('define_schema', source: catalog.postgres),
+        transform.define_table(catalog.postgres.dimensions['user_account_state']),
+        transform.define_table(catalog.postgres.dimensions['user'])
+    end
+  end
+
+  context 'for hive schema' do
+    before do
+      catalog.schema :hive do
+        event 'tenant' do
+          attribute 'tenant_id', type: :integer, immutable: true
+          attribute 'account_state', type: :string
+          attribute 'premium_type', type: :string
+          attribute 'preferences', type: :json
+        end
+      end
+    end
+
+    subject(:result) { transform.define_schema(catalog, :hive).to_s }
+
+    it 'should render combined template' do
+      is_expected.to eq Masamune::Template.combine \
+        Masamune::Transform::Operator.new('define_schema', source: catalog.hive),
+        transform.define_event_view(catalog.hive.events['tenant'])
+    end
+  end
+end