masamune 0.11.0

data/lib/masamune/transform/deduplicate_dimension.psql.erb

@@ -0,0 +1,52 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+INSERT INTO
+  <%= target.name %> (<%= target.insert_columns.join(', ') %>, parent_id, record_id, start_at)
+SELECT DISTINCT
+  <%- target.insert_view_values.each do |value| -%>
+  <%= value %><%= ',' %>
+  <%- end -%>
+  parent_id,
+  record_id,
+  start_at
+FROM (
+  SELECT
+    <%- target.insert_view_values.each do |value| -%>
+    <%= value %><%= ',' %>
+    <%- end -%>
+    parent_id,
+    record_id,
+    start_at,
+    CASE
+    WHEN <%= target.insert_view_values.map { |value| "(LAG(#{value}) OVER w = #{value})" }.join(' AND ') %> THEN
+      1
+    ELSE
+      0
+    END AS duplicate
+  FROM
+    <%= source.name %>
+  WINDOW w AS (PARTITION BY <%= target.window.join(', ') %> ORDER BY start_at)
+) tmp
+WHERE
+  duplicate = 0
+;

data/lib/masamune/transform/deduplicate_dimension.rb

@@ -0,0 +1,53 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module DeduplicateDimension
+    extend ActiveSupport::Concern
+
+    def deduplicate_dimension(source, target)
+      Operator.new(__method__, source: source, target: target, presenters: { postgres: Postgres })
+    end
+
+    private
+
+    class Postgres < SimpleDelegator
+      def insert_columns(source = nil)
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def insert_view_values
+        consolidated_columns.map { |_, column| column.name }
+      end
+
+      def window(*extra)
+        (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
+      end
+
+      private
+
+      def consolidated_columns
+        unreserved_columns.reject { |_, column| column.surrogate_key }
+      end
+    end
+  end
+end

data/lib/masamune/transform/define_event_view.hql.erb

@@ -0,0 +1,51 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+DROP VIEW IF EXISTS <%= target.view_name %>;
+CREATE VIEW IF NOT EXISTS <%= target.view_name %> (
+  uuid,
+  type,
+  <%- target.view_columns.each do |value| -%>
+  <%= value %>,
+  <%- end -%>
+  delta,
+  created_at,
+  y, m, d ,h
+) PARTITIONED ON (y, m, d, h) AS
+SELECT DISTINCT
+  uuid,
+  type,
+  <%- target.view_values.each do |value| -%>
+  <%= value %>,
+  <%- end -%>
+  IF(type = '<%= target.update_type %>', 1, 0) AS delta,
+  ctime_iso8601 AS created_at,
+  y, m, d ,h
+FROM
+  events
+LATERAL VIEW
+  json_tuple(events.json, <%= target.view_columns.map { |value| "'#{value}'" }.join(', ') %>) event_data AS <%= target.view_columns.join(', ') %>
+WHERE
+  type = '<%= target.create_type %>' OR
+  type = '<%= target.update_type %>' OR
+  type = '<%= target.delete_type %>'
+;

data/lib/masamune/transform/define_event_view.rb

@@ -0,0 +1,60 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module DefineEventView
+    extend ActiveSupport::Concern
+
+    def define_event_view(target)
+      Operator.new(__method__, target: target, presenters: { hive: Hive }).tap do |operator|
+        logger.debug("#{target.id}\n" + operator.to_s) if target.debug
+      end
+    end
+
+    private
+
+    class Hive < SimpleDelegator
+      def view_name
+        "#{id}_events"
+      end
+
+      def view_columns
+        unreserved_columns.map do |_, column|
+          column.name
+        end
+      end
+
+      def view_values
+        unreserved_columns.map do |_, column|
+          case column.type
+          when :json
+            # NOTE could just use split "\t" to parse tsv output
+            %Q{CONCAT('"', REGEXP_REPLACE(#{column.name}, '"', '""'),  '"') AS #{column.name}}
+          else
+            column.name
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/masamune/transform/define_index.psql.erb

@@ -0,0 +1,34 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<%- target.index_columns.each do |column_names, unique, id| -%>
+<%- index_name = "#{target.name}_#{id}_index" -%>
+<%- unless target.temporary? -%>
+DO $$ BEGIN
+IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = '<%= index_name %>') THEN
+<%- end -%>
+CREATE <%= unique ? 'UNIQUE INDEX' : 'INDEX' %> <%= index_name %> ON <%= target.name %> (<%= column_names.join(', ') %>);
+<%- unless target.temporary? -%>
+END IF; END $$;
+
+<%- end -%>
+<%- end -%>

data/lib/masamune/transform/define_schema.hql.erb

@@ -0,0 +1,23 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+-- Empty

data/lib/masamune/transform/define_schema.psql.erb

@@ -0,0 +1,79 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+CREATE EXTENSION IF NOT EXISTS hstore;
+
+SET TIMEZONE TO 'UTC';
+
+CREATE OR REPLACE FUNCTION json_to_hstore(JSON)
+RETURNS HSTORE AS $$
+  SELECT HSTORE(RTRIM(LTRIM(REPLACE($1::TEXT, ':', '=>'), '{'), '}'))
+$$ LANGUAGE SQL RETURNS NULL ON NULL INPUT;
+
+-- Adapted from https://gist.github.com/mfenniak/3008378
+CREATE OR REPLACE FUNCTION hstore_merge(left HSTORE, right HSTORE) RETURNS HSTORE AS $$
+  SELECT COALESCE($1, '') || COALESCE($2, '');
+$$ LANGUAGE SQL;
+
+DO $$ BEGIN
+IF NOT EXISTS (SELECT * FROM pg_proc WHERE proname = 'hstore_merge' AND proisagg) THEN
+CREATE AGGREGATE hstore_merge (HSTORE) (
+  SFUNC = hstore_merge,
+  STYPE = HSTORE,
+  INITCOND = ''
+);
+END IF; END $$;
+
+CREATE OR REPLACE FUNCTION coalesce_merge(left ANYELEMENT, right ANYELEMENT) RETURNS ANYELEMENT AS $$
+  SELECT COALESCE($2, $1)
+$$ LANGUAGE SQL;
+
+DO $$ BEGIN
+IF NOT EXISTS (SELECT * FROM pg_proc WHERE proname = 'coalesce_merge' AND proisagg) THEN
+CREATE AGGREGATE coalesce_merge (ANYELEMENT) (
+  SFUNC = coalesce_merge,
+  STYPE = ANYELEMENT
+);
+END IF; END $$;
+
+-- Adapted from http://stackoverflow.com/a/6945493/809290
+CREATE OR REPLACE FUNCTION sequence_owner(VARCHAR) RETURNS VARCHAR AS $$
+SELECT
+  CONCAT(t.relname::VARCHAR, '.', a.attname::VARCHAR)
+FROM
+  pg_class s
+JOIN
+  pg_depend d ON d.objid = s.oid AND d.classid = 'pg_class'::regclass AND d.refclassid = 'pg_class'::regclass
+JOIN
+  pg_class t ON t.oid = d.refobjid
+JOIN
+  pg_namespace n ON n.oid = t.relnamespace
+JOIN
+  pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid
+WHERE
+  s.relname = $1 AND
+  s.relkind='S' AND
+  d.deptype='a'
+LIMIT 1
+;
+$$ LANGUAGE SQL;

data/lib/masamune/transform/define_schema.rb

@@ -0,0 +1,56 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/define_table'
+require 'masamune/transform/define_event_view'
+
+module Masamune::Transform
+  module DefineSchema
+    include DefineTable
+    include DefineEventView
+
+    extend ActiveSupport::Concern
+
+    def define_schema(catalog, store_id)
+      context = catalog[store_id]
+      operators = []
+
+      operators += context.extra(:pre)
+
+      context.dimensions.each do |_, dimension|
+        operators << define_table(dimension)
+      end
+
+      context.facts.each do |_, fact|
+        operators << define_table(fact)
+      end
+
+      context.events.each do |_, event|
+        operators << define_event_view(event)
+      end
+
+      operators += context.extra(:post)
+
+      Operator.new __method__, *operators, source: context
+    end
+  end
+end

data/lib/masamune/transform/define_table.hql.erb

@@ -0,0 +1,34 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+CREATE TABLE IF NOT EXISTS <%= target.name %>
+(
+  <%- target.reference_columns.each do |column| -%>
+  <%= column.name %> <%= column.hql_type %>,
+  <%- end -%>
+  <%- target.measures.each do |_, measure| -%>
+  <%= measure.name %> <%= measure.hql_type %>,
+  <%- end -%>
+  <%= target.time_key.name %> <%= target.time_key.hql_type %>
+)
+PARTITIONED BY (<%= target.partition_by %>)
+TBLPROPERTIES ('serialization.null.format' = '');