masamune 0.11.0

data/lib/masamune/transform/define_table.psql.erb

@@ -0,0 +1,95 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<% files ||= [] %>
+
+<%- target.children.each do |child| -%>
+<%= render 'define_table.psql.erb', target: child %>
+<%- end -%>
+
+<%- target.enum_columns.each do |_, column| -%>
+DO $$ BEGIN
+IF NOT EXISTS (SELECT 1 FROM pg_type t WHERE LOWER(t.typname) = LOWER('<%= column.sql_type %>')) THEN
+CREATE TYPE <%= column.sql_type %> AS ENUM (<%= column.values.map { |value| "'#{value}'" }.join(', ') %>);
+END IF; END $$;
+<%- end -%>
+
+<%- target.sequence_columns.each do |_, column| -%>
+DO $$ BEGIN
+IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = '<%= column.sequence_id %>') THEN
+CREATE SEQUENCE <%= column.sequence_id %>;
+ALTER SEQUENCE <%= column.sequence_id %> RESTART <%= column.sequence_offset %>;
+END IF; END $$;
+<%- end -%>
+
+<%- if target.temporary? -%>
+CREATE TEMPORARY TABLE IF NOT EXISTS <%= target.name %>
+<%- else -%>
+CREATE TABLE IF NOT EXISTS <%= target.name %>
+<%- end -%>
+(
+  <%- target.defined_columns.each do |column, last| -%>
+  <%= column.as_psql %><%= ',' unless last %>
+  <%- end -%>
+);
+
+<%- target.sequence_columns.each do |_, column| -%>
+DO $$ BEGIN
+IF NOT EXISTS (SELECT 1 WHERE sequence_owner('<%= column.sequence_id %>') = '<%= column.qualified_name %>') THEN
+ALTER SEQUENCE <%= column.sequence_id %> OWNED BY <%= column.qualified_name %>;
+END IF; END $$;
+<%- end -%>
+
+<%- files.each do |file| -%>
+<%-
+  copy_options = []
+  copy_options << "FORMAT 'csv'" if target.store.format == :csv
+  copy_options << "DELIMITER '\t'" if target.store.format == :tsv
+  copy_options << "HEADER true" if target.store.headers
+-%>
+COPY <%= target.name %> FROM '<%= file %>' WITH (<%= copy_options.join(", ") %>);
+<%- end -%>
+
+<%= render 'define_unique.psql.erb', target: target %>
+<%= render 'define_index.psql.erb', target: target %>
+
+<% target.insert_rows.each do |row| %>
+INSERT INTO <%= target.name %> (<%= row.insert_columns.join(', ') %>)
+SELECT <%= row.insert_values.join(', ') %>
+WHERE NOT EXISTS (SELECT 1 FROM <%= target.name %> WHERE <%= row.insert_constraints.join(' AND ') %>);
+<%- end -%>
+
+<% target.aliased_rows.each do |row| %>
+<%- row.natural_keys.each do |column| -%>
+CREATE OR REPLACE FUNCTION <%= row.name(column) %>
+RETURNS <%= column.sql_type %> IMMUTABLE AS $$
+  SELECT <%= row.sql_value(column) %>;
+$$ LANGUAGE SQL;
+
+<%- end -%>
+
+CREATE OR REPLACE FUNCTION <%= row.name %>
+RETURNS <%= target.surrogate_key.sql_type %> IMMUTABLE AS $$
+  SELECT <%= target.surrogate_key.name %> FROM <%= target.name %> WHERE <%= row.insert_constraints.join(' AND ') %>;
+$$ LANGUAGE SQL;
+
+<%- end -%>

data/lib/masamune/transform/define_table.rb

@@ -0,0 +1,40 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module DefineTable
+    extend ActiveSupport::Concern
+
+    def define_table(target, files = [])
+      return if target.implicit
+      Operator.new(__method__, target: target, files: Masamune::Schema::Map.convert_files(files), presenters: { hive: Hive }).tap do |operator|
+        logger.debug("#{target.id}\n" + operator.to_s) if target.debug
+      end
+    end
+
+    class Hive < SimpleDelegator
+      def partition_by
+        partitions.map { |_, column| "#{column.name} #{column.hql_type}" }.join(', ')
+      end
+    end
+  end
+end

data/lib/masamune/transform/define_unique.psql.erb

@@ -0,0 +1,30 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<%- target.unique_constraints.each do |column_names, id| -%>
+<%- constraint_name = "#{target.name}_#{id}_key" -%>
+DO $$ BEGIN
+IF NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.relname = '<%= constraint_name %>') THEN
+ALTER TABLE <%= target.name %> ADD CONSTRAINT <%= constraint_name %> UNIQUE(<%= column_names.join(', ') %>);
+END IF; END $$;
+
+<%- end -%>

data/lib/masamune/transform/insert_reference_values.psql.erb

@@ -0,0 +1,43 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+<%- if target.insert_columns(source).any? -%>
+CREATE TEMPORARY TABLE IF NOT EXISTS <%= target.stage_table.name %> (LIKE <%= target.name %> INCLUDING ALL);
+
+INSERT INTO
+  <%= target.stage_table.name %> (<%= target.insert_columns(source).join(', ') %>)
+SELECT DISTINCT
+  <%- target.insert_values(source).each do |value, last| -%>
+  <%= value %><%= ',' unless last %>
+  <%- end -%>
+FROM
+  <%= source.name %>
+<%- if target.insert_constraints(source).any? -%>
+WHERE
+  <%- target.insert_constraints(source).each do |constraint, last| -%>
+  <%= constraint %><%= ' AND' unless last %>
+  <%- end -%>
+<%- end -%>
+;
+
+<%= target.bulk_upsert(target.stage_table, target) %>
+<%- end -%>

data/lib/masamune/transform/insert_reference_values.rb

@@ -0,0 +1,64 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/bulk_upsert'
+
+module Masamune::Transform
+  module InsertReferenceValues
+    extend ActiveSupport::Concern
+
+    def insert_reference_values(source, target)
+      operators = []
+      target.insert_references.each do |_, reference|
+        operators << Operator.new(__method__, source: source, target: reference, presenters: { postgres: Postgres })
+      end
+      Operator.new *operators
+    end
+
+    private
+
+    class Postgres < SimpleDelegator
+      include BulkUpsert
+      include Masamune::LastElement
+
+      def insert_columns(source)
+        source.shared_columns(stage_table).map { |_, columns| columns.first.name }
+      end
+
+      def insert_values(source)
+        source.shared_columns(stage_table).map do |column, _|
+          if column.adjacent.try(:default)
+            "COALESCE(#{column.name}, #{column.adjacent.sql_value(column.adjacent.default)})"
+          else
+            column.name
+          end
+        end
+      end
+      method_with_last_element :insert_values
+
+      def insert_constraints(source)
+        source.shared_columns(stage_table).reject { |column, _| column.null || column.default || column.adjacent.try(:default) }.map { |column, _| "#{column.name} IS NOT NULL"}
+      end
+      method_with_last_element :insert_constraints
+    end
+  end
+end

data/lib/masamune/transform/load_dimension.rb

@@ -0,0 +1,47 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/define_table'
+require 'masamune/transform/stage_dimension'
+require 'masamune/transform/insert_reference_values'
+require 'masamune/transform/bulk_upsert'
+
+module Masamune::Transform
+  module LoadDimension
+    include DefineTable
+    include StageDimension
+    include InsertReferenceValues
+    include BulkUpsert
+
+    extend ActiveSupport::Concern
+
+    def load_dimension(files, source, target)
+      target = target.type == :four ? target.ledger_table : target
+      source = source.stage_table(suffix: 'file', table: target, inherit: false)
+      Operator.new \
+        define_table(source, files),
+        insert_reference_values(source, target),
+        stage_dimension(source, target),
+        bulk_upsert(target.stage_table, target)
+    end
+  end
+end

data/lib/masamune/transform/load_fact.rb

@@ -0,0 +1,45 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/transform/define_table'
+require 'masamune/transform/stage_fact'
+require 'masamune/transform/insert_reference_values'
+require 'masamune/transform/bulk_upsert'
+
+module Masamune::Transform
+  module LoadFact
+    include DefineTable
+    include StageFact
+    include InsertReferenceValues
+    include BulkUpsert
+
+    extend ActiveSupport::Concern
+
+    def load_fact(files, source, target, date)
+      source = source.stage_table(suffix: 'file', table: target, inherit: false)
+      Operator.new \
+        define_table(source, files),
+        insert_reference_values(source, target),
+        stage_fact(source, target, date)
+    end
+  end
+end

data/lib/masamune/transform/operator.rb

@@ -0,0 +1,96 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  class Operator
+    def initialize(*args)
+      options     = args.last.is_a?(Hash) ? args.pop : {}
+      @templates  = args
+      @source     = options.delete(:source)
+      @target     = options.delete(:target)
+      @presenters = options.delete(:presenters) || {}
+      @locals     = options
+    end
+
+    def source
+      return unless @source
+      @presenters.key?(source_store.try(:type)) ? @presenters[source_store.try(:type)].new(@source) : @source
+    end
+
+    def target
+      return unless @target
+      @presenters.key?(target_store.try(:type)) ? @presenters[target_store.try(:type)].new(@target) : @target
+    end
+
+    def to_s
+      result = []
+      @templates.each do |template|
+        case template
+        when Operator
+          result << template
+        when Symbol, String
+          result << template_eval(template)
+        end
+      end
+      Masamune::Template.combine(*result)
+    end
+
+    def to_file
+      Tempfile.new('masamune').tap do |file|
+        file.write(to_s)
+        file.close
+      end.path
+    end
+
+    private
+
+    def source_store
+      return @source if @source.is_a?(Masamune::Schema::Store)
+      @source.try(:store)
+    end
+
+    def target_store
+      return @target if @target.is_a?(Masamune::Schema::Store)
+      @target.try(:store)
+    end
+
+    def template_eval(template)
+      return File.read(template) if File.exists?(template.to_s)
+      Masamune::Template.render_to_string(template_file(template), @locals.merge(source: source, target: target))
+    end
+
+    def template_file(template_prefix)
+      File.expand_path(File.join(__FILE__, '..', "#{template_prefix}.#{template_suffix}.erb"))
+    end
+
+    def template_suffix
+      case (target_store || source_store).try(:type)
+      when :postgres
+        'psql'
+      when :hive
+        'hql'
+      else
+        'txt'
+      end
+    end
+  end
+end