masamune 0.13.8 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4d4d75d505e70be3e5c119ccb677ad9e9846a85
-  data.tar.gz: 2518fa319f34c708c94b363473b9e0ab4cf73b75
+  metadata.gz: feca9f373a42ecd946e9682b08213c3357ce7437
+  data.tar.gz: 51cf1a4d353fb1f607299937e60c66437213d35c
 SHA512:
-  metadata.gz: 43e4b96852c3cacaadf6a82169eefd892d962329f151ced5923369281340cf0a2b49e1edadc0de59cb1080f89b2c91ebfa61378e6bcae602ccd7af361ccb793a
-  data.tar.gz: e5aabba71b0721d37537f59d0744ebb02a66e73dcc060a98704bf8247b9ec229c3db8a3a338a4603c2b57e780a97cec663da5ecb653a67ff09fc6acd577ecf13
+  metadata.gz: 2e1d3c79fbe068bb79cb653cb746845dbaf7c33bc4525727cfa569b40407578b970418ee701ecb4ed0ed15004fb48d183c14398c03e6267fa8f5429d0184d0cf
+  data.tar.gz: f62cd40539a9ac2036ee312e9abf9055792463f3340faaed9857bd2dfb16d06cf3fe12e32ba41bc1b3676b23e512641abcaf07c929fe3e0e70bc5b4240d002ea

data/lib/masamune.rb

@@ -20,26 +20,29 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+require 'active_support/concern'
+
 module Masamune
   require 'masamune/environment'
   require 'masamune/has_environment'
   require 'masamune/io'
   require 'masamune/template'
-  require 'masamune/commands'
   require 'masamune/last_element'
-  require 'masamune/actions'
-  require 'masamune/helpers'
   require 'masamune/configuration'
-  require 'masamune/data_plan'
   require 'masamune/thor'
   require 'masamune/thor_loader'
   require 'masamune/filesystem'
   require 'masamune/cached_filesystem'
   require 'masamune/method_logger'
   require 'masamune/after_initialize_callbacks'
+  require 'masamune/topological_hash'
+
+  require 'masamune/data_plan'
+  require 'masamune/commands'
+  require 'masamune/actions'
+  require 'masamune/helpers'
   require 'masamune/schema'
   require 'masamune/transform'
-  require 'masamune/topological_hash'
 
   extend self
   extend Masamune::HasEnvironment

data/lib/masamune/actions.rb

@@ -22,18 +22,6 @@
 
 module Masamune
   module Actions
-    require 'masamune/actions/hive'
-    require 'masamune/actions/s3cmd'
-    require 'masamune/actions/hadoop_streaming'
-    require 'masamune/actions/hadoop_filesystem'
-    require 'masamune/actions/postgres'
-    require 'masamune/actions/postgres_admin'
-    require 'masamune/actions/filesystem'
-    require 'masamune/actions/date_parse'
-    require 'masamune/actions/data_flow'
-    require 'masamune/actions/elastic_mapreduce'
-    require 'masamune/actions/execute'
-    require 'masamune/actions/transform'
-    require 'masamune/actions/invoke_parallel'
+    Dir["#{File.dirname(__FILE__)}/actions/*.rb"].each { |f| require f }
   end
 end

data/lib/masamune/actions/data_flow.rb

@@ -21,7 +21,8 @@
 #  THE SOFTWARE.
 
 require 'chronic'
-require 'active_support/concern'
+
+require 'masamune/actions/date_parse'
 
 module Masamune::Actions
   module DataFlow

data/lib/masamune/actions/date_parse.rb

@@ -21,7 +21,6 @@
 #  THE SOFTWARE.
 
 require 'chronic'
-require 'active_support/concern'
 
 module Masamune::Actions
   module DateParse

data/lib/masamune/actions/elastic_mapreduce.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
-
 module Masamune::Actions
   module ElasticMapreduce
     extend ActiveSupport::Concern

data/lib/masamune/actions/filesystem.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
-
 module Masamune::Actions
   module Filesystem
     extend ActiveSupport::Concern

data/lib/masamune/actions/hive.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
-
 require 'masamune/transform/define_schema'
 
 module Masamune::Actions

data/lib/masamune/actions/invoke_parallel.rb

@@ -20,9 +20,10 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
 require 'parallel'
 
+require 'masamune/actions/execute'
+
 module Masamune::Actions
   module InvokeParallel
     extend ActiveSupport::Concern

data/lib/masamune/actions/postgres.rb

@@ -20,7 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
 require 'masamune/transform/define_schema'
 require 'masamune/actions/postgres_admin'
 

data/lib/masamune/actions/s3cmd.rb

@@ -20,6 +20,8 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+require 'masamune/commands/s3cmd'
+
 module Masamune::Actions
   module S3Cmd
     include Masamune::Commands::S3Cmd::ClassMethods

data/lib/masamune/actions/transform.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
-
 require 'masamune/actions/postgres'
 
 require 'masamune/transform/load_dimension'

data/lib/masamune/after_initialize_callbacks.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
-
 module Masamune
   module AfterInitializeCallbacks
     extend ActiveSupport::Concern

data/lib/masamune/commands.rb

@@ -22,16 +22,6 @@
 
 module Masamune
   module Commands
-    require 'masamune/commands/hive'
-    require 'masamune/commands/s3cmd'
-    require 'masamune/commands/hadoop_streaming'
-    require 'masamune/commands/hadoop_filesystem'
-    require 'masamune/commands/elastic_mapreduce'
-    require 'masamune/commands/postgres_common'
-    require 'masamune/commands/postgres'
-    require 'masamune/commands/postgres_admin'
-    require 'masamune/commands/interactive'
-    require 'masamune/commands/shell'
-    require 'masamune/commands/retry_with_backoff'
+    Dir["#{File.dirname(__FILE__)}/commands/*.rb"].each { |f| require f }
   end
 end

data/lib/masamune/commands/postgres.rb

@@ -23,6 +23,7 @@
 require 'delegate'
 
 require 'masamune/string_format'
+require 'masamune/commands/postgres_common'
 
 module Masamune::Commands
   class Postgres < SimpleDelegator

data/lib/masamune/commands/postgres_admin.rb

@@ -22,6 +22,8 @@
 
 require 'delegate'
 
+require 'masamune/commands/postgres_common'
+
 module Masamune::Commands
   class PostgresAdmin < SimpleDelegator
     include Masamune::Commands::PostgresCommon

data/lib/masamune/configuration.rb

@@ -27,6 +27,8 @@ require 'pp'
 
 require 'active_support/core_ext/hash'
 
+require 'masamune/has_environment'
+
 class Masamune::Configuration
   extend Forwardable
   include Masamune::HasEnvironment

data/lib/masamune/data_plan/engine.rb

@@ -23,6 +23,8 @@
 require 'active_support'
 require 'active_support/core_ext/numeric/time'
 
+require 'masamune/has_environment'
+
 class Masamune::DataPlan::Engine
   MAX_DEPTH = 10
 

data/lib/masamune/filesystem.rb

@@ -21,6 +21,8 @@
 #  THE SOFTWARE.
 
 require 'masamune/has_environment'
+require 'masamune/actions/s3cmd'
+require 'masamune/actions/hadoop_filesystem'
 
 module Masamune
   class Filesystem

data/lib/masamune/helpers.rb

@@ -22,6 +22,6 @@
 
 module Masamune
   module Helpers
-    require 'masamune/helpers/postgres'
+    Dir["#{File.dirname(__FILE__)}/helpers/*.rb"].each { |f| require f }
   end
 end

data/lib/masamune/last_element.rb

@@ -20,8 +20,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-require 'active_support/concern'
-
 module Masamune
   module LastElement
     extend ActiveSupport::Concern

data/lib/masamune/schema/dimension.rb

@@ -62,7 +62,7 @@ module Masamune::Schema
       when :two
         [:start_at, :end_at, :version, :last_modified_at]
       when :four
-        [:parent_id, :record_id, :start_at, :end_at, :version, :last_modified_at]
+        [:start_at, :end_at, :version, :last_modified_at]
       when :ledger
         [:source_kind, :source_uuid, :start_at, :last_modified_at, :delta]
       else
@@ -103,8 +103,6 @@ module Masamune::Schema
       when :four
         children << ledger_table
         # FIXME derive type from from parent
-        initialize_column! id: 'parent_id', type: :integer, null: true, reference: TableReference.new(ledger_table)
-        initialize_column! id: 'record_id', type: :integer, null: true, reference: TableReference.new(ledger_table)
         initialize_column! id: 'start_at', type: :timestamp, default: 'TO_TIMESTAMP(0)', index: [:start_at, :natural], unique: :natural
         initialize_column! id: 'end_at', type: :timestamp, null: true, index: :end_at
         initialize_column! id: 'version', type: :integer, default: 1, null: true

data/lib/masamune/schema/store.rb

@@ -22,6 +22,8 @@
 
 require 'active_support/core_ext/hash'
 
+require 'masamune/has_environment'
+
 module Masamune::Schema
   class Store
     include Masamune::HasEnvironment

data/lib/masamune/schema/table.rb

@@ -20,6 +20,8 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+require 'masamune/last_element'
+
 module Masamune::Schema
   class Table
     include Masamune::LastElement

data/lib/masamune/template.rb

@@ -42,7 +42,10 @@ module Masamune
       return partial_file if Pathname.new(partial_file).absolute?
       @paths.each do |path|
         file = File.expand_path(File.join(path, partial_file))
-        return file if File.exists?(file) && File.file?(file)
+        if File.exists?(file) && File.file?(file)
+          @paths << File.dirname(file)
+          return file 
+        end
       end
       raise IOError, "File not found: #{partial_file}"
     end

data/lib/masamune/thor.rb

@@ -22,8 +22,8 @@
 
 require 'date'
 require 'thor'
-require 'active_support/concern'
 
+require 'masamune/has_environment'
 require 'masamune/after_initialize_callbacks'
 
 module Masamune

data/lib/masamune/transform.rb

@@ -22,26 +22,6 @@
 
 module Masamune
   module Transform
-    require 'masamune/transform/operator'
-
-    require 'masamune/transform/define_table'
-    require 'masamune/transform/define_schema'
-    require 'masamune/transform/denormalize_table'
-
-    require 'masamune/transform/bulk_upsert'
-    require 'masamune/transform/insert_reference_values'
-
-    require 'masamune/transform/stage_dimension'
-    require 'masamune/transform/stage_fact'
-
-    require 'masamune/transform/load_dimension'
-    require 'masamune/transform/load_fact'
-
-    require 'masamune/transform/snapshot_dimension'
-    require 'masamune/transform/deduplicate_dimension'
-    require 'masamune/transform/consolidate_dimension'
-    require 'masamune/transform/relabel_dimension'
-
-    require 'masamune/transform/rollup_fact'
+    Dir["#{File.dirname(__FILE__)}/transform/*.rb"].each { |f| require f }
   end
 end

data/lib/masamune/transform/bulk_upsert.rb

@@ -25,28 +25,7 @@ module Masamune::Transform
     extend ActiveSupport::Concern
 
     def bulk_upsert(source, target)
-      Operator.new(__method__, source: source, target: target, presenters: { postgres: Postgres})
-    end
-
-    private
-
-    class Postgres < SimpleDelegator
-      include Masamune::LastElement
-
-      def update_columns
-        columns.values.reject { |column| reserved_column_ids.include?(column.id) || column.surrogate_key || column.natural_key || column.unique.any? || column.auto_reference || column.ignore }
-      end
-      method_with_last_element :update_columns
-
-      def insert_columns
-        columns.values.reject { |column| column.surrogate_key || column.auto_reference || column.ignore }
-      end
-      method_with_last_element :insert_columns
-
-      def unique_columns
-        columns.values.select { |column| column.unique.any? && !column.null }
-      end
-      method_with_last_element :unique_columns
+      Operator.new(__method__, source: source, target: target)
     end
   end
 end

data/lib/masamune/transform/common.rb

@@ -0,0 +1,27 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module Common
+    Dir["#{File.dirname(__FILE__)}/common/*.rb"].each { |f| require f }
+  end
+end

data/lib/masamune/transform/common/denormalize_table.rb

@@ -0,0 +1,90 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'masamune/last_element'
+
+module Masamune::Transform::Common
+  class DenormalizeTable
+    def initialize(options = {})
+      @target   = options[:target]
+      @columns  = options[:columns]
+      @order_by = options[:order_by]
+    end
+
+    def locals
+      { target: target, columns: @columns, order_by: @order_by }
+    end
+
+    def target
+      TargetPresenter.new(@target)
+    end
+
+    private
+
+    class TargetPresenter < SimpleDelegator
+      include Masamune::LastElement
+
+      def select_columns(column_names)
+        column_names.map do |column_name|
+          next unless column = dereference_column_name(column_name)
+          if column.reference
+            if column.reference.implicit || column.reference.degenerate
+              "#{column.name} AS #{column.name}"
+            else
+              "#{column.foreign_key_name} AS #{column.name}"
+            end
+          else
+            column.qualified_name
+          end
+        end.compact
+      end
+      method_with_last_element :select_columns
+
+      def join_alias(reference)
+        reference.label ? "#{reference.name} AS #{[reference.label, reference.name].compact.join('_')}" : reference.name
+      end
+
+      def join_conditions(column_names)
+        {}.tap do |conditions|
+          column_names.each do |column_name|
+            next unless column = dereference_column_name(column_name)
+            next unless column.reference
+            next if column.reference.degenerate
+            adjacent_reference = references[column.reference.id]
+            next unless adjacent_reference
+            adjacent_column = columns[adjacent_reference.foreign_key_name]
+            next unless adjacent_column
+            conditions[join_alias(column.reference)] = "#{column.reference.surrogate_key.qualified_name(column.reference.label)} = #{adjacent_column.qualified_name}"
+          end
+        end
+      end
+
+      def order_by_columns(column_names)
+        column_names.map do |column_name|
+          next unless column = dereference_column_name(column_name)
+          column.name
+        end.compact
+      end
+      method_with_last_element :order_by_columns
+    end
+  end
+end