masamune 0.11.5 → 0.11.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 575463d3df3f7dd11eb973b5611b67d2c8feeeda
-  data.tar.gz: fc08c068a9161aa2cafcbad96266e589bf77b3af
+  metadata.gz: 16d125ee3baa02af550e3e1be25c02f2e01f3ffa
+  data.tar.gz: 0ed7d28ecc6daaf83699138ac0b4b1acdb8363f9
 SHA512:
-  metadata.gz: 14298e5d7168af6f03d8b26acc31ec8a3302a990e8ac68faab544db2f028bb61b5579f48916e7d833665c7976a73b1ec36d1318be3e11059f11d7d9c694a9a42
-  data.tar.gz: 5debba170513313ab984f1d9a5de7516563c83511191a863512ab11ca3b76d0e599b0cd93ed9b72c7d3237ecc731ffdaa156a986933477d1be496d5d97a0a646
+  metadata.gz: 07aa680b40ea443bc1d6609b03c57a988a7d519739db1ce7cbefef54e09f408920b7424e73815f4fb4c0b588a2adec984b35432c220cfdc08bb19bfb0a161879
+  data.tar.gz: 01794b4dca6b07019aa83976ca6e72620e4c8dd3cbb1ff13b59fbc9d941bf7d2cd7ad59c5d2ac009585702d30554d3990b927bbb2b8fc9bda4a9b4569f6d24de

data/lib/masamune/actions/data_flow.rb

@@ -57,6 +57,21 @@ module Masamune::Actions
       Set.new File.read(value).split(/\s+/)
     end
 
+    def execute(options = {})
+      raise Thor::RequiredArgumentMissingError, "No value provided for required options '--start' or '--at'" unless options[:start] || options[:at] || options[:sources] || options[:targets]
+      raise Thor::MalformattedArgumentError, "Cannot specify both option '--sources' and option '--targets'" if options[:sources] && options[:targets]
+
+      desired_sources = Masamune::DataPlan::Set.new current_command_name, parse_file_type(:sources)
+      desired_targets = Masamune::DataPlan::Set.new current_command_name, parse_file_type(:targets)
+
+      if start_time && stop_time
+        desired_targets.merge engine.targets_for_date_range(current_command_name, start_time, stop_time)
+      end
+
+      engine.prepare(current_command_name, options.merge(sources: desired_sources, targets: desired_targets))
+      engine.execute(current_command_name, options)
+    end
+
     private
 
     included do |base|
@@ -68,23 +83,11 @@ module Masamune::Actions
       end
 
       base.after_initialize(:final) do |thor, options|
-        # Only execute this block if DataPlan::Engine is not currently executing
-        next if thor.engine.executing?
         thor.engine.environment = thor.environment
         thor.engine.filesystem.environment = thor.environment
-
-        raise Thor::RequiredArgumentMissingError, "No value provided for required options '--start' or '--at'" unless options[:start] || options[:at] || options[:sources] || options[:targets]
-        raise Thor::MalformattedArgumentError, "Cannot specify both option '--sources' and option '--targets'" if options[:sources] && options[:targets]
-
-        desired_sources = Masamune::DataPlan::Set.new thor.current_command_name, thor.parse_file_type(:sources)
-        desired_targets = Masamune::DataPlan::Set.new thor.current_command_name, thor.parse_file_type(:targets)
-
-        if thor.start_time && thor.stop_time
-          desired_targets.merge thor.engine.targets_for_date_range(thor.current_command_name, thor.start_time, thor.stop_time)
+        thor.environment.with_process_lock(:data_flow_after_initialize) do
+          thor.execute(options)
         end
-
-        thor.engine.prepare(thor.current_command_name, options.merge(sources: desired_sources, targets: desired_targets))
-        thor.engine.execute(thor.current_command_name, options)
         exit 0 if thor.top_level?
       end if defined?(base.after_initialize)
     end

data/lib/masamune/data_plan/builder.rb

@@ -58,9 +58,7 @@ class Masamune::DataPlan::Builder
 
   def thor_command_wrapper
     Proc.new do |engine, rule, _|
-      engine.environment.with_exclusive_lock(rule) do
-        engine.environment.parent.invoke(rule)
-      end
+      engine.environment.parent.invoke(rule)
     end
   end
 end

data/lib/masamune/data_plan/engine.rb

@@ -141,10 +141,6 @@ class Masamune::DataPlan::Engine
     clear!
   end
 
-  def executing?
-    @current_depth > 0
-  end
-
   def constrain_max_depth(rule)
     @current_depth += 1
     raise "Max depth of #{MAX_DEPTH} exceeded for rule '#{rule}'" if @current_depth > MAX_DEPTH

data/lib/masamune/environment.rb

@@ -53,16 +53,18 @@ module Masamune
       @mutex ||= Mutex.new
     end
 
-    def with_exclusive_lock(name, &block)
+    def with_exclusive_lock(name, options = {}, &block)
       raise 'filesystem path :run_dir not defined' unless filesystem.has_path?(:run_dir)
       lock_name = [name, configuration.lock].compact.join(':')
       logger.debug("acquiring lock '#{lock_name}'")
       lock_file = lock_file(lock_name)
-      lock_status = lock_file.flock(File::LOCK_EX | File::LOCK_NB)
+      lock_mode = File::LOCK_EX
+      lock_mode |= File::LOCK_NB if options[:non_blocking]
+      lock_status = lock_file.flock(lock_mode)
       if lock_status == 0
         yield if block_given?
       else
-        logger.error "acquire lock attempt failed for '#{lock_name}'"
+        logger.error "acquire lock attempt failed for '#{lock_name}'" unless options[:non_blocking]
       end
     ensure
       if lock_file
@@ -71,6 +73,12 @@ module Masamune
       end
     end
 
+    def with_process_lock(name)
+      with_exclusive_lock("#{name}_#{Process.pid}", non_blocking: true) do
+        yield
+      end
+    end
+
     def log_file_template
       @log_file_template || "#{Time.now.to_i}-#{$$}.log"
     end

data/lib/masamune/helpers/postgres.rb

@@ -39,7 +39,7 @@ module Masamune::Helpers
     end
 
     def database_exists?
-      postgres(exec: 'SELECT version();', fail_fast: false).success?
+      @database_exists ||= postgres(exec: 'SELECT version();', fail_fast: false).success?
     end
 
     def table_exists?(table)

data/lib/masamune/thor.rb

@@ -160,6 +160,18 @@ module Masamune
           def top_level?
             self.current_command_name == ARGV.first
           end
+
+          def invoke_command(command, *args)
+            environment.with_exclusive_lock(command.name, non_blocking: true) do
+              super
+            end
+          end
+
+          def invoke(name = nil, *args)
+            environment.with_exclusive_lock(name, non_blocking: top_level?) do
+              super
+            end
+          end
         end
 
         private

data/lib/masamune/transform.rb

@@ -27,6 +27,7 @@ module Masamune
     require 'masamune/transform/define_table'
     require 'masamune/transform/define_event_view'
     require 'masamune/transform/define_schema'
+    require 'masamune/transform/denormalize_table'
 
     require 'masamune/transform/bulk_upsert'
     require 'masamune/transform/insert_reference_values'

data/lib/masamune/transform/denormalize_table.psql.erb

@@ -0,0 +1,39 @@
+--  The MIT License (MIT)
+--
+--  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+--
+--  Permission is hereby granted, free of charge, to any person obtaining a copy
+--  of this software and associated documentation files (the "Software"), to deal
+--  in the Software without restriction, including without limitation the rights
+--  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+--  copies of the Software, and to permit persons to whom the Software is
+--  furnished to do so, subject to the following conditions:
+--
+--  The above copyright notice and this permission notice shall be included in
+--  all copies or substantial portions of the Software.
+--
+--  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+--  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+--  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+--  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+--  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+--  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+--  THE SOFTWARE.
+
+SELECT
+  <%- target.select_columns(columns).each do |column, last| -%>
+  <%= column %><%= ',' unless last %>
+  <%- end -%>
+FROM
+  <%= target.name %>
+<%- target.join_conditions(columns).each do |table, condition| -%>
+JOIN
+  <%= table %>
+ON
+  <%= condition %>
+<%- end -%>
+ORDER BY
+  <%- target.order_by_columns(columns).each do |column, last| -%>
+  <%= column %><%= ',' unless last %>
+  <%- end -%>
+;

data/lib/masamune/transform/denormalize_table.rb

@@ -0,0 +1,71 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Transform
+  module DenormalizeTable
+    extend ActiveSupport::Concern
+
+    def denormalize_table(target, columns = [])
+      Operator.new(__method__, target: target, columns: columns, presenters: { postgres: Postgres })
+    end
+
+    private
+
+    class Postgres < SimpleDelegator
+      include Masamune::LastElement
+
+      def select_columns(column_names)
+        column_names.map do |column_name|
+          next unless column = dereference_column_name(column_name)
+          if column.reference
+            "#{column.foreign_key_name} AS #{column.name}"
+          else
+            column.qualified_name
+          end
+        end.compact
+      end
+      method_with_last_element :select_columns
+
+      def join_conditions(column_names)
+        {}.tap do |conditions|
+          column_names.each do |column_name|
+            next unless column = dereference_column_name(column_name)
+            next unless column.reference
+            adjacent_reference = references[column.reference.id]
+            next unless adjacent_reference
+            adjacent_column = columns[adjacent_reference.foreign_key_name]
+            next unless adjacent_column
+            conditions[column.reference.name] = "#{column.reference.surrogate_key.qualified_name} = #{adjacent_column.qualified_name}"
+          end
+        end
+      end
+
+      def order_by_columns(column_names)
+        column_names.map do |column_name|
+          next unless column = dereference_column_name(column_name)
+          column.name
+        end.compact
+      end
+      method_with_last_element :order_by_columns
+    end
+  end
+end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.11.5'
+  VERSION = '0.11.6'
 end

data/spec/masamune/transform/denormalize_table_spec.rb

@@ -0,0 +1,125 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'spec_helper'
+
+describe Masamune::Transform::DenormalizeTable do
+  before do
+    catalog.schema :postgres do
+      dimension 'cluster', type: :mini do
+        column 'id', type: :sequence, surrogate_key: true, auto: true
+        column 'name', type: :string
+
+        row name: 'current_database()', attributes: {default: true}
+      end
+
+      dimension 'date', type: :date do
+        column 'date_id', type: :integer, unique: true, index: true, natural_key: true
+      end
+
+      dimension 'tenant', type: :two do
+        column 'tenant_id', type: :integer, index: true, natural_key: true
+      end
+
+      dimension 'user', type: :two do
+        column 'tenant_id', type: :integer, index: true, natural_key: true
+        column 'user_id', type: :integer, index: true, natural_key: true
+      end
+
+      dimension 'user_agent', type: :mini do
+        column 'name', type: :string, unique: true, index: 'shared'
+        column 'version', type: :string, unique: true, index: 'shared', default: 'Unknown'
+        column 'mobile', type: :boolean, unique: true, index: 'shared', default: false
+        column 'description', type: :string, null: true, ignore: true
+      end
+
+      fact 'visits', partition: 'y%Ym%m' do
+        references :cluster
+        references :date
+        references :tenant
+        references :user
+        references :user_agent
+        measure 'total', type: :integer
+      end
+    end
+  end
+
+  let(:target) { catalog.postgres.visits_fact }
+  let(:columns) do
+    [
+      'date.date_id',
+      'tenant.tenant_id',
+      'user.tenant_id',
+      'user.user_id',
+      'user_agent.name',
+      'user_agent.version',
+      'total',
+      'time_key'
+    ]
+  end
+
+  context 'with postgres fact' do
+    subject(:result) { transform.denormalize_table(target, columns).to_s }
+
+    it 'should eq render denormalize_table template' do
+      is_expected.to eq <<-EOS.strip_heredoc
+      SELECT
+        date_dimension.date_id AS date_dimension_date_id,
+        tenant_dimension.tenant_id AS tenant_dimension_tenant_id,
+        user_dimension.tenant_id AS user_dimension_tenant_id,
+        user_dimension.user_id AS user_dimension_user_id,
+        user_agent_type.name AS user_agent_type_name,
+        user_agent_type.version AS user_agent_type_version,
+        visits_fact.total,
+        visits_fact.time_key
+      FROM
+        visits_fact
+      JOIN
+        date_dimension
+      ON
+        date_dimension.id = visits_fact.date_dimension_id
+      JOIN
+        tenant_dimension
+      ON
+        tenant_dimension.id = visits_fact.tenant_dimension_id
+      JOIN
+        user_dimension
+      ON
+        user_dimension.id = visits_fact.user_dimension_id
+      JOIN
+        user_agent_type
+      ON
+        user_agent_type.id = visits_fact.user_agent_type_id
+      ORDER BY
+        date_dimension_date_id,
+        tenant_dimension_tenant_id,
+        user_dimension_tenant_id,
+        user_dimension_user_id,
+        user_agent_type_name,
+        user_agent_type_version,
+        total,
+        time_key
+      ;
+      EOS
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.11.5
+  version: 0.11.6
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-15 00:00:00.000000000 Z
+date: 2015-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -114,14 +114,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.1
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rb-readline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -264,6 +278,8 @@ files:
 - lib/masamune/transform/define_table.psql.erb
 - lib/masamune/transform/define_table.rb
 - lib/masamune/transform/define_unique.psql.erb
+- lib/masamune/transform/denormalize_table.psql.erb
+- lib/masamune/transform/denormalize_table.rb
 - lib/masamune/transform/insert_reference_values.psql.erb
 - lib/masamune/transform/insert_reference_values.rb
 - lib/masamune/transform/load_dimension.rb
@@ -339,6 +355,7 @@ files:
 - spec/masamune/transform/define_table.dimension_spec.rb
 - spec/masamune/transform/define_table.fact_spec.rb
 - spec/masamune/transform/define_table.table_spec.rb
+- spec/masamune/transform/denormalize_table_spec.rb
 - spec/masamune/transform/insert_reference_values.dimension_spec.rb
 - spec/masamune/transform/insert_reference_values.fact_spec.rb
 - spec/masamune/transform/load_dimension_spec.rb
@@ -442,6 +459,7 @@ test_files:
 - spec/masamune/transform/define_table.dimension_spec.rb
 - spec/masamune/transform/define_table.fact_spec.rb
 - spec/masamune/transform/define_table.table_spec.rb
+- spec/masamune/transform/denormalize_table_spec.rb
 - spec/masamune/transform/insert_reference_values.dimension_spec.rb
 - spec/masamune/transform/insert_reference_values.fact_spec.rb
 - spec/masamune/transform/load_dimension_spec.rb