masamune 0.18.4 → 0.18.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74acb711585f7828067e24f3598a32ea3a60e9d9
-  data.tar.gz: a65eb2509f2b91f10b2819c64603496d41bde2d8
+  metadata.gz: 22e1c9eb4862cc2a74652f543e04efef5540467e
+  data.tar.gz: 1f8e4cdbf34061ebbe4bb379186b0479cb237d11
 SHA512:
-  metadata.gz: f1be2f3708efa65705e0ff2745027d564f9e8035b034764c062e04d19fb089ffc03d7383104fa7b3773fa658d8d11f237ef18694243d941a6f74ec3491467c50
-  data.tar.gz: fd906239e801f5b6f72b675f93e6ff48bcb004827f2f14da6be7be870d8f5fed4bfd177c5c28629e4f14a41b95bb67655d3d7e254b709bd4f69e3e8b340419b2
+  metadata.gz: 9e2898f8c2bb0984088592ae9bb6acc292db8357a92bb5401351429349284c2b2f0041e6fff45bff5892a0067f5e70bc60fe445300af201950762290ca88fcd4
+  data.tar.gz: 75ea31bb20edc4de25757e29bc8c8ef9a5b6d78eb38a3beb83d7b9419a8ee4455958f28e4ccd943874a7da547c16628e0870ac2cd441620e105e1bf1a14a05ff

data/lib/masamune/actions/aws_emr.rb

@@ -42,13 +42,15 @@ module Masamune::Actions
 
     included do |base|
       base.class_option :cluster_id, desc: 'AWS EMR cluster_id ID (Hint: `masamune-emr-aws list-clusters`)' if defined?(base.class_option)
-      base.after_initialize(:early) do |thor, options|
-        next unless thor.configuration.commands.aws_emr.any?
-        next unless thor.configuration.commands.aws_emr.fetch(:enabled, true)
-        thor.configuration.commands.aws_emr[:cluster_id] = options[:cluster_id] if options[:cluster_id]
-        next unless options[:initialize]
-        thor.validate_cluster_id!
-      end if defined?(base.after_initialize)
+      if defined?(base.after_initialize)
+        base.after_initialize(:early) do |thor, options|
+          next unless thor.configuration.commands.aws_emr.any?
+          next unless thor.configuration.commands.aws_emr.fetch(:enabled, true)
+          thor.configuration.commands.aws_emr[:cluster_id] = options[:cluster_id] if options[:cluster_id]
+          next unless options[:initialize]
+          thor.validate_cluster_id!
+        end
+      end
     end
   end
 end

data/lib/masamune/actions/data_flow.rb

@@ -88,14 +88,16 @@ module Masamune::Actions
         class_option :resolve, type: :boolean, desc: 'Recursively resolve data dependencies', default: true
       end
 
-      base.after_initialize(:final) do |thor, options|
-        thor.engine.environment = thor.environment
-        thor.engine.filesystem.environment = thor.environment
-        thor.environment.with_process_lock(:data_flow_after_initialize) do
-          thor.send(:prepare_and_execute, options)
+      if defined?(base.after_initialize)
+        base.after_initialize(:final) do |thor, options|
+          thor.engine.environment = thor.environment
+          thor.engine.filesystem.environment = thor.environment
+          thor.environment.with_process_lock(:data_flow_after_initialize) do
+            thor.send(:prepare_and_execute, options)
+          end
+          exit 0 if thor.top_level?
         end
-        exit 0 if thor.top_level?
-      end if defined?(base.after_initialize)
+      end
     end
 
     # rubocop:disable Style/ClassVars

data/lib/masamune/actions/execute.rb

@@ -44,17 +44,21 @@ module Masamune::Actions
         end
       end
 
-      klass.class_eval do
-        define_method(:stdin) do
-          @stdin ||= StringIO.new(opts[:input])
+      if opts[:input]
+        klass.class_eval do
+          define_method(:stdin) do
+            @stdin ||= StringIO.new(opts[:input])
+          end
         end
-      end if opts[:input]
+      end
 
-      klass.class_eval do
-        define_method(:handle_stdout) do |line, line_no|
-          yield(line, line_no)
+      if block_given?
+        klass.class_eval do
+          define_method(:handle_stdout) do |line, line_no|
+            yield(line, line_no)
+          end
         end
-      end if block_given?
+      end
 
       command = klass.new(self)
       command = Masamune::Commands::RetryWithBackoff.new(command, opts)

data/lib/masamune/actions/hive.rb

@@ -60,16 +60,20 @@ module Masamune::Actions
     end
 
     included do |base|
-      base.after_initialize do |thor, options|
-        next unless options[:initialize]
-        thor.create_hive_database_if_not_exists
-        thor.load_hive_schema
-      end if defined?(base.after_initialize)
+      if defined?(base.after_initialize)
+        base.after_initialize do |thor, options|
+          next unless options[:initialize]
+          thor.create_hive_database_if_not_exists
+          thor.load_hive_schema
+        end
+      end
 
-      base.after_initialize(:later) do |thor, options|
-        next unless options[:dry_run]
-        raise ::Thor::InvocationError, 'Dry run of hive failed' unless thor.hive(exec: 'SHOW TABLES;', safe: true, fail_fast: false).success?
-      end if defined?(base.after_initialize)
+      if defined?(base.after_initialize)
+        base.after_initialize(:later) do |thor, options|
+          next unless options[:dry_run]
+          raise ::Thor::InvocationError, 'Dry run of hive failed' unless thor.hive(exec: 'SHOW TABLES;', safe: true, fail_fast: false).success?
+        end
+      end
     end
   end
 end

data/lib/masamune/actions/invoke_parallel.rb

@@ -69,6 +69,7 @@ module Masamune::Actions
         when true
           "--#{k.to_s.tr('_', '-')}"
         when false
+          nil
         else
           ["--#{k.to_s.tr('_', '-')}", v]
         end

data/lib/masamune/actions/postgres.rb

@@ -42,17 +42,18 @@ module Masamune::Actions
     end
 
     def create_postgres_database_if_not_exists
-      if configuration.commands.postgres.key?(:database)
-        postgres_admin(action: :create, database: configuration.commands.postgres[:database], safe: true) unless postgres_helper.database_exists?
-      end
+      return unless configuration.commands.postgres.key?(:database)
+      return if postgres_helper.database_exists?
+      postgres_admin(action: :create, database: configuration.commands.postgres[:database], safe: true)
     end
 
     def load_postgres_setup_files
+      return unless configuration.commands.postgres.key?(:setup_files)
       configuration.commands.postgres[:setup_files].each do |file|
         configuration.with_quiet do
           postgres(file: file, retries: 0)
         end
-      end if configuration.commands.postgres.key?(:setup_files)
+      end
     end
 
     def load_postgres_schema
@@ -66,12 +67,14 @@ module Masamune::Actions
     end
 
     included do |base|
-      base.after_initialize do |thor, options|
-        next unless options[:initialize]
-        thor.create_postgres_database_if_not_exists
-        thor.load_postgres_setup_files
-        thor.load_postgres_schema
-      end if defined?(base.after_initialize)
+      if defined?(base.after_initialize)
+        base.after_initialize do |thor, options|
+          next unless options[:initialize]
+          thor.create_postgres_database_if_not_exists
+          thor.load_postgres_setup_files
+          thor.load_postgres_schema
+        end
+      end
     end
   end
 end

data/lib/masamune/cached_filesystem.rb

@@ -74,11 +74,12 @@ module Masamune
       end
 
       file_regexp = glob_to_regexp(file_or_glob, options)
+      return unless depth.zero?
       @cache.get(dirname).each do |entry|
         next if entry.name == dirname
         next unless entry.name =~ file_regexp
         yield entry
-      end if depth.zero?
+      end
     end
 
     # FIXME: cache eviction policy can be more precise
@@ -100,15 +101,15 @@ module Masamune
         return if @filesystem.root_path?(path)
         put(File.join(@filesystem.dirname(path), '.'), OpenStruct.new(name: @filesystem.dirname(path)))
         paths = path_split(path)
-        elems = paths.reverse.inject(entry) { |a, e| { e => a } }
+        elems = paths.reverse.inject(entry) { |acc, elem| { elem => acc } }
         @cache.deep_merge!(elems)
       end
 
       def get(path)
         return unless path
         paths = path_split(path)
-        elem = paths.inject(@cache) { |a, e| a.is_a?(Hash) ? a.fetch(e, {}) : a }
-        emit(elem)
+        value = paths.inject(@cache) { |acc, elem| acc.is_a?(Hash) ? acc.fetch(elem, {}) : acc }
+        emit(value)
       rescue KeyError
         EMPTY_SET
       end

data/lib/masamune/commands/postgres.rb

@@ -93,10 +93,9 @@ module Masamune::Commands
         logger.debug("#{output}:\n" + File.read(output))
       end
 
-      if @exec
-        console("postgres exec '#{strip_sql(@exec)}' #{'into ' + @output if @output}")
-        @file = exec_file
-      end
+      return unless @exec
+      console("postgres exec '#{strip_sql(@exec)}' #{'into ' + @output if @output}")
+      @file = exec_file
     end
 
     def handle_stdout(line, _line_no)

data/lib/masamune/commands/shell.rb

@@ -81,11 +81,8 @@ module Masamune::Commands
     end
 
     def command_args
-      if @delegate.respond_to?(:command_args) && @delegate.command_args
-        Array.wrap(@delegate.command_args).flatten.compact.map(&:to_s)
-      else
-        raise 'no command_args'
-      end
+      raise 'no command_args' unless @delegate.respond_to?(:command_args) && @delegate.command_args
+      Array.wrap(@delegate.command_args).flatten.compact.map(&:to_s)
     end
 
     def command_bin

data/lib/masamune/data_plan/elem.rb

@@ -53,9 +53,8 @@ class Masamune::DataPlan::Elem
   end
 
   def complete?
-    if rule.for_targets?
-      sources.existing.map(&:start_date).uniq.length == sources.map(&:start_date).uniq.length
-    end
+    return false unless rule.for_targets?
+    sources.existing.map(&:start_date).uniq.length == sources.map(&:start_date).uniq.length
   end
 
   def last_modified_at

data/lib/masamune/data_plan/engine.rb

@@ -121,22 +121,26 @@ class Masamune::DataPlan::Engine
     @target_rules[rule].try(:prepare)
     @source_rules[rule].try(:prepare)
 
-    constrain_max_depth(rule) do
-      sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, sources|
-        prepare(derived_rule, targets: sources.map(&:input)) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
+    if options.fetch(:resolve, true)
+      constrain_max_depth(rule) do
+        sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, sources|
+          prepare(derived_rule, targets: sources.map(&:input)) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
+        end
       end
-    end if options.fetch(:resolve, true)
+    end
     clear!
   end
 
   def execute(rule, options = {})
     return if targets(rule).actionable.empty?
 
-    constrain_max_depth(rule) do
-      sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, _sources|
-        execute(derived_rule, options) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
+    if options.fetch(:resolve, true)
+      constrain_max_depth(rule) do
+        sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, _sources|
+          execute(derived_rule, options) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
+        end
       end
-    end if options.fetch(:resolve, true)
+    end
 
     @command_rules[rule].call(self, rule, options)
     clear!

data/lib/masamune/data_plan/set.rb

@@ -145,6 +145,7 @@ class Masamune::DataPlan::Set < Set
   def convert_elem(elem)
     case elem
     when nil
+      nil
     when Masamune::DataPlan::Elem
       elem
     when String
@@ -157,6 +158,7 @@ class Masamune::DataPlan::Set < Set
   def convert_enum(enum)
     case enum
     when nil
+      nil
     when Array
       enum.flatten.uniq
     when Set, self.class, Enumerator

data/lib/masamune/filesystem.rb

@@ -410,7 +410,7 @@ module Masamune
         when :hdfs
           hadoop_fs('-chown', '-R', [user, group].compact.join(':'), *file_set)
         when :s3
-          # NOTE intentionally skip
+          nil # NOTE intentionally skip
         when :local
           FileUtils.chown_R(user, group, file_set, file_util_args)
         end

data/lib/masamune/io.rb

@@ -21,13 +21,17 @@
 #  THE SOFTWARE.
 
 class IO
-  def wait_readable(timeout = 0)
-    rs, _ws = IO.select([self], [], [], timeout)
-    rs && rs[0]
-  end unless IO.method_defined?(:wait_readable)
+  unless IO.method_defined?(:wait_readable)
+    def wait_readable(timeout = 0)
+      rs, _ws = IO.select([self], [], [], timeout)
+      rs && rs[0]
+    end
+  end
 
-  def wait_writable(timeout = 0)
-    _rs, ws = IO.select([], [self], [], timeout)
-    ws && ws[0]
-  end unless IO.method_defined?(:wait_writable)
+  unless IO.method_defined?(:wait_writable)
+    def wait_writable(timeout = 0)
+      _rs, ws = IO.select([], [self], [], timeout)
+      ws && ws[0]
+    end
+  end
 end

data/lib/masamune/schema/column.rb

@@ -476,7 +476,7 @@ module Masamune::Schema
       when Array
         value.map { |elem| ruby_value(elem, false) }
       when String
-        Array.wrap(JSON.load(value)).map { |elem| ruby_value(elem, false) }
+        Array.wrap(JSON.load(value)).map { |elem| ruby_value(elem, false) } # rubocop:disable Security/JSONLoad
       when nil
         []
       end

data/lib/masamune/schema/table.rb

@@ -352,14 +352,16 @@ module Masamune::Schema
     def unique_constraints_map
       @unique_constraints_map ||= begin
         map = Hash.new { |h, k| h[k] = [] }
-        columns.each do |_, column|
-          next if column.auto_reference
-          column.unique.each do |unique|
-            map[unique] += auto_surrogate_keys.map(&:name)
-            map[unique] << column.name
-            map[unique].uniq!
+        unless temporary?
+          columns.each do |_, column|
+            next if column.auto_reference
+            column.unique.each do |unique|
+              map[unique] += auto_surrogate_keys.map(&:name)
+              map[unique] << column.name
+              map[unique].uniq!
+            end
           end
-        end unless temporary?
+        end
         Hash[map.sort_by { |k, v| [v.length, k.to_s] }]
       end
     end

data/lib/masamune/transform/postgres/stage_fact.rb

@@ -88,10 +88,12 @@ module Masamune::Transform::Postgres
               coalesce_values << cross_references.map { |cross_reference, cross_column| cross_column.qualified_name(cross_reference.label) }
             end
 
-            column.reference.auto_surrogate_keys.each do |auto_surrogate_key|
-              next unless auto_surrogate_key.default
-              conditions[reference_name].conditions << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
-            end if column.reference
+            if column.reference
+              column.reference.auto_surrogate_keys.each do |auto_surrogate_key|
+                next unless auto_surrogate_key.default
+                conditions[reference_name].conditions << "#{auto_surrogate_key.qualified_name(reference.label)} = #{auto_surrogate_key.default}"
+              end
+            end
 
             if column.reference && !column.reference.default.nil? && column.adjacent.natural_key
               coalesce_values << column.reference.default(column.adjacent)

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.18.4'.freeze
+  VERSION = '0.18.5'.freeze
 end

data/spec/masamune/filesystem_spec.rb

@@ -747,7 +747,7 @@ shared_examples_for 'Filesystem' do
     end
 
     it { is_expected.to eq(%w(/tmp/c/00.txt /tmp/b/01.txt /tmp/a/02.txt)) }
-    it { expect { |b| instance.glob_sort('/tmp/*', order: :basename, &b) }.to yield_successive_args(*%w(/tmp/c/00.txt /tmp/b/01.txt /tmp/a/02.txt)) }
+    it { expect { |b| instance.glob_sort('/tmp/*', order: :basename, &b) }.to yield_successive_args('/tmp/c/00.txt', '/tmp/b/01.txt', '/tmp/a/02.txt') }
   end
 
   describe '#copy_file_to_file' do

data/spec/support/masamune/job_example_group.rb

@@ -20,6 +20,7 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+require_relative 'example_group'
 require_relative 'shared_example_group'
 
 module Masamune::JobExampleGroup

data/spec/support/masamune/step_example_group.rb

@@ -20,6 +20,8 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+require_relative 'shared_example_group'
+
 module Masamune::StepExampleGroup
   extend ActiveSupport::Concern
 

data/spec/support/masamune/task_example_group.rb

@@ -20,6 +20,7 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
+require_relative 'example_group'
 require_relative 'shared_example_group'
 
 module Masamune::TaskExampleGroup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.18.4
+  version: 0.18.5
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-25 00:00:00.000000000 Z
+date: 2016-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor