masamune 0.17.12 → 0.17.13

data/lib/masamune/commands/s3cmd.rb

@@ -25,12 +25,12 @@ require 'delegate'
 module Masamune::Commands
   class S3Cmd < SimpleDelegator
     DEFAULT_ATTRIBUTES =
-    {
-      :path     => 's3cmd',
-      :options  => [],
-      :extra    => [],
-      :block    => nil
-    }
+      {
+        path: 's3cmd',
+        options: [],
+        extra: [],
+        block: nil
+      }.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
@@ -47,7 +47,7 @@ module Masamune::Commands
       args.flatten
     end
 
-    def handle_stdout(line, line_no)
+    def handle_stdout(line, _line_no)
       @block.call(line) if @block
     end
 

data/lib/masamune/commands/shell.rb

@@ -57,30 +57,22 @@ module Masamune::Commands
     end
 
     def before_execute
-      if configuration.verbose
-        trace(command_args)
-      end
+      trace(command_args) if configuration.verbose
 
-      if @delegate.respond_to?(:before_execute)
-        @delegate.before_execute
-      end
+      @delegate.before_execute if @delegate.respond_to?(:before_execute)
     end
 
     def after_execute
-      if @delegate.respond_to?(:after_execute)
-        @delegate.after_execute
-      end
+      @delegate.after_execute if @delegate.respond_to?(:after_execute)
     end
 
     def around_execute(&block)
-      if configuration.dry_run && !safe
-        return OpenStruct.new(:success? => true)
-      end
+      return OpenStruct.new(success?: true) if configuration.dry_run && !safe
 
       if @delegate.respond_to?(:around_execute)
         @delegate.around_execute(&block)
       else
-        block.call
+        yield
       end
     end
 
@@ -101,7 +93,7 @@ module Masamune::Commands
     end
 
     def execute
-      status = OpenStruct.new(:success? => false, :exitstatus => 1)
+      status = OpenStruct.new(success?: false, exitstatus: 1)
 
       before_execute
       status = around_execute do
@@ -109,9 +101,7 @@ module Masamune::Commands
       end
       after_execute
 
-      unless status.success?
-        handle_failure(exit_code(status))
-      end
+      handle_failure(exit_code(status)) unless status.success?
       status
     rescue Interrupt
       handle_failure(SIGINT_EXIT_STATUS)
@@ -123,42 +113,39 @@ module Masamune::Commands
       logger.debug("execute: #{command_info}")
 
       Open3.popen3(command_env, *command_args) do |p_stdin, p_stdout, p_stderr, t_stdin|
-        p_stdin.wait_writable(PIPE_TIMEOUT) or raise "IO stdin not ready for write in #{PIPE_TIMEOUT}"
+        p_stdin.wait_writable(PIPE_TIMEOUT) || raise("IO stdin not ready for write in #{PIPE_TIMEOUT}")
 
-        Thread.new {
+        Thread.new do
           if @delegate.respond_to?(:stdin)
             @delegate.stdin.rewind
-            while line = @delegate.stdin.gets
+            until @delegate.stdin.eof?
+              line = @delegate.stdin.gets
               trace(line.chomp)
               p_stdin.puts line
               p_stdin.flush
             end
           else
-            while !p_stdin.closed? do
+            until p_stdin.closed?
               input = Readline.readline('', true).strip
               p_stdin.puts input
               p_stdin.flush
             end
           end
           p_stdin.close unless p_stdin.closed?
-        }
+        end
 
-        t_stderr = Thread.new {
-          while !p_stderr.eof?  do
-            handle_stderr(p_stderr)
-          end
+        t_stderr = Thread.new do
+          handle_stderr(p_stderr) until p_stderr.eof?
           p_stderr.close unless p_stderr.closed?
-        }
+        end
 
-        t_stdout = Thread.new {
-          while !p_stdout.eof?  do
-            handle_stdout(p_stdout)
-          end
+        t_stdout = Thread.new do
+          handle_stdout(p_stdout) until p_stdout.eof?
           p_stdout.close unless p_stdout.closed?
-        }
+        end
 
-        [t_stderr, t_stdout, t_stdin].compact.each { |t| t.join }
-        logger.debug("status: #{t_stdin.value.to_s}")
+        [t_stderr, t_stdout, t_stdin].compact.each(&:join)
+        logger.debug("status: #{t_stdin.value}")
         t_stdin.value
       end
     end
@@ -186,9 +173,7 @@ module Masamune::Commands
     end
 
     def handle_failure(status)
-      if @delegate.respond_to?(:handle_failure)
-        @delegate.handle_failure(status)
-      end
+      @delegate.handle_failure(status) if @delegate.respond_to?(:handle_failure)
       raise failure_message(status) if fail_fast
     end
 
@@ -208,7 +193,9 @@ module Masamune::Commands
     end
 
     def detach
-      STDIN.close; STDOUT.close; STDERR.close
+      STDIN.close
+      STDOUT.close
+      STDERR.close
     end
 
     def command_info

data/lib/masamune/configuration.rb

@@ -42,7 +42,7 @@ class Masamune::Configuration
   attr_accessor :backoff
   attr_accessor :params
 
-  COMMANDS = %w(aws_emr hive hadoop_streaming hadoop_filesystem s3cmd postgres postgres_admin)
+  COMMANDS = %w(aws_emr hive hadoop_streaming hadoop_filesystem s3cmd postgres postgres_admin).freeze
   COMMANDS.each do |command|
     attr_accessor command
     define_method(command) do
@@ -51,7 +51,7 @@ class Masamune::Configuration
   end
 
   def initialize(environment)
-    self.environment   = environment
+    self.environment = environment
     self.quiet    = false
     self.verbose  = false
     self.debug    = false
@@ -76,7 +76,7 @@ class Masamune::Configuration
           load_paths(value)
         elsif command == 'params'
           raise ArgumentError, 'params section must only contain key value pairs' unless value.is_a?(Hash)
-          self.params.merge! value
+          params.merge! value
         end
       end
       logger.debug("Loaded configuration #{config_file}")
@@ -97,7 +97,7 @@ class Masamune::Configuration
 
   def to_s
     io = StringIO.new
-    rep = {"path" => filesystem.paths}
+    rep = { 'path' => filesystem.paths }
     COMMANDS.each do |command|
       rep[command] = send(command)
     end
@@ -121,8 +121,9 @@ class Masamune::Configuration
 
   def_delegators :filesystem, :add_path, :get_path
 
-  def with_quiet(&block)
-    prev_quiet, self.quiet = quiet, true
+  def with_quiet
+    prev_quiet = quiet
+    self.quiet = true
     yield
   ensure
     self.quiet = prev_quiet
@@ -135,9 +136,7 @@ class Masamune::Configuration
   end
 
   class << self
-    def default_config_file=(config_file)
-      @default_config_file = config_file
-    end
+    attr_writer :default_config_file
 
     def default_config_file
       @default_config_file ||= File.join(File.expand_path('../../../', __FILE__), 'config', 'masamune.yml.erb')

data/lib/masamune/data_plan/builder.rb

@@ -33,8 +33,8 @@ class Masamune::DataPlan::Builder
       commands.each do |name|
         command_name = "#{namespaces.shift}:#{name}"
 
-        source_options = sources_for[name] || sources_anon.shift or next
-        target_options = targets_for[name] || targets_anon.shift or next
+        (source_options = sources_for[name] || sources_anon.shift) || next
+        (target_options = targets_for[name] || targets_anon.shift) || next
         next if source_options[:skip] || target_options[:skip]
 
         engine.add_source_rule(command_name, source_options)
@@ -48,16 +48,16 @@ class Masamune::DataPlan::Builder
   private
 
   def partition_by_for(annotations)
-    with_for, anon = annotations.partition { |opts| opts.has_key?(:for) }
+    with_for, anon = annotations.partition { |opts| opts.key?(:for) }
     decl = {}
     with_for.each do |opts|
-      decl[opts[:for]] = opts.reject { |k,_| k == :for }
+      decl[opts[:for]] = opts.reject { |k, _| k == :for }
     end
     [decl, anon]
   end
 
   def thor_command_wrapper
-    Proc.new do |engine, rule, _|
+    proc do |engine, rule, _|
       engine.environment.parent.invoke(rule)
     end
   end

data/lib/masamune/data_plan/elem.rb

@@ -36,13 +36,13 @@ class Masamune::DataPlan::Elem
   def input
     @input ||= start_time.strftime(strftime_format)
   end
-  alias :path :input
-  alias :table :input
+  alias path input
+  alias table input
 
   def partition
     input.split('_').last
   end
-  alias :suffix :partition
+  alias suffix partition
 
   def exists?
     if rule.for_path?
@@ -140,11 +140,11 @@ class Masamune::DataPlan::Elem
   end
 
   def next(i = 1)
-    self.class.new(@rule, start_time.advance(@rule.time_step => +1*i), @options)
+    self.class.new(@rule, start_time.advance(@rule.time_step => +1 * i), @options)
   end
 
   def prev(i = 1)
-    self.class.new(@rule, start_time.advance(@rule.time_step => -1*i), @options)
+    self.class.new(@rule, start_time.advance(@rule.time_step => -1 * i), @options)
   end
 
   def round(grain)
@@ -163,7 +163,7 @@ class Masamune::DataPlan::Elem
     uniq_constraint.hash
   end
 
-  # FIXME should consider stop_time for correctness
+  # FIXME: should consider stop_time for correctness
   def <=>(other)
     if start_time < other.start_time
       1
@@ -175,7 +175,7 @@ class Masamune::DataPlan::Elem
   end
 
   def inspect
-    {rule: rule, input: input, start_date: start_time.to_s, stop_date: stop_time.to_s, :options => options}.to_s
+    { rule: rule, input: input, start_date: start_time.to_s, stop_date: stop_time.to_s, options: options }.to_s
   end
 
   protected

data/lib/masamune/data_plan/engine.rb

@@ -31,12 +31,12 @@ class Masamune::DataPlan::Engine
   include Masamune::HasEnvironment
 
   def initialize
-    @target_rules = Hash.new
-    @source_rules = Hash.new
-    @command_rules = Hash.new
-    @targets = Hash.new { |set,rule| set[rule] = Masamune::DataPlan::Set.new(@target_rules[rule]) }
-    @sources = Hash.new { |set,rule| set[rule] = Masamune::DataPlan::Set.new(@source_rules[rule]) }
-    @set_cache = Hash.new { |cache,level| cache[level] = Hash.new }
+    @target_rules = {}
+    @source_rules = {}
+    @command_rules = {}
+    @targets = Hash.new { |set, rule| set[rule] = Masamune::DataPlan::Set.new(@target_rules[rule]) }
+    @sources = Hash.new { |set, rule| set[rule] = Masamune::DataPlan::Set.new(@source_rules[rule]) }
+    @set_cache = Hash.new { |cache, level| cache[level] = {} }
     @current_depth = 0
   end
 
@@ -64,17 +64,14 @@ class Masamune::DataPlan::Engine
     @command_rules[rule] = command
   end
 
-  # TODO use constructed reference instead
+  # TODO: use constructed reference instead
   def rule_for_target(target)
-    target_matches = @target_rules.select { |rule, matcher| matcher.primary? && matcher.matches?(target) }
-    source_matches = @source_rules.select { |rule, matcher| matcher.matches?(target) }
+    target_matches = @target_rules.select { |_rule, matcher| matcher.primary? && matcher.matches?(target) }
+    source_matches = @source_rules.select { |_rule, matcher| matcher.matches?(target) }
 
     if target_matches.empty?
-      if source_matches.empty?
-        raise "No rule matches target #{target}"
-      else
-        Masamune::DataPlan::Rule::TERMINAL
-      end
+      raise "No rule matches target #{target}" if source_matches.empty?
+      Masamune::DataPlan::Rule::TERMINAL
     else
       logger.error("Multiple rules match target #{target}") if target_matches.length > 1
       target_matches.map(&:first).first
@@ -136,7 +133,7 @@ class Masamune::DataPlan::Engine
     return if targets(rule).actionable.empty?
 
     constrain_max_depth(rule) do
-      sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, sources|
+      sources(rule).group_by { |source| rule_for_target(source.input) }.each do |derived_rule, _sources|
         execute(derived_rule, options) if derived_rule != Masamune::DataPlan::Rule::TERMINAL
       end
     end if options.fetch(:resolve, true)

data/lib/masamune/data_plan/rule.rb

@@ -97,7 +97,7 @@ class Masamune::DataPlan::Rule
       if for_path?
         engine.filesystem.eval_path(path)
       elsif for_table_with_partition?
-        [table , partition].join('_')
+        [table, partition].join('_')
       elsif for_table?
         table
       end.to_s
@@ -151,10 +151,11 @@ class Masamune::DataPlan::Rule
     return Set.new(to_enum(:generate, start_time, stop_time)) unless block_given?
     instance = bind_date_or_time(start_time)
 
-    begin
+    loop do
       yield instance
       instance = instance.next
-    end while instance.start_time <= stop_time
+      break unless instance.start_time <= stop_time
+    end
   end
 
   def generate_via_unify(elem, rule)
@@ -162,10 +163,11 @@ class Masamune::DataPlan::Rule
     instance = unify(elem, rule)
 
     stop_time = instance.start_time.advance(time_step => 1)
-    begin
+    loop do
       yield instance
       instance = instance.next
-    end while instance.start_time < stop_time
+      break unless instance.start_time < stop_time
+    end
   end
 
   def tz
@@ -219,17 +221,17 @@ class Masamune::DataPlan::Rule
 
   def adjacent_matches(instance)
     return Set.new(to_enum(:adjacent_matches, instance)) unless block_given?
-    (-window .. -1).each do |i|
+    (-window..-1).each do |i|
       yield instance.prev(i.abs)
     end
     yield instance
-    (1 .. window).each do |i|
+    (1..window).each do |i|
       yield instance.next(i)
     end
   end
 
   def inspect
-    {type: type, pattern: pattern, options: options}.to_s
+    { type: type, pattern: pattern, options: options }.to_s
   end
 
   def strftime_format
@@ -296,7 +298,7 @@ class Masamune::DataPlan::Rule
 
   def match_data_hash(match_data = nil)
     return unless match_data.present?
-    Hash.new.tap do |hash|
+    {}.tap do |hash|
       match_data.names.map(&:to_sym).each do |key|
         hash[key] = match_data[key.to_sym]
       end
@@ -304,8 +306,8 @@ class Masamune::DataPlan::Rule
   end
 
   def matched_date(matched_data)
-    if timestamp = matched_data[:timestamp]
-      Time.at(timestamp.to_i).to_datetime
+    if matched_data[:timestamp]
+      Time.at(matched_data[:timestamp].to_i).to_datetime
     else
       DateTime.new(*matched_data.values_at(:year, :month, :day, :hour).compact.map(&:to_i))
     end
@@ -316,6 +318,6 @@ class Masamune::DataPlan::Rule
   end
 
   def options_for_elem
-    @options.reject { |k,_| [:path, :table].include?(k) }
+    @options.reject { |k, _| [:path, :table].include?(k) }
   end
 end

data/lib/masamune/data_plan/set.rb

@@ -23,7 +23,7 @@
 require 'set'
 
 class Masamune::DataPlan::Set < Set
-  EMPTY = self.new
+  EMPTY = new
 
   attr_reader :rule
 
@@ -46,14 +46,14 @@ class Masamune::DataPlan::Set < Set
 
   def missing
     return self.class.new(rule, to_enum(__method__)) unless block_given?
-    self.each do |elem|
+    each do |elem|
       yield elem if elem.explode.none?
     end
   end
 
   def existing
     return self.class.new(rule, to_enum(__method__)) unless block_given?
-    self.each do |elem|
+    each do |elem|
       elem.explode.each do |new_elem|
         yield new_elem
       end
@@ -62,7 +62,7 @@ class Masamune::DataPlan::Set < Set
 
   def adjacent
     return self.class.new(rule, to_enum(__method__)) unless block_given?
-    self.each do |elem|
+    each do |elem|
       @rule.adjacent_matches(elem).each do |adj_elem|
         yield adj_elem
       end
@@ -72,7 +72,7 @@ class Masamune::DataPlan::Set < Set
   def stale
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
     return self.class.new(rule, to_enum(__method__)) unless block_given?
-    self.each do |target|
+    each do |target|
       yield target if target.sources.existing.any? { |source| target_stale?(source, target) }
     end
   end
@@ -81,8 +81,8 @@ class Masamune::DataPlan::Set < Set
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
     return self.class.new(rule, to_enum(__method__)) unless block_given?
     set = Set.new
-    self.each do |target|
-      yield target if set.add?(target) unless target.complete?
+    each do |target|
+      yield target unless target.complete? || !set.add?(target)
     end
   end
 
@@ -110,11 +110,11 @@ class Masamune::DataPlan::Set < Set
     end
   end
 
-  # TODO detect & warn or correct if coarser grain set is incomplete
+  # TODO: detect & warn or correct if coarser grain set is incomplete
   def with_grain(grain)
     return self.class.new(rule.round(grain), to_enum(:with_grain, grain)) unless block_given?
     seen = Set.new
-    self.each do |elem|
+    each do |elem|
       granular_elem = elem.round(grain)
       yield granular_elem if seen.add?(granular_elem)
     end
@@ -122,8 +122,8 @@ class Masamune::DataPlan::Set < Set
 
   def targets
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_targets?
-    return self.class.new(self.first.targets.rule, to_enum(__method__)) unless block_given?
-    self.each do |elem|
+    return self.class.new(first.targets.rule, to_enum(__method__)) unless block_given?
+    each do |elem|
       elem.targets do |target|
         yield target
       end
@@ -132,8 +132,8 @@ class Masamune::DataPlan::Set < Set
 
   def sources
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
-    return self.class.new(self.first.sources.rule, to_enum(__method__)) unless block_given?
-    self.each do |elem|
+    return self.class.new(first.sources.rule, to_enum(__method__)) unless block_given?
+    each do |elem|
       elem.sources do |source|
         yield source
       end