masamune 0.11.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4aa0472dd6fa4bb472d79caf9a6c17a4570baad2
-  data.tar.gz: 853b036996129473d7f9f8dee3129f198988face
+  metadata.gz: 06ac5ed8ef2f9949a444065bef219d2efd144b94
+  data.tar.gz: bf620007bd3016250133cda30feb6c03182ab1d4
 SHA512:
-  metadata.gz: b48a634347ae17d3a214be9ee13b021a7099c992b6e23382347ab44019b7d5209e6c3d73af0957067fff8abb9b3365c6a40fe880ef88cb56f67f4f7cfcab62cc
-  data.tar.gz: 861919413965988c50128a731f05e0ba9fd1bc9a5fc7a410038b23ce1e2e06fb4624889b11fb0142f321d0d2a8cc657e15a22f06a7efbc646674a4e99c228c16
+  metadata.gz: 589c931738aec24331ada2bab69a52175be9d27dc8a11e3a695ca7abbd38762582a5e3c93dafc61a6bbc7fac7f76d19ed6206a55a8e333795e6c113f0d64761c
+  data.tar.gz: cb8d8062f36d5ebc0bc72c3a8f3ba3c8acff5afa2f14186ddcf8b6844ca77cc9fc333b3802cab27443843ec8c8ed191e10f1274aea196b5be4c1acce7c8e8fbf

data/lib/masamune.rb

@@ -26,7 +26,6 @@ module Masamune
   require 'masamune/io'
   require 'masamune/template'
   require 'masamune/commands'
-  require 'masamune/accumulate'
   require 'masamune/last_element'
   require 'masamune/actions'
   require 'masamune/helpers'

data/lib/masamune/actions.rb

@@ -34,5 +34,6 @@ module Masamune
     require 'masamune/actions/elastic_mapreduce'
     require 'masamune/actions/execute'
     require 'masamune/actions/transform'
+    require 'masamune/actions/invoke_parallel'
   end
 end

data/lib/masamune/actions/execute.rb

@@ -46,7 +46,7 @@ module Masamune::Actions
       end if block_given?
 
       command = Masamune::Commands::Shell.new(klass.new(self), {fail_fast: false}.merge(opts))
-      command.execute
+      opts.fetch(:interactive, true) ? command.execute : command.replace(opts)
     end
   end
 end

data/lib/masamune/actions/invoke_parallel.rb

@@ -0,0 +1,73 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+require 'parallel'
+
+module Masamune::Actions
+  module InvokeParallel
+    extend ActiveSupport::Concern
+
+    include Masamune::Actions::Execute
+
+    included do |base|
+      base.class_option :max_tasks, aliases: '-p', :type => :numeric, :desc => 'Maximum number of tasks to execute in parallel', :default => 4
+    end
+
+    def invoke_parallel(*task_group)
+      opts = task_group.last.is_a?(Hash) ? task_group.pop.dup : {}
+      max_tasks = [opts.delete(:max_tasks), task_group.count].min
+      console("Setting max_tasks to #{max_tasks}")
+      bail_fast task_group, opts if opts[:version]
+      Parallel.each(task_group, in_processes: max_tasks) do |task_name|
+        begin
+          execute(thor_wrapper, task_name, *task_args(opts), interactive: false, detach: false)
+        rescue SystemExit
+        end
+      end
+    end
+
+    private
+
+    def thor_wrapper
+      'thor'
+    end
+
+    def bail_fast(task_group, opts = {})
+      task_name = task_group.first
+      execute($PROGRAM_NAME, task_name, *task_args(opts))
+      exit
+    end
+
+    def task_args(opts = {})
+      opts.map do |k,v|
+        case v
+        when true
+          "--#{k.to_s.gsub('_', '-')}"
+        when false
+        else
+          ["--#{k.to_s.gsub('_', '-')}", v]
+        end
+      end.flatten.compact
+    end
+  end
+end

data/lib/masamune/cached_filesystem.rb

@@ -22,8 +22,6 @@
 
 module Masamune
   class CachedFilesystem < SimpleDelegator
-    include Masamune::Accumulate
-
     def initialize(filesystem)
       super filesystem
       @filesystem = filesystem
@@ -38,12 +36,12 @@ module Masamune
       @cache.key?(file) || glob(file).include?(file) || @cache.key?(file)
     end
 
-    def glob(file_or_glob, &block)
+    def glob(file_or_glob)
+      return Set.new(to_enum(:glob, file_or_glob)) unless block_given?
       glob_stat(file_or_glob) do |entry|
         yield entry.name unless entry.name == dirname(file_or_glob)
       end
     end
-    method_accumulate :glob
 
     def stat(file_or_dir)
       raise ArgumentError, 'cannot contain wildcard' if file_or_dir.include?('*')

data/lib/masamune/commands/shell.rb

@@ -41,13 +41,13 @@ module Masamune::Commands
       @stderr_line_no = 0
     end
 
-    def replace
+    def replace(opts = {})
       logger.debug('replace: ' + command_args.join(' '))
       before_execute
       around_execute do
         pid = Process.fork
         if pid
-          STDIN.close; STDOUT.close; STDERR.close
+          detach if opts.fetch(:detach, true)
           Process.waitpid(pid)
           exit
         else
@@ -198,5 +198,9 @@ module Masamune::Commands
       return code unless status
       status.exitstatus
     end
+
+    def detach
+      STDIN.close; STDOUT.close; STDERR.close
+    end
   end
 end

data/lib/masamune/data_plan/elem.rb

@@ -23,7 +23,6 @@
 class Masamune::DataPlan::Elem
   MISSING_MODIFIED_AT = Time.new(0)
 
-  include Masamune::Accumulate
   include Comparable
 
   attr_reader :rule, :options
@@ -69,39 +68,39 @@ class Masamune::DataPlan::Elem
     end || MISSING_MODIFIED_AT
   end
 
-  def explode(&block)
+  def explode
+    return Set.new(to_enum(__method__)) unless block_given?
     if rule.for_path?
       file_glob = path
       file_glob += '/' unless path.include?('*') || path.include?('.')
       file_glob += '*' unless path.include?('*')
-      rule.engine.filesystem.glob(file_glob) do |new_path|
+      rule.engine.filesystem.glob(file_glob).each do |new_path|
         yield rule.bind_input(new_path)
       end
     elsif rule.for_table_with_partition?
       yield self if exists?
     end
   end
-  method_accumulate :explode
 
-  def targets(&block)
+  def targets
     return Masamune::DataPlan::Set::EMPTY if @rule.for_targets?
-    rule.engine.targets_for_source(rule.name, self) do |target|
+    return Masamune::DataPlan::Set.new(rule.engine.get_target_rule(rule.name), to_enum(__method__)) unless block_given?
+    rule.engine.targets_for_source(rule.name, self).each do |target|
       yield target
     end
   end
-  method_accumulate :targets, lambda { |elem| Masamune::DataPlan::Set.new(elem.rule.engine.get_target_rule(elem.rule.name)) }
 
   def target
     targets.first
   end
 
-  def sources(&block)
+  def sources
     return Masamune::DataPlan::Set::EMPTY if @rule.for_sources?
-    rule.engine.sources_for_target(rule.name, self) do |source|
+    return Masamune::DataPlan::Set.new(rule.engine.get_source_rule(rule.name), to_enum(__method__)) unless block_given?
+    rule.engine.sources_for_target(rule.name, self).each do |source|
       yield source
     end
   end
-  method_accumulate :sources, lambda { |elem| Masamune::DataPlan::Set.new(elem.rule.engine.get_source_rule(elem.rule.name)) }
 
   def source
     sources.first

data/lib/masamune/data_plan/engine.rb

@@ -27,7 +27,6 @@ class Masamune::DataPlan::Engine
   MAX_DEPTH = 10
 
   include Masamune::HasEnvironment
-  include Masamune::Accumulate
 
   def initialize
     @target_rules = Hash.new
@@ -80,34 +79,34 @@ class Masamune::DataPlan::Engine
     end
   end
 
-  def targets_for_date_range(rule, start, stop, &block)
+  def targets_for_date_range(rule, start, stop)
+    return Masamune::DataPlan::Set.new(get_target_rule(rule), to_enum(:targets_for_date_range, rule, start, stop)) unless block_given?
     target_template = @target_rules[rule]
     return unless target_template
-    target_template.generate(start.to_time.utc, stop.to_time.utc) do |target|
+    target_template.generate(start.to_time.utc, stop.to_time.utc).each do |target|
       yield target
     end
   end
-  method_accumulate :targets_for_date_range, lambda { |engine, rule, _, _| Masamune::DataPlan::Set.new(engine.get_target_rule(rule)) }
 
-  def targets_for_source(rule, source, &block)
+  def targets_for_source(rule, source)
+    return Masamune::DataPlan::Set.new(get_target_rule(rule), to_enum(:targets_for_source, rule, source)) unless block_given?
     source_template = @source_rules[rule]
     target_template = @target_rules[rule]
     source_instance = source_template.bind_input(source)
-    source_template.generate_via_unify(source_instance, target_template) do |target|
+    source_template.generate_via_unify(source_instance, target_template).each do |target|
       yield target
     end
   end
-  method_accumulate :targets_for_source, lambda { |engine, rule, source| Masamune::DataPlan::Set.new(engine.get_target_rule(rule)) }
 
-  def sources_for_target(rule, target, &block)
+  def sources_for_target(rule, target)
+    return Masamune::DataPlan::Set.new(get_source_rule(rule), to_enum(:sources_for_target, rule, target)) unless block_given?
     source_template = @source_rules[rule]
     target_template = @target_rules[rule]
     target_instance = target_template.bind_input(target)
-    target_template.generate_via_unify(target_instance, source_template) do |source|
+    target_template.generate_via_unify(target_instance, source_template).each do |source|
       yield source
     end
   end
-  method_accumulate :sources_for_target, lambda { |engine, rule, target| Masamune::DataPlan::Set.new(engine.get_source_rule(rule)) }
 
   def targets(rule)
     @set_cache[:targets_for_rule][rule] ||= @targets[rule].union(@sources[rule].targets)
@@ -146,7 +145,7 @@ class Masamune::DataPlan::Engine
     @current_depth > 0
   end
 
-  def constrain_max_depth(rule, &block)
+  def constrain_max_depth(rule)
     @current_depth += 1
     raise "Max depth of #{MAX_DEPTH} exceeded for rule '#{rule}'" if @current_depth > MAX_DEPTH
     yield

data/lib/masamune/data_plan/rule.rb

@@ -31,8 +31,6 @@ require 'date'
 class Masamune::DataPlan::Rule
   TERMINAL = nil
 
-  include Masamune::Accumulate
-
   attr_reader :engine, :name, :type, :options
 
   def initialize(engine, name, type, options = {})
@@ -128,7 +126,8 @@ class Masamune::DataPlan::Rule
     rule.bind_date(elem.start_time)
   end
 
-  def generate(start_time, stop_time, &block)
+  def generate(start_time, stop_time)
+    return Set.new(to_enum(:generate, start_time, stop_time)) unless block_given?
     instance = bind_date(start_time)
 
     begin
@@ -136,9 +135,9 @@ class Masamune::DataPlan::Rule
       instance = instance.next
     end while instance.start_time <= stop_time
   end
-  method_accumulate :generate
 
-  def generate_via_unify(elem, rule, &block)
+  def generate_via_unify(elem, rule)
+    return Set.new(to_enum(:generate_via_unify, elem, rule)) unless block_given?
     instance = unify(elem, rule)
 
     stop_time = instance.start_time.advance(time_step => 1)
@@ -147,7 +146,6 @@ class Masamune::DataPlan::Rule
       instance = instance.next
     end while instance.start_time < stop_time
   end
-  method_accumulate :generate_via_unify
 
   def tz
     ActiveSupport::TimeZone[@options.fetch(:tz, 'UTC')]
@@ -199,6 +197,7 @@ class Masamune::DataPlan::Rule
   end
 
   def adjacent_matches(instance)
+    return Set.new(to_enum(:adjacent_matches, instance)) unless block_given?
     (-window .. -1).each do |i|
       yield instance.prev(i.abs)
     end
@@ -207,7 +206,6 @@ class Masamune::DataPlan::Rule
       yield instance.next(i)
     end
   end
-  method_accumulate :adjacent_matches
 
   def inspect
     {type: type, pattern: pattern, options: options}.to_s

data/lib/masamune/data_plan/set.rb

@@ -25,8 +25,6 @@ require 'set'
 class Masamune::DataPlan::Set < Set
   EMPTY = self.new
 
-  include Masamune::Accumulate
-
   attr_reader :rule
 
   def initialize(rule, enum = nil)
@@ -46,50 +44,51 @@ class Masamune::DataPlan::Set < Set
     super convert_elem(elem)
   end
 
-  def missing(&block)
+  def missing
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     self.each do |elem|
-      yield elem if elem.explode.empty?
+      yield elem if elem.explode.count < 1
     end
   end
-  method_accumulate :missing, lambda { |set| set.class.new(set.rule) }
 
-  def existing(&block)
+  def existing
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     self.each do |elem|
-      elem.explode do |new_elem|
+      elem.explode.each do |new_elem|
         yield new_elem
       end
     end
   end
-  method_accumulate :existing, lambda { |set| set.class.new(set.rule) }
 
-  def adjacent(&block)
+  def adjacent
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     self.each do |elem|
-      @rule.adjacent_matches(elem) do |adj_elem|
+      @rule.adjacent_matches(elem).each do |adj_elem|
         yield adj_elem
       end
     end
   end
-  method_accumulate :adjacent, lambda { |set| set.class.new(set.rule) }
 
-  def stale(&block)
+  def stale
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     self.each do |target|
       yield target if target.sources.existing.any? { |source| target_stale?(source, target) }
     end
   end
-  method_accumulate :stale, lambda { |set| set.class.new(set.rule) }
 
-  def incomplete(&block)
+  def incomplete
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     set = Set.new
     self.each do |target|
       yield target if set.add?(target) unless target.complete?
     end
   end
-  method_accumulate :incomplete, lambda { |set| set.class.new(set.rule) }
 
-  def actionable(&block)
+  def actionable
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     set = Set.new
     missing.each do |target|
       yield target if set.add?(target)
@@ -101,41 +100,42 @@ class Masamune::DataPlan::Set < Set
       yield target if set.add?(target)
     end
   end
-  method_accumulate :actionable, lambda { |set| set.class.new(set.rule) }
 
-  def updateable(&block)
+  def updateable
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
+    return self.class.new(rule, to_enum(__method__)) unless block_given?
     set = Set.new
     actionable.each do |target|
       yield target if set.add?(target) && target.sources.existing.any?
     end
   end
-  method_accumulate :updateable, lambda { |set| set.class.new(set.rule) }
 
   # TODO detect & warn or correct if coarser grain set is incomplete
-  def with_grain(grain, &block)
+  def with_grain(grain)
+    return self.class.new(rule.round(grain), to_enum(:with_grain, grain)) unless block_given?
     seen = Set.new
     self.each do |elem|
       granular_elem = elem.round(grain)
       yield granular_elem if seen.add?(granular_elem)
     end
   end
-  method_accumulate :with_grain, lambda { |set, grain| set.class.new(set.rule.round(grain)) }
 
   def targets
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_targets?
-    self.class.new(self.first.targets.rule).tap do |set|
-      self.each do |elem|
-        set.merge elem.targets
+    return self.class.new(self.first.targets.rule, to_enum(__method__)) unless block_given?
+    self.each do |elem|
+      elem.targets.each do |target|
+        yield target
       end
     end
   end
 
   def sources
     return Masamune::DataPlan::Set::EMPTY if empty? || @rule.for_sources?
-    self.class.new(self.first.sources.rule).tap do |set|
-      self.each do |elem|
-        set.merge elem.sources
+    return self.class.new(self.first.sources.rule, to_enum(__method__)) unless block_given?
+    self.each do |elem|
+      elem.sources.each do |source|
+        yield source
       end
     end
   end
@@ -159,7 +159,7 @@ class Masamune::DataPlan::Set < Set
     when nil
     when Array
       enum.flatten.uniq
-    when Set, self.class
+    when Set, self.class, Enumerator
       enum
     when String
       [enum]

data/lib/masamune/environment.rb

@@ -89,9 +89,7 @@ module Masamune
     end
 
     def logger
-      @logger ||= Logger.new(log_file_io).tap do
-        symlink_latest_log
-      end
+      @logger ||= Logger.new(log_file_io)
     end
 
     def console(*a)
@@ -151,14 +149,5 @@ module Masamune
         configuration.debug ? $stderr : nil
       end
     end
-
-    def symlink_latest_log
-      return unless filesystem.has_path?(:log_dir)
-      latest = filesystem.path(:log_dir, 'latest')
-      FileUtils.rm(latest) if File.exists?(latest)
-      FileUtils.ln_s(log_file_name, latest)
-    rescue => e
-      logger.error(e)
-    end
   end
 end

data/lib/masamune/filesystem.rb

@@ -25,7 +25,6 @@ require 'masamune/has_environment'
 module Masamune
   class Filesystem
     include Masamune::HasEnvironment
-    include Masamune::Accumulate
     include Masamune::Actions::S3Cmd
     include Masamune::Actions::HadoopFilesystem
 
@@ -88,7 +87,7 @@ module Masamune
       path[0] != '/'
     end
 
-    def parent_paths(path, &block)
+    def parent_paths(path)
       if prefix = remote_prefix(path)
         node = path.split(prefix).last
       else
@@ -96,25 +95,26 @@ module Masamune
         node = path
       end
 
-      return if prefix.blank? && node.blank?
+      return [] if prefix.blank? && node.blank?
       parent_paths = node ? File.expand_path(node, '/').split('/') : []
       parent_paths.reject! { |x| x.blank? }
       parent_paths.prepend('/') if node =~ %r{\A/}
       tmp = []
+      result = []
       parent_paths.each do |part|
         tmp << part
         current_path = prefix + File.join(tmp)
         break if current_path == path
-        yield current_path
+        result << current_path
       end
+      result
     end
-    method_accumulate :parent_paths
 
     def root_path?(path)
       raise ArgumentError, 'path cannot be nil' if path.nil?
       raise ArgumentError, 'path cannot be blank' if path.blank?
       raise ArgumentError, 'path cannot be relative' if relative_path?(path)
-      parent_paths(path).length < 1
+      parent_paths(path).count < 1
     end
 
     def resolve_file(paths = [])
@@ -167,7 +167,8 @@ module Masamune
       end
     end
 
-    def glob_stat(pattern, &block)
+    def glob_stat(pattern)
+      return Set.new(to_enum(:glob_stat, pattern)) unless block_given?
       case type(pattern)
       when :hdfs
         hadoop_fs('-ls', '-R', pattern, safe: true) do |line|
@@ -193,7 +194,6 @@ module Masamune
         end
       end
     end
-    method_accumulate :glob_stat
 
     def stat(file_or_dir)
       raise ArgumentError, 'cannot contain wildcard' if file_or_dir.include?('*')
@@ -219,7 +219,8 @@ module Masamune
       end
     end
 
-    def glob(pattern, &block)
+    def glob(pattern)
+      return Set.new(to_enum(:glob, pattern)) unless block_given?
       case type(pattern)
       when :hdfs
         file_glob, file_regexp = glob_split(pattern)
@@ -245,7 +246,6 @@ module Masamune
         end
       end
     end
-    method_accumulate :glob
 
     def glob_sort(pattern, options = {})
       result = glob(pattern)

data/lib/masamune/schema/catalog.rb

@@ -115,7 +115,7 @@ module Masamune::Schema
       @stores[store_id.to_sym]
     end
 
-    def table(id, options = {}, &block)
+    def table(id, options = {})
       @context.push(options)
       yield if block_given?
       @context.tables[id] ||= Masamune::Schema::Table.new(@context.options.merge(id: id))
@@ -124,7 +124,7 @@ module Masamune::Schema
       @context.pop
     end
 
-    def dimension(id, options = {}, &block)
+    def dimension(id, options = {})
       @context.push(options)
       yield if block_given?
       @context.dimensions[id] ||= Masamune::Schema::Dimension.new(@context.options.merge(id: id))
@@ -133,7 +133,7 @@ module Masamune::Schema
       @context.pop
     end
 
-    def column(id, options = {}, &block)
+    def column(id, options = {})
       @context.options[:columns] << dereference_column(id, options)
     end
 
@@ -152,7 +152,7 @@ module Masamune::Schema
       @context.options[:rows] << Masamune::Schema::Row.new(attributes)
     end
 
-    def fact(id, options = {}, &block)
+    def fact(id, options = {})
       @context.push(options)
       grain = Array.wrap(options.delete(:grain) || [])
       fact_attributes(grain).each do |attributes|
@@ -164,15 +164,15 @@ module Masamune::Schema
       @context.pop
     end
 
-    def partition(id, options = {}, &block)
+    def partition(id, options = {})
       @context.options[:columns] << Masamune::Schema::Column.new(options.merge(id: id, partition: true))
     end
 
-    def measure(id, options = {}, &block)
+    def measure(id, options = {})
       @context.options[:columns] << Masamune::Schema::Column.new(options.merge(id: id, measure: true))
     end
 
-    def file(id, options = {}, &block)
+    def file(id, options = {})
       format_options = options.extract!(:format, :headers)
       @context.push(options)
       yield if block_given?
@@ -182,7 +182,7 @@ module Masamune::Schema
       @context.pop
     end
 
-    def event(id, options = {}, &block)
+    def event(id, options = {})
       @context.push(options)
       yield if block_given?
       @context.events[id] = HasMap.new Masamune::Schema::Event.new(@context.options.merge(id: id))
@@ -190,7 +190,7 @@ module Masamune::Schema
       @context.pop
     end
 
-    def attribute(id, options = {}, &block)
+    def attribute(id, options = {})
       @context.options[:attributes] << Masamune::Schema::Event::Attribute.new(options.merge(id: id))
     end
 

data/lib/masamune/schema/column.rb

@@ -376,7 +376,7 @@ module Masamune::Schema
 
     # TODO: Add ELEMENT REFERENCES
     def reference_constraint
-      return if parent.temporary?
+      return if parent && parent.temporary?
       return if degenerate?
       return if array_value?
       if reference && reference.surrogate_key.type == type
@@ -465,7 +465,7 @@ module Masamune::Schema
       if reference
         !(reference.null || reference.default)
       else
-        surrogate_key || natural_key
+        surrogate_key || natural_key || !(null || default)
       end
     end
 

data/lib/masamune/schema/table.rb

@@ -126,7 +126,8 @@ module Masamune::Schema
     # TODO: Default to GIN for array columns
     def index_columns
       index_column_map.map do |_, column_names|
-        [column_names, reverse_unique_constraints_map.key?(column_names.sort), short_md5(column_names)]
+        unique_index = reverse_unique_constraints_map.key?(column_names.sort)
+        [column_names, unique_index, short_md5(column_names)]
       end
     end
 
@@ -292,7 +293,7 @@ module Masamune::Schema
           column.unique.each do |unique|
             map[unique] << column.name
           end
-        end
+        end unless temporary?
         Hash[map.sort_by { |k, v| v.length }]
       end
     end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.11.0'
+  VERSION = '0.11.1'
 end

data/{lib/masamune/accumulate.rb → spec/masamune/actions/invoke_parallel_spec.rb}

@@ -20,41 +20,29 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-module Masamune
-  module Accumulate
-    def accumulate(method, accumulator, *args)
-      accumulator.call(self, *args).tap do |elems|
-        send(method, *args) do |elem|
-          elems << elem
-        end
-      end
+require 'spec_helper'
+
+describe Masamune::Actions::InvokeParallel do
+  let(:klass) do
+    Class.new(Thor) do
+      include Masamune::HasEnvironment
+      include Masamune::Actions::InvokeParallel
     end
+  end
 
-    module ClassMethods
-      def method_accumulate(method, accumulator = lambda { |_, *args| Array.new })
-        self.class_eval do
-          new_method = "#{method}_with_accumulate".to_sym
-          old_method = "#{method}_without_accumulate".to_sym
-          alias_method old_method, method
+  let(:instance) { klass.new }
 
-          define_method(new_method) do |*args, &block|
-            if block
-              instance = accumulator.call(self, *args)
-              send(old_method, *args) do |elem|
-                next if instance.respond_to?(:add?) && !instance.add?(elem)
-                block.call elem
-              end
-            else
-              accumulate(old_method, accumulator, *args)
-            end
-          end
-          alias_method method, new_method
-        end
+  describe '.invoke_parallel' do
+    context 'with a simple thor command' do
+      before do
+        mock_command(/\Athor list/, mock_success)
+      end
+
+      subject do
+        instance.invoke_parallel('list', max_tasks: 1)
       end
-    end
 
-    def self.included(base)
-      base.extend ClassMethods
+      it { expect { subject }.to_not raise_error }
     end
   end
 end

data/spec/masamune/cached_filesystem_spec.rb

@@ -40,8 +40,8 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
       expect(cached_filesystem.exists?('/a/b/c/4.txt')).to eq(false)
       expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
-      expect(cached_filesystem.glob('/a/b/c/*')).not_to be_empty
-      expect(cached_filesystem.glob('/a/b/c/*.txt')).not_to be_empty
+      expect(cached_filesystem.glob('/a/b/c/*').count).to eq(3)
+      expect(cached_filesystem.glob('/a/b/c/*.txt').count).to eq(3)
       expect(cached_filesystem.stat('/a/b/c/1.txt')).to_not be_nil
       expect(cached_filesystem.stat('/a/b/c/2.txt')).to_not be_nil
       expect(cached_filesystem.stat('/a/b/c/3.txt')).to_not be_nil
@@ -60,7 +60,9 @@ describe Masamune::CachedFilesystem do
     end
 
     it 'calls Filesystem#glob_stat once for multiple calls' do
-      expect(cached_filesystem.glob('/a/b/*')).to eq(['/a/b/c/1.txt', '/a/b/c/2.txt', '/a/b/c/3.txt'])
+      expect(cached_filesystem.glob('/a/b/*')).to include '/a/b/c/1.txt'
+      expect(cached_filesystem.glob('/a/b/*')).to include '/a/b/c/2.txt'
+      expect(cached_filesystem.glob('/a/b/*')).to include '/a/b/c/3.txt'
       expect(cached_filesystem.exists?('/a/b/c/1.txt')).to eq(true)
       expect(cached_filesystem.exists?('/a/b/c/2.txt')).to eq(true)
       expect(cached_filesystem.exists?('/a/b/c/3.txt')).to eq(true)
@@ -87,8 +89,8 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/y=2013/m=1/d=22/00000')).to eq(true)
       expect(cached_filesystem.exists?('/y=2013/m=1/d=22')).to eq(true)
       expect(cached_filesystem.exists?('/y=2013/m=1/d=2')).to eq(false)
-      expect(cached_filesystem.glob('/y=2013/m=1/*')).not_to be_empty
-      expect(cached_filesystem.glob('/y=2013/m=1/d=22/*')).not_to be_empty
+      expect(cached_filesystem.glob('/y=2013/m=1/*').count).to eq(1)
+      expect(cached_filesystem.glob('/y=2013/m=1/d=22/*').count).to eq(1)
       expect(cached_filesystem.stat('/y=2013/m=1/d=22/00000')).not_to be_nil
       expect(cached_filesystem.stat('/y=2013/m=1/d=22')).not_to be_nil
       expect(cached_filesystem.stat('/y=2013/m=1')).not_to be_nil
@@ -113,15 +115,15 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/logs/box4_123.txt')).to eq(false)
       expect(cached_filesystem.exists?('/logs/box4_456.txt')).to eq(false)
       expect(cached_filesystem.exists?('/logs/box')).to eq(false)
-      expect(cached_filesystem.glob('/logs/*')).not_to be_empty
-      expect(cached_filesystem.glob('/logs/*.txt')).not_to be_empty
-      expect(cached_filesystem.glob('/logs/box1_*.txt')).not_to be_empty
-      expect(cached_filesystem.glob('/logs/box2_*.txt')).not_to be_empty
-      expect(cached_filesystem.glob('/logs/box3_*.txt')).not_to be_empty
-      expect(cached_filesystem.glob('/logs/box*.txt').size).to eq(3)
-      expect(cached_filesystem.glob('/logs/box*.csv')).to be_empty
-      expect(cached_filesystem.glob('/logs/box')).to be_empty
-      expect(cached_filesystem.glob('/logs/box/*')).to be_empty
+      expect(cached_filesystem.glob('/logs/*').count).to eq(3)
+      expect(cached_filesystem.glob('/logs/*.txt').count).to eq(3)
+      expect(cached_filesystem.glob('/logs/box1_*.txt').count).to eq(1)
+      expect(cached_filesystem.glob('/logs/box2_*.txt').count).to eq(1)
+      expect(cached_filesystem.glob('/logs/box3_*.txt').count).to eq(1)
+      expect(cached_filesystem.glob('/logs/box*.txt').count).to eq(3)
+      expect(cached_filesystem.glob('/logs/box*.csv').count).to eq(0)
+      expect(cached_filesystem.glob('/logs/box').count).to eq(0)
+      expect(cached_filesystem.glob('/logs/box/*').count).to eq(0)
       expect(cached_filesystem.stat('/logs/box1_123.txt')).to_not be_nil
       expect(cached_filesystem.stat('/logs/box1_456.txt')).to be_nil
       expect(cached_filesystem.stat('/logs/box2_123.txt')).to_not be_nil
@@ -149,13 +151,13 @@ describe Masamune::CachedFilesystem do
       expect(cached_filesystem.exists?('/a/b/c')).to eq(true)
       expect(cached_filesystem.exists?('/a/b')).to eq(true)
       expect(cached_filesystem.exists?('/a')).to eq(true)
-      expect(cached_filesystem.glob('/a')).to be_empty
-      expect(cached_filesystem.glob('/a/*')).not_to be_empty
-      expect(cached_filesystem.glob('/a/b')).to be_empty
-      expect(cached_filesystem.glob('/a/b/*')).not_to be_empty
-      expect(cached_filesystem.glob('/a/b/c')).not_to be_empty
-      expect(cached_filesystem.glob('/a/b/c/*')).to be_empty
-      expect(cached_filesystem.glob('/a/b/c/*.txt')).to be_empty
+      expect(cached_filesystem.glob('/a').count).to eq(0)
+      expect(cached_filesystem.glob('/a/*').count).to eq(1)
+      expect(cached_filesystem.glob('/a/b').count).to eq(0)
+      expect(cached_filesystem.glob('/a/b/*').count).to eq(1)
+      expect(cached_filesystem.glob('/a/b/c').count).to eq(1)
+      expect(cached_filesystem.glob('/a/b/c/*').count).to eq(0)
+      expect(cached_filesystem.glob('/a/b/c/*.txt').count).to eq(0)
       expect(cached_filesystem.stat('/a/b/c/1.txt')).to be_nil
       expect(cached_filesystem.stat('/a/b/c/2.txt')).to be_nil
       expect(cached_filesystem.stat('/a/b/c/3.txt')).to be_nil

data/spec/masamune/data_plan/set_spec.rb

@@ -43,7 +43,7 @@ describe Masamune::DataPlan::Set do
     end
 
     context 'when all missing' do
-      it { expect(missing.size).to eq(3) }
+      it { expect(missing.count).to eq(3) }
       it { expect(missing).to include '/table/y=2013/m=01/d=01' }
       it { expect(missing).to include '/table/y=2013/m=01/d=02' }
       it { expect(missing).to include '/table/y=2013/m=01/d=03' }
@@ -53,7 +53,7 @@ describe Masamune::DataPlan::Set do
       before do
         fs.touch!('/table/y=2013/m=01/d=01', '/table/y=2013/m=01/d=02')
       end
-      it { expect(missing.size).to eq(3) }
+      it { expect(missing.count).to eq(3) }
       it { expect(missing).to include '/table/y=2013/m=01/d=01' }
       it { expect(missing).to include '/table/y=2013/m=01/d=02' }
       it { expect(missing).to include '/table/y=2013/m=01/d=03' }
@@ -63,7 +63,7 @@ describe Masamune::DataPlan::Set do
       before do
         fs.touch!('/table/y=2013/m=01/d=01/0000', '/table/y=2013/m=01/d=02/0000')
       end
-      it { expect(missing.size).to eq(1) }
+      it { expect(missing.count).to eq(1) }
       it { expect(missing).to include '/table/y=2013/m=01/d=03' }
     end
 
@@ -93,7 +93,7 @@ describe Masamune::DataPlan::Set do
         before do
           fs.touch!('/log/20130101.random_1.log', '/log/20130101.random_2.log')
         end
-        it { expect(existing.size).to eq(1) }
+        it { expect(existing.count).to eq(1) }
         it { expect(existing).to include '/log/20130101.random_1.log' }
       end
 
@@ -102,7 +102,7 @@ describe Masamune::DataPlan::Set do
           fs.touch!('/log/20130101.random_1.log', '/log/20130101.random_2.log')
           fs.touch!('/log/20130102.random_1.log', '/log/20130102.random_2.log')
         end
-        it { expect(existing.size).to eq(2) }
+        it { expect(existing.count).to eq(2) }
         it { expect(existing).to include '/log/20130101.random_1.log' }
         it { expect(existing).to include '/log/20130102.random_1.log' }
       end
@@ -119,7 +119,7 @@ describe Masamune::DataPlan::Set do
         before do
           fs.touch!('/log/20130101.random_1.log', '/log/20130101.random_2.log')
         end
-        it { expect(existing.size).to eq(2) }
+        it { expect(existing.count).to eq(2) }
         it { expect(existing).to include '/log/20130101.random_1.log' }
         it { expect(existing).to include '/log/20130101.random_2.log' }
       end
@@ -129,7 +129,7 @@ describe Masamune::DataPlan::Set do
           fs.touch!('/log/20130101.random_1.log', '/log/20130101.random_2.log')
           fs.touch!('/log/20130102.random_1.log', '/log/20130102.random_2.log')
         end
-        it { expect(existing.size).to eq(4) }
+        it { expect(existing.count).to eq(4) }
         it { expect(existing).to include '/log/20130101.random_1.log' }
         it { expect(existing).to include '/log/20130101.random_2.log' }
         it { expect(existing).to include '/log/20130102.random_1.log' }
@@ -156,7 +156,7 @@ describe Masamune::DataPlan::Set do
         allow(instance.rule).to receive(:window) { 1 }
       end
 
-      it { expect(sources.size).to eq(3) }
+      it { expect(sources.count).to eq(3) }
       it { expect(sources).to include '/log/20121231.*.log' }
       it { expect(sources).to include '/log/20130101.*.log' }
       it { expect(sources).to include '/log/20130102.*.log' }
@@ -169,7 +169,7 @@ describe Masamune::DataPlan::Set do
         before do
           fs.touch!('/log/20130101.random_1.log', '/log/20130101.random_2.log')
         end
-        it { expect(existing.size).to eq(2) }
+        it { expect(existing.count).to eq(2) }
         it { expect(existing).to include '/log/20130101.random_1.log' }
         it { expect(existing).to include '/log/20130101.random_2.log' }
       end
@@ -181,7 +181,7 @@ describe Masamune::DataPlan::Set do
           fs.touch!('/log/20130102.random_1.log', '/log/20130102.random_2.log')
           fs.touch!('/log/20130103.random_1.log', '/log/20130103.random_2.log')
         end
-        it { expect(existing.size).to eq(6) }
+        it { expect(existing.count).to eq(6) }
         it { expect(existing).to include '/log/20121231.random_1.log' }
         it { expect(existing).to include '/log/20121231.random_2.log' }
         it { expect(existing).to include '/log/20130101.random_1.log' }
@@ -230,7 +230,7 @@ describe Masamune::DataPlan::Set do
         fs.touch!('/log/20130101.random_1.log', mtime: future_time)
       end
 
-      it { expect(stale_targets.size).to eq(1) }
+      it { expect(stale_targets.count).to eq(1) }
       it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
     end
 
@@ -239,7 +239,7 @@ describe Masamune::DataPlan::Set do
         fs.touch!('/log/20130101.random_2.log', mtime: future_time)
       end
 
-      it { expect(stale_targets.size).to eq(1) }
+      it { expect(stale_targets.count).to eq(1) }
       it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
     end
 
@@ -248,7 +248,7 @@ describe Masamune::DataPlan::Set do
         fs.touch!('/log/20130101.random_1.log', mtime: present_time)
       end
 
-      it { expect(stale_targets.size).to eq(1) }
+      it { expect(stale_targets.count).to eq(1) }
       it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
     end
 
@@ -260,7 +260,7 @@ describe Masamune::DataPlan::Set do
         fs.touch!('/log/20130103.random_2.log', mtime: future_time)
       end
 
-      it { expect(stale_targets.size).to eq(3) }
+      it { expect(stale_targets.count).to eq(3) }
       it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
       it { expect(stale_targets).to include '/table/y=2013/m=01/d=02' }
       it { expect(stale_targets).to include '/table/y=2013/m=01/d=03' }
@@ -288,7 +288,7 @@ describe Masamune::DataPlan::Set do
     context 'when :day' do
       let(:grain) { :day }
       it 'has 6 items' do
-        expect(subject.size).to eq(6)
+        expect(subject.count).to eq(6)
       end
       it { is_expected.to include '/table/y=2012/m=12/d=29' }
       it { is_expected.to include '/table/y=2012/m=12/d=30' }
@@ -301,7 +301,7 @@ describe Masamune::DataPlan::Set do
     context 'when :month' do
       let(:grain) { :month }
       it 'has 3 items' do
-        expect(subject.size).to eq(3)
+        expect(subject.count).to eq(3)
       end
       it { is_expected.to include '/table/y=2012/m=12' }
       it { is_expected.to include '/table/y=2013/m=01' }
@@ -311,7 +311,7 @@ describe Masamune::DataPlan::Set do
     context 'when :year' do
       let(:grain) { :year }
       it 'has 2 items' do
-        expect(subject.size).to eq(2)
+        expect(subject.count).to eq(2)
       end
       it { is_expected.to include '/table/y=2012' }
       it { is_expected.to include '/table/y=2013' }
@@ -366,7 +366,7 @@ describe Masamune::DataPlan::Set do
     end
 
     context 'when all incomplete' do
-      it { expect(incomplete.size).to eq(2) }
+      it { expect(incomplete.count).to eq(2) }
       it { expect(incomplete).to include '/table/y=2014/m=01' }
       it { expect(incomplete).to include '/table/y=2014/m=02' }
     end
@@ -378,7 +378,7 @@ describe Masamune::DataPlan::Set do
         end
       end
 
-      it { expect(incomplete.size).to eq(1) }
+      it { expect(incomplete.count).to eq(1) }
       it { expect(incomplete).to include '/table/y=2014/m=02' }
     end
 
@@ -390,7 +390,7 @@ describe Masamune::DataPlan::Set do
         end
       end
 
-      it { expect(incomplete.size).to eq(0) }
+      it { expect(incomplete.count).to eq(0) }
     end
   end
 
@@ -432,7 +432,7 @@ describe Masamune::DataPlan::Set do
         it 'actionable is equivalent to stale' do
           expect(actionable).to eq(instance.stale)
         end
-        it { expect(updateable.size).to eq(3) }
+        it { expect(updateable.count).to eq(3) }
         it { expect(updateable).to include '/table/y=2013/m=01/d=01' }
         it { expect(updateable).to include '/table/y=2013/m=01/d=02' }
         it { expect(updateable).to include '/table/y=2013/m=01/d=03' }
@@ -464,7 +464,7 @@ describe Masamune::DataPlan::Set do
         it 'actionable is equivalent to missing' do
           expect(actionable).to eq(instance.missing)
         end
-        it { expect(updateable.size).to eq(1) }
+        it { expect(updateable.count).to eq(1) }
         it { expect(updateable).to include '/table/y=2013/m=01/d=01' }
       end
 
@@ -477,7 +477,7 @@ describe Masamune::DataPlan::Set do
         it 'actionable is equivalent to missing' do
           expect(actionable).to eq(instance.missing)
         end
-        it { expect(updateable.size).to eq(3) }
+        it { expect(updateable.count).to eq(3) }
         it { expect(updateable).to include '/table/y=2013/m=01/d=01' }
         it { expect(updateable).to include '/table/y=2013/m=01/d=02' }
         it { expect(updateable).to include '/table/y=2013/m=01/d=03' }
@@ -495,9 +495,9 @@ describe Masamune::DataPlan::Set do
 
     context 'when sources are missing' do
       it 'should chain expectedly' do
-        expect(instance.targets.size).to eq(2)
-        expect(instance.targets.sources.existing.size).to eq(0)
-        expect(instance.targets.sources.existing.targets.size).to eq(0)
+        expect(instance.targets.count).to eq(2)
+        expect(instance.targets.sources.existing.count).to eq(0)
+        expect(instance.targets.sources.existing.targets.count).to eq(0)
       end
     end
 
@@ -507,10 +507,10 @@ describe Masamune::DataPlan::Set do
       end
 
       it 'should chain expectedly' do
-        expect(instance.targets.size).to eq(2)
-        expect(instance.targets.sources.existing.size).to eq(4)
-        expect(instance.targets.sources.existing.targets.size).to eq(2)
-        expect(instance.targets.sources.existing.targets.sources.existing.size).to eq(4)
+        expect(instance.targets.count).to eq(2)
+        expect(instance.targets.sources.existing.count).to eq(4)
+        expect(instance.targets.sources.existing.targets.count).to eq(2)
+        expect(instance.targets.sources.existing.targets.sources.existing.count).to eq(4)
       end
     end
   end

data/spec/masamune/filesystem_spec.rb

@@ -590,20 +590,20 @@ shared_examples_for 'Filesystem' do
 
     context 'local no matches' do
       let(:pattern) { File.join(new_dir, '*') }
-      it { is_expected.to be_empty }
+      it { expect(subject.count).to eq(0) }
       it { expect { |b| instance.glob(pattern, &b) }.to_not yield_control }
     end
 
     context 'local one matches' do
       let(:pattern) { File.join(File.dirname(old_file), '*') }
-      it { is_expected.not_to be_empty }
+      it { expect(subject.count).to eq(1) }
       it { expect { |b| instance.glob(pattern, &b) }.to yield_with_args(old_file) }
     end
 
     context 'local one matches (recursive)' do
       let(:pattern) { File.join(tmp_dir, '*') }
       it 'has 2 items' do
-        expect(subject.size).to eq(2)
+        expect(subject.count).to eq(2)
       end
       it { is_expected.to include old_dir }
       it { is_expected.to include old_file }
@@ -613,7 +613,7 @@ shared_examples_for 'Filesystem' do
     context 'local one matches (with suffix)' do
       let(:pattern) { File.join(File.dirname(old_file), '*.txt') }
       it 'has 1 item' do
-        expect(subject.size).to eq(1)
+        expect(subject.count).to eq(1)
       end
       it { is_expected.to include old_file }
       it { expect { |b| instance.glob(pattern, &b) }.to yield_with_args(old_file) }
@@ -629,7 +629,7 @@ shared_examples_for 'Filesystem' do
           and_yield('')
       end
       let(:pattern) { File.join(new_dir, '*') }
-      it { is_expected.to be_empty }
+      it { expect(subject.count).to eq(0) }
       it { expect { |b| instance.glob('file://' + pattern, &b) }.to_not yield_control }
     end
 
@@ -643,7 +643,7 @@ shared_examples_for 'Filesystem' do
           and_yield("drwxrwxrwt   - root     wheel         68 2015-02-24 12:09 #{old_file}")
       end
       let(:pattern) { File.join(File.dirname(old_file), '*') }
-      it { is_expected.not_to be_empty }
+      it { expect(subject.count).to eq(1) }
       it { expect { |b| instance.glob('file://' + pattern, &b) }.to yield_with_args('file://' + old_file) }
     end
 
@@ -655,7 +655,7 @@ shared_examples_for 'Filesystem' do
         expect(filesystem).to receive(:s3cmd).with('ls', '--recursive', "s3://bucket/dir/*", safe: true).at_most(:once)
       end
 
-      it { is_expected.to be_empty }
+      it { expect(subject.count).to eq(0) }
     end
 
     context 's3 no matches with implicit glob results' do
@@ -667,7 +667,7 @@ shared_examples_for 'Filesystem' do
           and_yield(%q(2013-05-24 18:53      2912   s3://bucket/dir/02.txt))
       end
 
-      it { is_expected.to be_empty }
+      it { expect(subject.count).to eq(0) }
     end
 
     context 's3 one matches' do

data/spec/masamune/schema/column_spec.rb

@@ -652,6 +652,16 @@ describe Masamune::Schema::Column do
 
     context 'by default' do
       let(:column) { described_class.new id: 'name', type: :string }
+      it { is_expected.to eq(true) }
+    end
+
+    context 'when :null true' do
+      let(:column) { described_class.new id: 'name', type: :string, null: true }
+      it { is_expected.to eq(false) }
+    end
+
+    context 'when column has default' do
+      let(:column) { described_class.new id: 'name', type: :string, default: 'missing' }
       it { is_expected.to eq(false) }
     end
 

data/spec/masamune/schema/table_spec.rb

@@ -82,6 +82,9 @@ describe Masamune::Schema::Table do
     end
 
     it { expect(table.name).to eq('user_table') }
+    it { expect(table.unique_columns).to include :tenant_id }
+    it { expect(table.unique_columns).to include :user_id }
+    it { expect(table.stage_table.unique_columns).to be_empty }
   end
 
   context 'with enum column' do

data/spec/support/masamune/mock_filesystem.rb

@@ -23,8 +23,6 @@
 require 'delegate'
 
 class Masamune::MockFilesystem < Delegator
-  include Masamune::Accumulate
-
   def initialize
     @filesystem = Masamune::Filesystem.new
     @filesystem.add_path :root_dir, File.expand_path('../../../', __FILE__)
@@ -42,19 +40,20 @@ class Masamune::MockFilesystem < Delegator
     @files.keys.include?(file)
   end
 
-  def glob(pattern, &block)
+  def glob(pattern)
+    return Set.new(to_enum(:glob, pattern)) unless block_given?
     file_regexp = glob_to_regexp(pattern)
     @files.keys.each do |name|
       yield name if name =~ file_regexp
     end
   end
-  method_accumulate :glob
 
   def glob_sort(pattern, options = {})
     glob(pattern)
   end
 
-  def glob_stat(pattern, &block)
+  def glob_stat(pattern)
+    return Set.new(to_enum(:glob_stat, pattern)) unless block_given?
     file_regexp = glob_to_regexp(pattern, recursive: true)
     @files.each do |name, stat|
       yield stat if name =~ file_regexp

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.11.1
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-09 00:00:00.000000000 Z
+date: 2015-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -183,7 +197,6 @@ files:
 - bin/masamune-psql
 - bin/masamune-shell
 - lib/masamune.rb
-- lib/masamune/accumulate.rb
 - lib/masamune/actions.rb
 - lib/masamune/actions/data_flow.rb
 - lib/masamune/actions/date_parse.rb
@@ -193,6 +206,7 @@ files:
 - lib/masamune/actions/hadoop_filesystem.rb
 - lib/masamune/actions/hadoop_streaming.rb
 - lib/masamune/actions/hive.rb
+- lib/masamune/actions/invoke_parallel.rb
 - lib/masamune/actions/postgres.rb
 - lib/masamune/actions/postgres_admin.rb
 - lib/masamune/actions/s3cmd.rb
@@ -291,6 +305,7 @@ files:
 - spec/masamune/actions/hadoop_filesystem_spec.rb
 - spec/masamune/actions/hadoop_streaming_spec.rb
 - spec/masamune/actions/hive_spec.rb
+- spec/masamune/actions/invoke_parallel_spec.rb
 - spec/masamune/actions/postgres_admin_spec.rb
 - spec/masamune/actions/postgres_spec.rb
 - spec/masamune/actions/s3cmd_spec.rb
@@ -393,6 +408,7 @@ test_files:
 - spec/masamune/actions/hadoop_filesystem_spec.rb
 - spec/masamune/actions/hadoop_streaming_spec.rb
 - spec/masamune/actions/hive_spec.rb
+- spec/masamune/actions/invoke_parallel_spec.rb
 - spec/masamune/actions/postgres_admin_spec.rb
 - spec/masamune/actions/postgres_spec.rb
 - spec/masamune/actions/s3cmd_spec.rb