masamune 0.17.12 → 0.17.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1eb266f52acc5ee8e0c52a49902c99818f6f12b5
-  data.tar.gz: d508a06edbd0ee12026452da4d6dcb8bc4d67602
+  metadata.gz: 211a98d1efc9791eeceb07e89df0c2ad70c27a4a
+  data.tar.gz: 2a7240f993c7a1252f75f009be836b5b6b5bdd32
 SHA512:
-  metadata.gz: 3e07e29a9d2ae2f8d30ca1e4955ff0a9be518f99ea6776aabc9ee5410dd79a3b4c99e28b2ce4cc0a772d4c0e7f693b18e6789567f5a8d7b201bf72b5ce9fe3dc
-  data.tar.gz: ae3fddf024a995705f361a0b11de610d1efe7f75e4211dbadcb604cd0ef410f6f5beb7821fdcb6cb3ee0ec3147d243cd8dea6e10485ff9b034646cdd7ef0972f
+  metadata.gz: 91022241697047ed01744ef834602314e35b90144c50d84699d4c21ab3403f4333188c7a340fac00a90433f92fe382654a852d7817cc43fd0942f5e7ef24f1a7
+  data.tar.gz: 13b3a769d6edf9efa8b50e6a982b59c3d0241c9a38558700e6938a30748aea1953c8b83e6a1ea834141446b5557e8886bb2681b5ae7bfdfeb98d3d3c59465449

data/Rakefile

@@ -37,16 +37,16 @@ RSpec::Core::RakeTask.new(:spec)
 namespace :spec do
   desc 'Run Rspec unit code examples'
   RSpec::Core::RakeTask.new(:unit) do |spec|
-    spec.pattern = "spec/**/*_spec.rb"
+    spec.pattern = 'spec/**/*_spec.rb'
   end
 
   desc 'Run Rspec acceptance code examples'
   RSpec::Core::RakeTask.new(:acceptance) do |spec|
-    spec.pattern = "examples/**/*_spec.rb"
+    spec.pattern = 'examples/**/*_spec.rb'
   end
 
   desc 'Run All Rspec code examples'
   task all: [:unit, :acceptance]
 end
 
-task :default => :spec
+task default: :spec

data/bin/masamune-aws-emr

@@ -21,6 +21,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-$: << File.expand_path('../../lib/', __FILE__)
+$LOAD_PATH << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/aws_emr_thor'
 Masamune::Tasks::AwsEmrThor.start(ARGV)

data/bin/masamune-dump

@@ -21,6 +21,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-$: << File.expand_path('../../lib/', __FILE__)
+$LOAD_PATH << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/dump_thor'
 Masamune::Tasks::DumpThor.start(ARGV)

data/bin/masamune-hive

@@ -21,6 +21,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-$: << File.expand_path('../../lib/', __FILE__)
+$LOAD_PATH << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/hive_thor'
 Masamune::Tasks::HiveThor.start(ARGV)

data/bin/masamune-psql

@@ -21,6 +21,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-$: << File.expand_path('../../lib/', __FILE__)
+$LOAD_PATH << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/postgres_thor'
 Masamune::Tasks::PostgresThor.start(ARGV)

data/bin/masamune-shell

@@ -21,6 +21,6 @@
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 #  THE SOFTWARE.
 
-$: << File.expand_path('../../lib/', __FILE__)
+$LOAD_PATH << File.expand_path('../../lib/', __FILE__)
 require 'masamune/tasks/shell_thor'
 Masamune::Tasks::ShellThor.start(ARGV)

data/lib/masamune.rb

@@ -43,6 +43,6 @@ module Masamune
   require 'masamune/schema'
   require 'masamune/transform'
 
-  extend self
+  extend self # rubocop:disable Style/ModuleFunction
   extend Masamune::HasEnvironment
 end

data/lib/masamune/actions/aws_emr.rb

@@ -37,11 +37,11 @@ module Masamune::Actions
     def validate_cluster_id!
       cluster_id = configuration.aws_emr[:cluster_id]
       raise ::Thor::RequiredArgumentMissingError, "No value provided for required options '--cluster-id'" unless cluster_id
-      raise ::Thor::RequiredArgumentMissingError, %Q(AWS EMR cluster '#{cluster_id}' does not exist) unless aws_emr(action: 'describe-cluster', cluster_id: cluster_id, fail_fast: false).success?
+      raise ::Thor::RequiredArgumentMissingError, %(AWS EMR cluster '#{cluster_id}' does not exist) unless aws_emr(action: 'describe-cluster', cluster_id: cluster_id, fail_fast: false).success?
     end
 
     included do |base|
-      base.class_option :cluster_id, :desc => "AWS EMR cluster_id ID (Hint: `masamune-emr-aws list-clusters`)" if defined?(base.class_option)
+      base.class_option :cluster_id, desc: 'AWS EMR cluster_id ID (Hint: `masamune-emr-aws list-clusters`)' if defined?(base.class_option)
       base.after_initialize(:early) do |thor, options|
         next unless thor.configuration.aws_emr.any?
         next unless thor.configuration.aws_emr.fetch(:enabled, true)

data/lib/masamune/actions/data_flow.rb

@@ -50,11 +50,11 @@ module Masamune::Actions
       sources.first
     end
 
-    # TODO sources from file or input array
+    # TODO: sources from file or input array
     def parse_file_type(key)
       return Set.new unless key
-      value = options[key] or return Set.new
-      File.exists?(value) or raise Thor::MalformattedArgumentError, "Expected file value for '--#{key}'; got #{value}"
+      (value = options[key]) || (return Set.new)
+      File.exist?(value) || raise(Thor::MalformattedArgumentError, "Expected file value for '--#{key}'; got #{value}")
       Set.new File.read(value).split(/\s+/)
     end
 
@@ -78,14 +78,12 @@ module Masamune::Actions
     end
     module_function :reset_module!
 
-    private
-
     included do |base|
       base.extend ClassMethods
       base.class_eval do
-        class_option :sources, :desc => 'File of data sources to process'
-        class_option :targets, :desc => 'File of data targets to process'
-        class_option :resolve, :type => :boolean, :desc => 'Recursively resolve data dependencies', :default => true
+        class_option :sources, desc: 'File of data sources to process'
+        class_option :targets, desc: 'File of data targets to process'
+        class_option :resolve, type: :boolean, desc: 'Recursively resolve data dependencies', default: true
       end
 
       base.after_initialize(:final) do |thor, options|
@@ -98,12 +96,13 @@ module Masamune::Actions
       end if defined?(base.after_initialize)
     end
 
+    # rubocop:disable Style/ClassVars
     module ClassMethods
       def skip
         initialize_module!
         @@namespaces << namespace
-        @@sources << {skip: true}
-        @@targets << {skip: true}
+        @@sources << { skip: true }
+        @@targets << { skip: true }
       end
 
       def source(source_options = {})
@@ -151,5 +150,6 @@ module Masamune::Actions
         true
       end
     end
+    # rubocop:enable Style/ClassVars
   end
 end

data/lib/masamune/actions/date_parse.rb

@@ -30,20 +30,18 @@ module Masamune::Actions
       value = options[key]
       Chronic.parse(value).tap do |datetime_value|
         logger.debug("Using '#{datetime_value}' for --#{key}") if value != datetime_value
-      end or raise Thor::MalformattedArgumentError, "Expected date time value for '--#{key}'; got #{value}"
+      end || raise(Thor::MalformattedArgumentError, "Expected date time value for '--#{key}'; got #{value}")
     end
 
-    private
-
     included do |base|
       base.class_eval do
         attr_accessor :start_value
         attr_accessor :stop_value
         attr_accessor :exact_value
 
-        class_option :start, :aliases => '-a', :desc => 'Start time'
-        class_option :stop, :aliases => '-b', :desc => 'Stop time'
-        class_option :at, :desc => 'Exact time'
+        class_option :start, aliases: '-a', desc: 'Start time'
+        class_option :stop, aliases: '-b', desc: 'Stop time'
+        class_option :at, desc: 'Exact time'
 
         no_tasks do
           def start_date

data/lib/masamune/actions/execute.rb

@@ -22,7 +22,7 @@
 
 module Masamune::Actions
   module Execute
-    def execute(*args, &block)
+    def execute(*args)
       opts = args.last.is_a?(Hash) ? args.pop : {}
       opts = opts.to_hash.symbolize_keys
 
@@ -52,13 +52,13 @@ module Masamune::Actions
 
       klass.class_eval do
         define_method(:handle_stdout) do |line, line_no|
-          block.call(line, line_no)
+          yield(line, line_no)
         end
       end if block_given?
 
       command = klass.new(self)
       command = Masamune::Commands::RetryWithBackoff.new(command, opts)
-      command = Masamune::Commands::Shell.new(command, {fail_fast: false}.merge(opts))
+      command = Masamune::Commands::Shell.new(command, { fail_fast: false }.merge(opts))
       opts.fetch(:interactive, false) ? command.replace(opts) : command.execute
     end
   end

data/lib/masamune/actions/filesystem.rb

@@ -28,7 +28,7 @@ module Masamune::Actions
       def filesystem
         defined?(environment) ? environment.filesystem : Masamune.environment.filesystem
       end
-      alias :fs :filesystem
+      alias fs filesystem
     end
     include ClassMethods
   end

data/lib/masamune/actions/hadoop_filesystem.rb

@@ -26,8 +26,8 @@ module Masamune::Actions
       opts = args.last.is_a?(Hash) ? args.pop : {}
       opts = opts.to_hash.symbolize_keys
       opts.reverse_merge!(fail_fast: false)
-      opts.merge!(extra: Array.wrap(args))
-      opts.merge!(block: block.to_proc) if block_given?
+      opts[:extra] = Array.wrap(args)
+      opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::HadoopFilesystem.new(environment, opts)
       command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hadoop_filesystem.slice(:retries, :backoff).merge(opts))

data/lib/masamune/actions/hive.rb

@@ -30,7 +30,7 @@ module Masamune::Actions
 
     def hive(opts = {}, &block)
       opts = opts.to_hash.symbolize_keys
-      opts.merge!(block: block.to_proc) if block_given?
+      opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::Hive.new(environment, opts)
       command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.aws_emr[:cluster_id]
@@ -40,12 +40,12 @@ module Masamune::Actions
       command.interactive? ? command.replace : command.execute
     end
 
-    # TODO warn or error if database is not defined
+    # TODO: warn or error if database is not defined
     def create_hive_database_if_not_exists
       return if configuration.hive[:database] == 'default'
       sql = []
-      sql << %Q(CREATE DATABASE IF NOT EXISTS #{configuration.hive[:database]})
-      sql << %Q(LOCATION "#{configuration.hive[:location]}") if configuration.hive[:location]
+      sql << %(CREATE DATABASE IF NOT EXISTS #{configuration.hive[:database]})
+      sql << %(LOCATION "#{configuration.hive[:location]}") if configuration.hive[:location]
       hive(exec: sql.join(' ') + ';', database: nil)
     end
 
@@ -54,7 +54,7 @@ module Masamune::Actions
       hive(file: transform.to_file)
     rescue => e
       logger.error(e)
-      logger.error("Could not load schema")
+      logger.error('Could not load schema')
       logger.error("\n" + transform.to_s)
       exit
     end

data/lib/masamune/actions/invoke_parallel.rb

@@ -31,7 +31,7 @@ module Masamune::Actions
     include Masamune::Actions::Execute
 
     included do |base|
-      base.class_option :max_tasks, aliases: '-p', :type => :numeric, :desc => 'Maximum number of tasks to execute in parallel', :default => 4
+      base.class_option :max_tasks, aliases: '-p', type: :numeric, desc: 'Maximum number of tasks to execute in parallel', default: 4
     end
 
     def invoke_parallel(*task_group)
@@ -42,7 +42,7 @@ module Masamune::Actions
       Parallel.each(task_group, in_processes: max_tasks) do |task_name|
         begin
           execute(thor_wrapper, task_name, *task_args(opts), interactive: true, detach: false)
-        rescue SystemExit
+        rescue SystemExit # rubocop:disable Lint/HandleExceptions
         end
       end
     end
@@ -60,13 +60,13 @@ module Masamune::Actions
     end
 
     def task_args(opts = {})
-      opts.map do |k,v|
+      opts.map do |k, v|
         case v
         when true
-          "--#{k.to_s.gsub('_', '-')}"
+          "--#{k.to_s.tr('_', '-')}"
         when false
         else
-          ["--#{k.to_s.gsub('_', '-')}", v]
+          ["--#{k.to_s.tr('_', '-')}", v]
         end
       end.flatten.compact
     end

data/lib/masamune/actions/postgres.rb

@@ -32,7 +32,7 @@ module Masamune::Actions
 
     def postgres(opts = {}, &block)
       opts = opts.to_hash.symbolize_keys
-      opts.merge!(block: block.to_proc) if block_given?
+      opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::Postgres.new(environment, opts)
       command = Masamune::Commands::RetryWithBackoff.new(command, configuration.postgres.slice(:retries, :backoff).merge(opts))
@@ -42,9 +42,9 @@ module Masamune::Actions
     end
 
     def create_postgres_database_if_not_exists
-      unless postgres_helper.database_exists?
-        postgres_admin(action: :create, database: configuration.postgres[:database], safe: true)
-      end if configuration.postgres.has_key?(:database)
+      if configuration.postgres.key?(:database)
+        postgres_admin(action: :create, database: configuration.postgres[:database], safe: true) unless postgres_helper.database_exists?
+      end
     end
 
     def load_postgres_setup_files
@@ -52,7 +52,7 @@ module Masamune::Actions
         configuration.with_quiet do
           postgres(file: file, retries: 0)
         end
-      end if configuration.postgres.has_key?(:setup_files)
+      end if configuration.postgres.key?(:setup_files)
     end
 
     def load_postgres_schema
@@ -60,7 +60,7 @@ module Masamune::Actions
       postgres(file: transform.to_file, retries: 0)
     rescue => e
       logger.error(e)
-      logger.error("Could not load schema")
+      logger.error('Could not load schema')
       logger.error("\n" + transform.to_s)
       exit
     end

data/lib/masamune/actions/postgres_admin.rb

@@ -22,7 +22,7 @@
 
 module Masamune::Actions
   module PostgresAdmin
-    def postgres_admin(opts = {}, &block)
+    def postgres_admin(opts = {})
       opts = opts.to_hash.symbolize_keys
 
       command = Masamune::Commands::PostgresAdmin.new(environment, opts)

data/lib/masamune/actions/s3cmd.rb

@@ -29,8 +29,8 @@ module Masamune::Actions
     def s3cmd(*args, &block)
       opts = args.last.is_a?(Hash) ? args.pop : {}
       opts = opts.to_hash.symbolize_keys
-      opts.merge!(extra: Array.wrap(args))
-      opts.merge!(block: block.to_proc) if block_given?
+      opts[:extra] = Array.wrap(args)
+      opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::S3Cmd.new(environment, opts)
       command = Masamune::Commands::RetryWithBackoff.new(command, configuration.s3cmd.slice(:retries, :backoff).merge(opts))
@@ -40,7 +40,7 @@ module Masamune::Actions
     end
 
     def s3_sync(src, dst)
-      s3cmd('sync', s3b(src), s3b(dst, :dir => true))
+      s3cmd('sync', s3b(src), s3b(dst, dir: true))
     end
   end
 end

data/lib/masamune/actions/transform.rb

@@ -35,7 +35,7 @@ module Masamune::Actions
 
     include Masamune::Actions::Postgres
 
-    # FIXME should eventually be able to include Transform directly instead of through wrapper
+    # FIXME: should eventually be able to include Transform directly instead of through wrapper
     class Wrapper
       extend Masamune::Transform::LoadDimension
       extend Masamune::Transform::ConsolidateDimension
@@ -78,20 +78,20 @@ module Masamune::Actions
     private
 
     def optional_apply_map(source_files, source, target, &block)
-      if source.respond_to?(:map) and map = source.map(to: target)
+      if source.respond_to?(:map) && (map = source.map(to: target))
         apply_map(map, source_files, source, target, &block)
       else
         yield source_files, source
       end
     end
 
-    def apply_map(map, source_files, source, target)
+    def apply_map(map, source_files, source, _target)
       Tempfile.open('masamune_transform') do |output|
         begin
           FileUtils.chmod(FILE_MODE, output.path)
           result = map.apply(source_files, output)
           result.debug = map.debug
-          logger.debug(File.read(output)) if (source.debug || result.debug)
+          logger.debug(File.read(output)) if source.debug || result.debug
           yield output, result
         ensure
           output.unlink

data/lib/masamune/after_initialize_callbacks.rb

@@ -31,18 +31,18 @@ module Masamune
       default:   0,
       later:   -10,
       final:   -20
-    }
+    }.freeze
 
     module ClassMethods
       # Callbacks registered with the highest priority are executed first, ties are broken by callback registration order
       def after_initialize(priority = :default, &block)
-        @after_initialize ||= Hash.new { |h,k| h[k] = [] }
+        @after_initialize ||= Hash.new { |h, k| h[k] = [] }
         @after_initialize[PRIORITY.fetch(priority, 0)] << block
       end
 
       def after_initialize_invoke(*a)
-        @after_initialize ||= Hash.new { |h,k| h[k] = [] }
-        @after_initialize.sort.reverse.each { |p, x| x.each { |y| y.call(*a) } }
+        @after_initialize ||= Hash.new { |h, k| h[k] = [] }
+        @after_initialize.sort.reverse.each { |_p, x| x.each { |y| y.call(*a) } }
       end
     end
 

data/lib/masamune/cached_filesystem.rb

@@ -81,7 +81,7 @@ module Masamune
       end if depth == 0
     end
 
-    # FIXME cache eviction policy can be more precise
+    # FIXME: cache eviction policy can be more precise
     [:touch!, :mkdir!, :copy_file_to_file, :copy_file_to_dir, :copy_dir, :remove_file, :remove_dir, :move_file_to_file, :move_file_to_dir, :move_dir, :write].each do |method|
       define_method(method) do |*args|
         clear!
@@ -89,8 +89,6 @@ module Masamune
       end
     end
 
-    private
-
     class PathCache
       def initialize(filesystem)
         @filesystem = filesystem
@@ -102,25 +100,23 @@ module Masamune
         return if @filesystem.root_path?(path)
         put(File.join(@filesystem.dirname(path), '.'), OpenStruct.new(name: @filesystem.dirname(path)))
         paths = path_split(path)
-        elems = paths.reverse.inject(entry) { |a, n| { n => a } }
+        elems = paths.reverse.inject(entry) { |a, e| { e => a } }
         @cache.deep_merge!(elems)
       end
 
       def get(path)
         return unless path
         paths = path_split(path)
-        elem = paths.inject(@cache) { |level, path| level.is_a?(Hash) ? level.fetch(path, {}) : level }
+        elem = paths.inject(@cache) { |a, e| a.is_a?(Hash) ? a.fetch(e, {}) : a }
         emit(elem)
       rescue KeyError
         EMPTY_SET
       end
 
       def any?(path)
-        if elem = get(path)
-          elem.any? { |entry| entry.name.start_with?(path) }
-        else
-          false
-        end
+        elem = get(path)
+        return false unless elem
+        elem.any? { |entry| entry.name.start_with?(path) }
       end
 
       private
@@ -142,7 +138,7 @@ module Masamune
       end
 
       def path_split(path)
-        path.split('/').reject { |x| x.blank? }
+        path.split('/').reject(&:blank?)
       end
     end
   end