masamune 0.17.12 → 0.17.13

data/lib/masamune/tasks/hive_thor.rb

@@ -29,26 +29,26 @@ module Masamune::Tasks
     include Masamune::Actions::AwsEmr
     include Masamune::Actions::Hive
 
-    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # FIXME: need to add an unnecessary namespace until this issue is fixed:
     # https://github.com/wycats/thor/pull/247
     namespace :hive
     skip_lock!
 
     desc 'hive', 'Launch a Hive session'
-    method_option :file, :aliases => '-f', :desc => 'SQL from files'
-    method_option :exec, :aliases => '-e', :desc => 'SQL from command line'
-    method_option :output, :aliases => '-o', :desc => 'Save SQL output to file'
-    method_option :delimiter, :desc => 'Hive row format delimiter', :default => "\001"
-    method_option :csv, :type => :boolean, :desc => 'Report SQL output in CSV format', :default => false
-    method_option :variables, :aliases => '-X', :type => :hash, :desc => 'Variables to substitute in SQL', :default => {}
-    method_option :retry, :type => :boolean, :desc => 'Retry SQL query in event of failure', :default => false
-    method_option :service, :desc => 'Start as a service', :default => false
+    method_option :file, aliases: '-f', desc: 'SQL from files'
+    method_option :exec, aliases: '-e', desc: 'SQL from command line'
+    method_option :output, aliases: '-o', desc: 'Save SQL output to file'
+    method_option :delimiter, desc: 'Hive row format delimiter', default: "\001"
+    method_option :csv, type: :boolean, desc: 'Report SQL output in CSV format', default: false
+    method_option :variables, aliases: '-X', type: :hash, desc: 'Variables to substitute in SQL', default: {}
+    method_option :retry, type: :boolean, desc: 'Retry SQL query in event of failure', default: false
+    method_option :service, desc: 'Start as a service', default: false
     def hive_exec
       hive_options = options.dup.with_indifferent_access
-      hive_options.merge!(print: true)
-      hive_options.merge!(retries: 0) unless options[:retry]
-      hive_options.merge!(file: File.expand_path(options[:file])) if options[:file]
-      hive_options.merge!(output: File.expand_path(options[:output])) if options[:output]
+      hive_options[:print] = true
+      hive_options[:retries] = 0 unless options[:retry]
+      hive_options[:file] = File.expand_path(options[:file]) if options[:file]
+      hive_options[:output] = File.expand_path(options[:output]) if options[:output]
       hive(hive_options)
     end
     default_task :hive_exec

data/lib/masamune/tasks/postgres_thor.rb

@@ -28,21 +28,21 @@ module Masamune::Tasks
     include Masamune::Thor
     include Masamune::Actions::Postgres
 
-    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # FIXME: need to add an unnecessary namespace until this issue is fixed:
     # https://github.com/wycats/thor/pull/247
     namespace :postgres
     skip_lock!
 
     desc 'psql', 'Launch a Postgres session'
-    method_option :file, :aliases => '-f', :desc => 'SQL from files'
-    method_option :exec, :aliases => '-e', :desc => 'SQL from command line'
-    method_option :output, :aliases => '-o', :desc => 'Save SQL output to file'
-    method_option :csv, :type => :boolean, :desc => 'Report SQL output in CSV format', :default => false
-    method_option :retry, :type => :boolean, :desc => 'Retry SQL query in event of failure', :default => false
+    method_option :file, aliases: '-f', desc: 'SQL from files'
+    method_option :exec, aliases: '-e', desc: 'SQL from command line'
+    method_option :output, aliases: '-o', desc: 'Save SQL output to file'
+    method_option :csv, type: :boolean, desc: 'Report SQL output in CSV format', default: false
+    method_option :retry, type: :boolean, desc: 'Retry SQL query in event of failure', default: false
     def psql_exec
       postgres_options = options.dup.with_indifferent_access
-      postgres_options.merge!(print: true)
-      postgres_options.merge!(retries: 0) unless options[:retry]
+      postgres_options[:print] = true
+      postgres_options[:retries] = 0 unless options[:retry]
       postgres(postgres_options)
     end
     default_task :psql_exec

data/lib/masamune/tasks/shell_thor.rb

@@ -29,15 +29,15 @@ module Masamune::Tasks
     include Masamune::Thor
     include Masamune::Actions::DataFlow
 
-    # FIXME need to add an unnecessary namespace until this issue is fixed:
+    # FIXME: need to add an unnecessary namespace until this issue is fixed:
     # https://github.com/wycats/thor/pull/247
     namespace :shell
     skip_lock!
 
     desc 'shell', 'Launch an interactive shell'
     skip
-    method_option :prompt, :desc => 'Set shell prompt', :default => 'masamune'
-    class_option :start, :aliases => '-a', :desc => 'Start time', default: '1 month ago'
+    method_option :prompt, desc: 'Set shell prompt', default: 'masamune'
+    class_option :start, aliases: '-a', desc: 'Start time', default: '1 month ago'
     def shell_exec
       Pry.start self, prompt: proc { options[:prompt] + '> ' }
     end

data/lib/masamune/template.rb

@@ -37,14 +37,14 @@ module Masamune
 
     private
 
-    # TODO unify with resolve_path
+    # TODO: unify with resolve_path
     def resolve_file(partial_file)
       return partial_file if Pathname.new(partial_file).absolute?
       @paths.each do |path|
         file = File.expand_path(File.join(path, partial_file))
-        if File.exists?(file) && File.file?(file)
+        if File.exist?(file) && File.file?(file)
           @paths << File.dirname(file)
-          return file 
+          return file
         end
       end
       raise IOError, "File not found: #{partial_file}"
@@ -59,7 +59,7 @@ module Masamune
       end
 
       def render_to_string(template, parameters = {})
-        instance = self.new(File.dirname(template))
+        instance = new(File.dirname(template))
         combine instance.render(template, parameters)
       end
 

data/lib/masamune/thor.rb

@@ -41,15 +41,16 @@ module Masamune
       '/opt/etc/masamune/config.yml',
       '/opt/etc/masamune/config.yml.erb',
       "#{ENV['HOME']}/.masamune/config.yml"
-    ]
+    ].freeze
 
     module ExtraArguments
       def parse_extra(argv)
-        if i = argv.index('--')
+        i = argv.index('--')
+        if i
           if i > 0
-            [argv[0 .. i-1], argv[i+1..-1]]
+            [argv[0..i - 1], argv[i + 1..-1]]
           else
-            [[], argv[i+1..-1]]
+            [[], argv[i + 1..-1]]
           end
         else
           [argv, []]
@@ -70,14 +71,14 @@ module Masamune
         super
       rescue SignalException => e
         raise e unless %w(SIGHUP SIGTERM).include?(e.to_s)
-        instance.logger.debug("Exiting at user request on #{e.to_s}")
+        instance.logger.debug("Exiting at user request on #{e}")
         exit 0
       rescue ::Thor::MalformattedArgumentError, ::Thor::RequiredArgumentMissingError => e
         raise e
       rescue => e
         instance.logger.error("#{e.message} (#{e.class}) backtrace:")
         e.backtrace.each { |x| instance.logger.error(x) }
-        $stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
+        $stderr.puts "For complete debug log see: #{instance.log_file_name}"
         abort e.message
       end
     end
@@ -94,31 +95,31 @@ module Masamune
         attr_accessor :extra
 
         namespace :masamune
-        class_option :help, :type => :boolean, :aliases => '-h', :desc => 'Show help', :default => false
-        class_option :quiet, :type => :boolean, :aliases => '-q', :desc => 'Suppress all output', :default => false
-        class_option :verbose, :type => :boolean, :aliases => '-v', :desc => 'Print command execution information', :default => false
-        class_option :debug, :type => :boolean, :aliases => '-d', :desc => 'Print debugging information', :default => false
-        class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Do not execute commands that modify state', :default => false
-        class_option :config, :desc => 'Configuration file'
-        class_option :version, :desc => 'Print version and exit', :type => :boolean
-        class_option :lock, :desc => 'Optional job lock name', :type => :string
-        class_option :initialize, :aliases => '--init', :desc => 'Initialize configured data stores', :type => :boolean, :default => false
-        class_option :'--', :desc => 'Extra pass through arguments'
-        def initialize(_args = [], _options = {}, _config = {})
-          self.environment.parent = self
-          self.filesystem.environment = self
+        class_option :help, type: :boolean, aliases: '-h', desc: 'Show help', default: false
+        class_option :quiet, type: :boolean, aliases: '-q', desc: 'Suppress all output', default: false
+        class_option :verbose, type: :boolean, aliases: '-v', desc: 'Print command execution information', default: false
+        class_option :debug, type: :boolean, aliases: '-d', desc: 'Print debugging information', default: false
+        class_option :dry_run, type: :boolean, aliases: '-n', desc: 'Do not execute commands that modify state', default: false
+        class_option :config, desc: 'Configuration file'
+        class_option :version, desc: 'Print version and exit', type: :boolean
+        class_option :lock, desc: 'Optional job lock name', type: :string
+        class_option :initialize, aliases: '--init', desc: 'Initialize configured data stores', type: :boolean, default: false
+        class_option :'--', desc: 'Extra pass through arguments'
+        def initialize(thor_args = [], thor_options = {}, thor_config = {})
+          environment.parent = self
+          filesystem.environment = self
           self.current_namespace = self.class.namespace unless self.class.namespace == 'masamune'
-          self.current_task_name = _config[:current_command].try(:name)
+          self.current_task_name = thor_config[:current_command].try(:name)
           self.current_command_name = current_namespace ? current_namespace + ':' + current_task_name : current_task_name
           self.class.instance = self
 
           define_current_dir
 
-          if _options.is_a?(Array)
-            _options, self.extra = self.class.parse_extra(_options)
+          if thor_options.is_a?(Array)
+            thor_options, self.extra = self.class.parse_extra(thor_options)
           end
 
-          super _args, _options, _config
+          super thor_args, thor_options, thor_config
 
           if display_help?
             display_help
@@ -133,7 +134,7 @@ module Masamune
             begin
               config.load(config_file)
             rescue
-              raise $! if options[:debug]
+              raise $ERROR_INFO if options[:debug]
               raise ::Thor::MalformattedArgumentError, "Could not load file provided for '--config'"
             end
 
@@ -160,7 +161,7 @@ module Masamune
           end
 
           def top_level?
-            self.current_command_name == ARGV.first
+            current_command_name == ARGV.first
           end
 
           def invoke_command(command, *args)

data/lib/masamune/transform/common/denormalize_table.rb

@@ -38,14 +38,13 @@ module Masamune::Transform::Common
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       include Masamune::LastElement
 
       def select_columns(column_names)
         column_names.map do |column_name|
-          next unless column = dereference_column_name(column_name)
+          column = dereference_column_name(column_name)
+          next unless column
           if column.reference
             if column.reference.implicit || column.reference.degenerate
               "#{column.name} AS #{column.name}"
@@ -66,8 +65,8 @@ module Masamune::Transform::Common
       def join_conditions(column_names)
         {}.tap do |conditions|
           column_names.each do |column_name|
-            next unless column = dereference_column_name(column_name)
-            next unless column.reference
+            column = dereference_column_name(column_name)
+            next unless column && column.reference
             next if column.reference.degenerate
             adjacent_reference = references[column.reference.id]
             next unless adjacent_reference
@@ -80,7 +79,8 @@ module Masamune::Transform::Common
 
       def order_by_columns(column_names)
         column_names.map do |column_name|
-          next unless column = dereference_column_name(column_name)
+          column = dereference_column_name(column_name)
+          next unless column
           column.name
         end.compact
       end

data/lib/masamune/transform/define_table.rb

@@ -52,7 +52,7 @@ module Masamune::Transform
 
     def glob_to_regexp(input)
       if input.include?('*')
-        %r|\A#{Regexp.escape(input).gsub('\\*', '.*?')}|
+        /\A#{Regexp.escape(input).gsub('\\*', '.*?')}/
       else
         /\A#{Regexp.escape(input)}\z/
       end

data/lib/masamune/transform/hive/define_table.rb

@@ -34,8 +34,6 @@ module Masamune::Transform::Hive
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       def partition_by
         return unless partitions.any?

data/lib/masamune/transform/insert_reference_values.rb

@@ -29,7 +29,7 @@ module Masamune::Transform
       target.insert_references.each do |_, reference|
         operators << Operator.new(__method__, source: source, target: reference)
       end
-      Operator.new *operators
+      Operator.new(*operators)
     end
   end
 end

data/lib/masamune/transform/operator.rb

@@ -51,8 +51,8 @@ module Masamune::Transform
     private
 
     def template_eval(template)
-      return File.read(template) if File.exists?(template.to_s) && template.to_s !~ /erb\Z/
-      template_file = File.exists?(template.to_s) ? template : template_file(template)
+      return File.read(template) if File.exist?(template.to_s) && template.to_s !~ /erb\Z/
+      template_file = File.exist?(template.to_s) ? template : template_file(template)
       if template_helper(template)
         Masamune::Template.render_to_string(template_file, template_helper(template).new(@options).locals)
       else
@@ -63,6 +63,7 @@ module Masamune::Transform
     def template_helper(template_name)
       "Masamune::Transform::#{template_type.to_s.camelize}::#{template_name.to_s.camelize}".constantize
     rescue NameError
+      nil
     end
 
     def template_file(template_name)

data/lib/masamune/transform/postgres/bulk_upsert.rb

@@ -37,8 +37,6 @@ module Masamune::Transform::Postgres
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       include Masamune::LastElement
 
@@ -59,4 +57,3 @@ module Masamune::Transform::Postgres
     end
   end
 end
-

data/lib/masamune/transform/postgres/deduplicate_dimension.rb

@@ -35,10 +35,8 @@ module Masamune::Transform::Postgres
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
-      def insert_columns(source = nil)
+      def insert_columns(_source = nil)
         consolidated_columns.map { |_, column| column.name }
       end
 
@@ -61,10 +59,11 @@ module Masamune::Transform::Postgres
       def duplicate_value_conditions(window)
         [].tap do |result|
           consolidated_columns.map do |_, column|
+            result <<
             if column.null
-              result << "((LAG(#{column.name}) OVER #{window} = #{column.name}) OR (LAG(#{column.name}) OVER #{window} IS NULL AND #{column.name} IS NULL))"
+              "((LAG(#{column.name}) OVER #{window} = #{column.name}) OR (LAG(#{column.name}) OVER #{window} IS NULL AND #{column.name} IS NULL))"
             else
-              result << "(LAG(#{column.name}) OVER #{window} = #{column.name})"
+              "(LAG(#{column.name}) OVER #{window} = #{column.name})"
             end
           end
         end
@@ -93,4 +92,3 @@ module Masamune::Transform::Postgres
     end
   end
 end
-

data/lib/masamune/transform/postgres/define_table.rb

@@ -21,7 +21,7 @@
 #  THE SOFTWARE.
 
 module Masamune::Transform::Postgres
-  class DefineTable 
+  class DefineTable
     def initialize(options = {})
       @target       = options[:target]
       @files        = options[:files] || []
@@ -46,7 +46,7 @@ module Masamune::Transform::Postgres
     end
 
     def skip_indexes?
-      !!@skip_indexes
+      @skip_indexes == true
     end
 
     def define_types?

data/lib/masamune/transform/postgres/insert_reference_values.rb

@@ -39,8 +39,6 @@ module Masamune::Transform::Postgres
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       include Masamune::LastElement
 
@@ -60,10 +58,9 @@ module Masamune::Transform::Postgres
       method_with_last_element :insert_values
 
       def insert_constraints(source)
-        source.shared_columns(stage_table).reject { |column, _| column.null || column.default || column.adjacent.try(:default) }.map { |column, _| "#{column.name} IS NOT NULL"}
+        source.shared_columns(stage_table).reject { |column, _| column.null || column.default || column.adjacent.try(:default) }.map { |column, _| "#{column.name} IS NOT NULL" }
       end
       method_with_last_element :insert_constraints
     end
   end
 end
-

data/lib/masamune/transform/postgres/relabel_dimension.rb

@@ -23,7 +23,7 @@
 module Masamune::Transform::Postgres
   class RelabelDimension
     def initialize(options = {})
-      @target   = options[:target]
+      @target = options[:target]
     end
 
     def locals
@@ -34,8 +34,6 @@ module Masamune::Transform::Postgres
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       def window(*extra)
         (columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq

data/lib/masamune/transform/postgres/rollup_fact.rb

@@ -36,8 +36,6 @@ module Masamune::Transform::Postgres
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       include Masamune::LastElement
 
@@ -49,7 +47,7 @@ module Masamune::Transform::Postgres
           next if column.auto_reference
           values << column.name
         end
-        measures.each do |_ ,measure|
+        measures.each do |_, measure|
           values << measure.name
         end
         values << time_key.name
@@ -66,7 +64,7 @@ module Masamune::Transform::Postgres
           next if column.auto_reference
           values << column.qualified_name
         end
-        source.measures.each do |_ ,measure|
+        source.measures.each do |_, measure|
           values << measure.aggregate_value
         end
         values << calculated_time_key(source)
@@ -102,7 +100,7 @@ module Masamune::Transform::Postgres
       def calculated_date_key(source)
         case grain
         when :hourly, :daily
-          "#{source.date_column.qualified_name}"
+          source.date_column.qualified_name.to_s
         when :monthly
           "to_char(date_trunc('month',#{source.date_column.qualified_name}::text::date),'YYYYMMDD')::integer"
         end

data/lib/masamune/transform/postgres/snapshot_dimension.rb

@@ -36,12 +36,10 @@ module Masamune::Transform::Postgres
       TargetPresenter.new(@target)
     end
 
-    private
-
     class TargetPresenter < SimpleDelegator
       include Masamune::LastElement
 
-      def insert_columns(source = nil)
+      def insert_columns(_source = nil)
         consolidated_columns.map { |_, column| column.name }
       end
 
@@ -80,4 +78,3 @@ module Masamune::Transform::Postgres
     end
   end
 end
-