masamune 0.17.15 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ecd8682fd1fc01a49affd8d28d3e299c601c75c
-  data.tar.gz: 45f3801d782a6cb2d2dc07e3a72a2f3286925bdc
+  metadata.gz: 78cc6a05c7d3d967eb2f5f66a89c9b2eb4d22bca
+  data.tar.gz: 7c2ae67465453c5064819aa16338ed2b1b73a656
 SHA512:
-  metadata.gz: 4f8ee35378be875d5e37154757a471d09f5364375a9330b105744ed2093ec4e1280b0ae1dfe0118b1b656bb18fc797353b438532aa57d97718420b8c72ee1406
-  data.tar.gz: ec1c92befce218f594d0688f4e801f80554f8470895b8dc616e6d4013b93d52cdc6b3005caec0d6ccc9173447cd43cfea12756c74d1ad4e3c16f50e55ef94cb9
+  metadata.gz: 1b4b73612c75ca6e73f4ecc49b68d034cae34acbfa4359f5588ab51dcfea2878e7890988fa5abca21d515b90d048005ec4a5282c6f306117c2e430e44b2061a8
+  data.tar.gz: a5d0c3525357265e77f5fdc09bcf6a0f2af9e9c81262d4bfed1bb1d809c7fb407704c7f68db2ad9ccb281d5c58ba075aa38c2c4f31fc45f3c6ed4def9cdf63e7

data/lib/masamune/actions/aws_emr.rb

@@ -28,14 +28,14 @@ module Masamune::Actions
       opts = opts.to_hash.symbolize_keys
 
       command = Masamune::Commands::AwsEmr.new(environment, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.aws_emr.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.aws_emr.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute
     end
 
     def validate_cluster_id!
-      cluster_id = configuration.aws_emr[:cluster_id]
+      cluster_id = configuration.commands.aws_emr[:cluster_id]
       raise ::Thor::RequiredArgumentMissingError, "No value provided for required options '--cluster-id'" unless cluster_id
       raise ::Thor::RequiredArgumentMissingError, %(AWS EMR cluster '#{cluster_id}' does not exist) unless aws_emr(action: 'describe-cluster', cluster_id: cluster_id, fail_fast: false).success?
     end
@@ -43,9 +43,9 @@ module Masamune::Actions
     included do |base|
       base.class_option :cluster_id, desc: 'AWS EMR cluster_id ID (Hint: `masamune-emr-aws list-clusters`)' if defined?(base.class_option)
       base.after_initialize(:early) do |thor, options|
-        next unless thor.configuration.aws_emr.any?
-        next unless thor.configuration.aws_emr.fetch(:enabled, true)
-        thor.configuration.aws_emr[:cluster_id] = options[:cluster_id] if options[:cluster_id]
+        next unless thor.configuration.commands.aws_emr.any?
+        next unless thor.configuration.commands.aws_emr.fetch(:enabled, true)
+        thor.configuration.commands.aws_emr[:cluster_id] = options[:cluster_id] if options[:cluster_id]
         next unless options[:initialize]
         thor.validate_cluster_id!
       end if defined?(base.after_initialize)

data/lib/masamune/actions/hadoop_filesystem.rb

@@ -30,7 +30,7 @@ module Masamune::Actions
       opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::HadoopFilesystem.new(environment, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hadoop_filesystem.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hadoop_filesystem.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute

data/lib/masamune/actions/hadoop_streaming.rb

@@ -26,8 +26,8 @@ module Masamune::Actions
       opts = opts.to_hash.symbolize_keys
 
       command = Masamune::Commands::HadoopStreaming.new(environment, aws_emr_options(opts))
-      command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.aws_emr[:cluster_id]
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hadoop_streaming.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.commands.aws_emr[:cluster_id]
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hadoop_streaming.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute
@@ -36,7 +36,7 @@ module Masamune::Actions
     private
 
     def aws_emr_options(opts = {})
-      return opts unless configuration.aws_emr[:cluster_id]
+      return opts unless configuration.commands.aws_emr[:cluster_id]
       opts.merge(quote: true, upload: false)
     end
   end

data/lib/masamune/actions/hive.rb

@@ -33,8 +33,8 @@ module Masamune::Actions
       opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::Hive.new(environment, opts)
-      command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.aws_emr[:cluster_id]
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hive.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.commands.aws_emr[:cluster_id]
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.hive.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute
@@ -42,10 +42,10 @@ module Masamune::Actions
 
     # TODO: warn or error if database is not defined
     def create_hive_database_if_not_exists
-      return if configuration.hive[:database] == 'default'
+      return if configuration.commands.hive[:database] == 'default'
       sql = []
-      sql << %(CREATE DATABASE IF NOT EXISTS #{configuration.hive[:database]})
-      sql << %(LOCATION "#{configuration.hive[:location]}") if configuration.hive[:location]
+      sql << %(CREATE DATABASE IF NOT EXISTS #{configuration.commands.hive[:database]})
+      sql << %(LOCATION "#{configuration.commands.hive[:location]}") if configuration.commands.hive[:location]
       hive(exec: sql.join(' ') + ';', database: nil)
     end
 

data/lib/masamune/actions/postgres.rb

@@ -35,24 +35,24 @@ module Masamune::Actions
       opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::Postgres.new(environment, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.postgres.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.postgres.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute
     end
 
     def create_postgres_database_if_not_exists
-      if configuration.postgres.key?(:database)
-        postgres_admin(action: :create, database: configuration.postgres[:database], safe: true) unless postgres_helper.database_exists?
+      if configuration.commands.postgres.key?(:database)
+        postgres_admin(action: :create, database: configuration.commands.postgres[:database], safe: true) unless postgres_helper.database_exists?
       end
     end
 
     def load_postgres_setup_files
-      configuration.postgres[:setup_files].each do |file|
+      configuration.commands.postgres[:setup_files].each do |file|
         configuration.with_quiet do
           postgres(file: file, retries: 0)
         end
-      end if configuration.postgres.key?(:setup_files)
+      end if configuration.commands.postgres.key?(:setup_files)
     end
 
     def load_postgres_schema

data/lib/masamune/actions/s3cmd.rb

@@ -33,7 +33,7 @@ module Masamune::Actions
       opts[:block] = block.to_proc if block_given?
 
       command = Masamune::Commands::S3Cmd.new(environment, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.s3cmd.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.commands.s3cmd.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute

data/lib/masamune/cached_filesystem.rb

@@ -78,7 +78,7 @@ module Masamune
         next if entry.name == dirname
         next unless entry.name =~ file_regexp
         yield entry
-      end if depth == 0
+      end if depth.zero?
     end
 
     # FIXME: cache eviction policy can be more precise

data/lib/masamune/commands/aws_emr.rb

@@ -29,19 +29,19 @@ module Masamune::Commands
     include Masamune::Actions::Execute
 
     DEFAULT_ATTRIBUTES =
-      {
-        path: 'aws',
-        extra: [],
-        config_file: nil,
-        action: nil,
-        cluster_id: nil,
-        interactive: false
-      }.freeze
+    {
+      path: 'aws',
+      extra: [],
+      config_file: nil,
+      action: nil,
+      cluster_id: nil,
+      interactive: false
+    }.with_indifferent_access.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
       @delegate = delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.aws_emr).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.aws_emr).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
     end
@@ -104,7 +104,7 @@ module Masamune::Commands
     end
 
     def handle_stdout(line, line_no)
-      if line_no == 0 && line.start_with?('ssh') && @delegate.respond_to?(:handle_stderr)
+      if line_no.zero? && line.start_with?('ssh') && @delegate.respond_to?(:handle_stderr)
         @delegate.handle_stderr(line, line_no)
       elsif @delegate.respond_to?(:handle_stdout)
         @delegate.handle_stdout(line, line_no)
@@ -118,7 +118,7 @@ module Masamune::Commands
     end
 
     def action_options
-      configuration.aws_emr.fetch(action.underscore.to_sym, {}).with_indifferent_access.fetch(:options, {}).reject { |key, _| @extra.include?(key.to_s) }
+      configuration.commands.aws_emr.fetch(action.underscore.to_sym, {}).with_indifferent_access.fetch(:options, {}).reject { |key, _| @extra.include?(key.to_s) }
     end
 
     def ssh_command?

data/lib/masamune/commands/hadoop_filesystem.rb

@@ -25,17 +25,17 @@ require 'delegate'
 module Masamune::Commands
   class HadoopFilesystem < SimpleDelegator
     DEFAULT_ATTRIBUTES =
-      {
-        path: 'hadoop',
-        options: [],
-        extra: [],
-        block: nil,
-        print: false
-      }.freeze
+    {
+      path: 'hadoop',
+      options: [],
+      extra: [],
+      block: nil,
+      print: false
+    }.with_indifferent_access.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.hadoop_filesystem).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.hadoop_filesystem).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
     end

data/lib/masamune/commands/hadoop_streaming.rb

@@ -49,13 +49,13 @@ module Masamune::Commands
       extra: [],
       upload: true,
       quote: false
-    }.freeze
+    }.with_indifferent_access.freeze
 
     attr_reader :input
 
     def initialize(delegate, attrs = {})
       super delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.hadoop_streaming).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.hadoop_streaming).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
       @input = Array.wrap(@input)

data/lib/masamune/commands/hive.rb

@@ -33,28 +33,28 @@ module Masamune::Commands
     PROMPT = 'hive>'.freeze
 
     DEFAULT_ATTRIBUTES =
-      {
-        path: 'hive',
-        options: [],
-        database: 'default',
-        setup_files: [],
-        schema_files: [],
-        file: nil,
-        exec: nil,
-        output: nil,
-        print: false,
-        block: nil,
-        variables: {},
-        buffer: nil,
-        service: false,
-        delimiter: "\001",
-        csv: false,
-        debug: false
-      }.freeze
+    {
+      path: 'hive',
+      options: [],
+      database: 'default',
+      setup_files: [],
+      schema_files: [],
+      file: nil,
+      exec: nil,
+      output: nil,
+      print: false,
+      block: nil,
+      variables: {},
+      buffer: nil,
+      service: false,
+      delimiter: "\001",
+      csv: false,
+      debug: false
+    }.with_indifferent_access.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.hive).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.hive).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
       raise ArgumentError, 'Cannot specify both file and exec' if @file && @exec

data/lib/masamune/commands/postgres.rb

@@ -31,28 +31,28 @@ module Masamune::Commands
     include Masamune::Commands::PostgresCommon
 
     DEFAULT_ATTRIBUTES =
-      {
-        path: 'psql',
-        options: [],
-        hostname: 'localhost',
-        database: 'postgres',
-        username: 'postgres',
-        pgpass_file: nil,
-        file: nil,
-        exec: nil,
-        input: nil,
-        output: nil,
-        print: false,
-        block: nil,
-        csv: false,
-        variables: {},
-        tuple_output: false,
-        debug: false
-      }.freeze
+    {
+      path: 'psql',
+      options: [],
+      hostname: 'localhost',
+      database: 'postgres',
+      username: 'postgres',
+      pgpass_file: nil,
+      file: nil,
+      exec: nil,
+      input: nil,
+      output: nil,
+      print: false,
+      block: nil,
+      csv: false,
+      variables: {},
+      tuple_output: false,
+      debug: false
+    }.with_indifferent_access.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.postgres).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.postgres).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
       raise ArgumentError, 'Cannot specify both file and exec' if @file && @exec

data/lib/masamune/commands/postgres_admin.rb

@@ -29,22 +29,22 @@ module Masamune::Commands
     include Masamune::Commands::PostgresCommon
 
     DEFAULT_ATTRIBUTES =
-      {
-        create_db_path: 'createdb',
-        drop_db_path: 'dropdb',
-        pg_dump_path: 'pg_dump',
-        options: [],
-        hostname: 'localhost',
-        username: 'postgres',
-        pgpass_file: nil,
-        action: nil,
-        database: nil,
-        output: nil
-      }.freeze
+    {
+      create_db_path: 'createdb',
+      drop_db_path: 'dropdb',
+      pg_dump_path: 'pg_dump',
+      options: [],
+      hostname: 'localhost',
+      username: 'postgres',
+      pgpass_file: nil,
+      action: nil,
+      database: nil,
+      output: nil
+    }.with_indifferent_access.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.postgres).merge(configuration.postgres_admin).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.postgres).merge(configuration.commands.postgres_admin).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
     end

data/lib/masamune/commands/s3cmd.rb

@@ -25,16 +25,16 @@ require 'delegate'
 module Masamune::Commands
   class S3Cmd < SimpleDelegator
     DEFAULT_ATTRIBUTES =
-      {
-        path: 's3cmd',
-        options: [],
-        extra: [],
-        block: nil
-      }.freeze
+    {
+      path: 's3cmd',
+      options: [],
+      extra: [],
+      block: nil
+    }.with_indifferent_access.freeze
 
     def initialize(delegate, attrs = {})
       super delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.s3cmd).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.commands.s3cmd).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
     end

data/lib/masamune/configuration.rb

@@ -26,43 +26,45 @@ require 'tilt/erb'
 require 'pp'
 
 require 'active_support/core_ext/hash'
+require 'hashie'
 
 require 'masamune/has_environment'
 
-class Masamune::Configuration
+class Masamune::Configuration < Hashie::Dash
   extend Forwardable
-  include Masamune::HasEnvironment
 
-  attr_accessor :quiet
-  attr_accessor :verbose
-  attr_accessor :debug
-  attr_accessor :dry_run
-  attr_accessor :lock
-  attr_accessor :retries
-  attr_accessor :backoff
-  attr_accessor :params
-
-  COMMANDS = %w(aws_emr hive hadoop_streaming hadoop_filesystem s3cmd postgres postgres_admin).freeze
-  COMMANDS.each do |command|
-    attr_accessor command
-    define_method(command) do
-      instance_variable_get("@#{command}").symbolize_keys!
+  include Hashie::Extensions::MergeInitializer
+  include Hashie::Extensions::IndifferentAccess
+
+  class << self
+    attr_writer :default_config_file
+
+    def default_config_file
+      @default_config_file ||= File.join(File.expand_path('../../../', __FILE__), 'config', 'masamune.yml.erb')
+    end
+
+    def default_commands
+      @default_commands ||= %i(aws_emr hive hadoop_streaming hadoop_filesystem s3cmd postgres postgres_admin)
     end
   end
 
-  def initialize(environment)
-    self.environment = environment
-    self.quiet    = false
-    self.verbose  = false
-    self.debug    = false
-    self.dry_run  = false
-    self.lock     = nil
-    self.retries  = 3
-    self.backoff  = 5
-    self.params   = HashWithIndifferentAccess.new
-
-    COMMANDS.each do |command|
-      instance_variable_set("@#{command}", {})
+  property :environment
+  include Masamune::HasEnvironment
+
+  property :quiet, default: false
+  property :verbose, default: false
+  property :debug, default: false
+  property :dry_run, default: false
+  property :lock
+  property :retries, default: 3
+  property :backoff, default: 5
+  property :params, default: Hashie::Mash.new
+  property :commands, default: Hashie::Mash.new { |h, k| h[k] = Hashie::Mash.new }
+
+  def initialize(*a)
+    super
+    self.class.default_commands.each do |command|
+      commands[command] = Hashie::Mash.new
     end
   end
 
@@ -70,8 +72,8 @@ class Masamune::Configuration
     @load_once ||= begin
       config_file = filesystem.eval_path(path)
       load_yaml_erb_file(config_file).each_pair do |command, value|
-        if COMMANDS.include?(command)
-          send("#{command}=", value)
+        if command == 'commands'
+          commands.merge!(value)
         elsif command == 'paths'
           load_paths(value)
         elsif command == 'params'
@@ -80,8 +82,8 @@ class Masamune::Configuration
         end
       end
       logger.debug("Loaded configuration #{config_file}")
-      load_catalog(configuration.postgres.fetch(:schema_files, []) + configuration.hive.fetch(:schema_files, []))
-      true
+      load_catalog(configuration.commands.postgres.fetch(:schema_files, []) + configuration.commands.hive.fetch(:schema_files, []))
+      self
     end
   end
 
@@ -95,18 +97,8 @@ class Masamune::Configuration
     end
   end
 
-  def to_s
-    io = StringIO.new
-    rep = { 'path' => filesystem.paths }
-    COMMANDS.each do |command|
-      rep[command] = send(command)
-    end
-    PP.pp(rep, io)
-    io.string
-  end
-
   def debug=(debug)
-    @debug = debug
+    self[:debug] = debug
     environment.reload_logger!
   end
 
@@ -135,14 +127,6 @@ class Masamune::Configuration
     YAML.load(t.result(binding))
   end
 
-  class << self
-    attr_writer :default_config_file
-
-    def default_config_file
-      @default_config_file ||= File.join(File.expand_path('../../../', __FILE__), 'config', 'masamune.yml.erb')
-    end
-  end
-
   def default_config_file
     self.class.default_config_file
   end