masamune 0.11.0

data/lib/masamune/commands/postgres_admin.rb

@@ -0,0 +1,72 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+module Masamune::Commands
+  class PostgresAdmin < SimpleDelegator
+    include Masamune::Commands::PostgresCommon
+
+    DEFAULT_ATTRIBUTES =
+    {
+      :create_db_path => 'createdb',
+      :drop_db_path   => 'dropdb',
+      :options        => [],
+      :hostname       => 'localhost',
+      :username       => 'postgres',
+      :pgpass_file    => nil,
+      :action         => nil,
+      :database       => nil
+    }
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.postgres).merge(configuration.postgres_admin).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    def command_args
+      raise ArgumentError, ':database must be given' unless @database
+      args = []
+      args << command_path
+      args << '--host=%s' % @hostname if @hostname
+      args << '--username=%s' % @username if @username
+      args << '--no-password'
+      args << @database
+      args.flatten.compact
+    end
+
+    private
+
+    def command_path
+      case @action
+      when :create
+        [@create_db_path]
+      when :drop
+        [@drop_db_path, '--if-exists']
+      else
+        raise ArgumentError, ':action must be :create or :drop'
+      end
+    end
+  end
+end

data/lib/masamune/commands/postgres_common.rb

@@ -0,0 +1,33 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Commands
+  module PostgresCommon
+    def command_env
+      { 'PGOPTIONS'=> '--client-min-messages=warning' }.tap do |env|
+        if @pgpass_file && File.readable?(@pgpass_file)
+          env['PGPASSFILE'] = @pgpass_file
+        end
+      end
+    end
+  end
+end

data/lib/masamune/commands/retry_with_backoff.rb

@@ -0,0 +1,60 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+module Masamune::Commands
+  class RetryWithBackoff < SimpleDelegator
+    MAX_RETRY_EXIT_STATUS = 8
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      @delegate     = delegate
+      @retries      = attrs.fetch(:retries, configuration.retries)
+      @backoff      = attrs.fetch(:backoff, configuration.backoff)
+      @retry_count  = 0
+    end
+
+    def around_execute(&block)
+      begin
+        status = if @delegate.respond_to?(:around_execute)
+          @delegate.around_execute(&block)
+        else
+          block.call
+        end
+        raise "exited with code: #{status.exitstatus}" unless status.success?
+        status
+      rescue => e
+        logger.error(e.to_s)
+        sleep @backoff
+        @retry_count += 1
+        unless @retry_count > @retries
+          logger.debug("retrying (#{@retry_count}/#{@retries})")
+          retry
+        else
+          logger.debug("max retries (#{@retries}) attempted, bailing")
+          OpenStruct.new(:success? => false, :exitstatus => MAX_RETRY_EXIT_STATUS)
+        end
+      end
+    end
+  end
+end

data/lib/masamune/commands/s3cmd.rb

@@ -0,0 +1,70 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+module Masamune::Commands
+  class S3Cmd < SimpleDelegator
+    DEFAULT_ATTRIBUTES =
+    {
+      :path     => 's3cmd',
+      :options  => [],
+      :extra    => [],
+      :block    => nil
+    }
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.s3cmd).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    def command_args
+      args = []
+      args << @path
+      args << @options.map(&:to_a)
+      args << @extra
+      args.flatten
+    end
+
+    def handle_stdout(line, line_no)
+      @block.call(line) if @block
+    end
+
+    module ClassMethods
+      def s3n(file, options = {})
+        file.dup.tap do |out|
+          out.sub!(%r{\As3://}, 's3n://')
+          out.sub!(%r{/?\z}, '/') if options[:dir]
+        end
+      end
+
+      def s3b(file, options = {})
+        file.dup.tap do |out|
+          out.sub!(%r{\As3n://}, 's3://')
+          out.sub!(%r{/?\z}, '/') if options[:dir]
+        end
+      end
+    end
+  end
+end

data/lib/masamune/commands/shell.rb

@@ -0,0 +1,202 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'open3'
+require 'readline'
+require 'ostruct'
+require 'delegate'
+
+module Masamune::Commands
+  class Shell < SimpleDelegator
+    SIGINT_EXIT_STATUS = 130
+    PIPE_TIMEOUT = 10
+
+    attr_accessor :safe, :fail_fast
+
+    def initialize(delegate, opts = {})
+      super delegate
+      @delegate       = delegate
+      self.safe       = opts.fetch(:safe, false)
+      self.fail_fast  = opts.fetch(:fail_fast, true)
+      @stdout_line_no = 0
+      @stderr_line_no = 0
+    end
+
+    def replace
+      logger.debug('replace: ' + command_args.join(' '))
+      before_execute
+      around_execute do
+        pid = Process.fork
+        if pid
+          STDIN.close; STDOUT.close; STDERR.close
+          Process.waitpid(pid)
+          exit
+        else
+          exec(command_env, *command_args)
+        end
+      end
+    end
+
+    def before_execute
+      logger.debug(command_args)
+
+      if configuration.verbose
+        trace(command_args)
+      end
+
+      if @delegate.respond_to?(:before_execute)
+        @delegate.before_execute
+      end
+    end
+
+    def after_execute
+      if @delegate.respond_to?(:after_execute)
+        @delegate.after_execute
+      end
+    end
+
+    def around_execute(&block)
+      if configuration.no_op && !safe
+        return OpenStruct.new(:success? => true)
+      end
+
+      if @delegate.respond_to?(:around_execute)
+        @delegate.around_execute(&block)
+      else
+        block.call
+      end
+    end
+
+    def command_env
+      (@delegate.respond_to?(:command_env) ? @delegate.command_env : {}).merge('TZ' => 'UTC')
+    end
+
+    def command_args
+      if @delegate.respond_to?(:command_args)
+        @delegate.command_args
+      else
+        raise 'no command_args'
+      end
+    end
+
+    def command_bin
+      command_args.first
+    end
+
+    def execute
+      status = OpenStruct.new(:success? => false, :exitstatus => 1)
+
+      before_execute
+      status = around_execute do
+        execute_block
+      end
+      after_execute
+
+      unless status.success?
+        handle_failure(exit_code(status))
+      end
+      status
+    rescue Interrupt
+      handle_failure(SIGINT_EXIT_STATUS)
+    rescue SystemExit
+      handle_failure(exit_code(status))
+    end
+
+    def execute_block
+      Open3.popen3(command_env, *command_args) do |p_stdin, p_stdout, p_stderr, t_stdin|
+        p_stdin.wait_writable(PIPE_TIMEOUT) or raise "IO stdin not ready for write in #{PIPE_TIMEOUT}"
+
+        Thread.new {
+          if @delegate.respond_to?(:stdin)
+            @delegate.stdin.rewind
+            while line = @delegate.stdin.gets
+              trace(line.chomp)
+              p_stdin.puts line
+              p_stdin.flush
+            end
+          else
+            while !p_stdin.closed? do
+              input = Readline.readline('', true).strip
+              p_stdin.puts input
+              p_stdin.flush
+            end
+          end
+          p_stdin.close unless p_stdin.closed?
+        }
+
+        t_stderr = Thread.new {
+          while !p_stderr.eof?  do
+            handle_stderr(p_stderr)
+          end
+          p_stderr.close unless p_stderr.closed?
+        }
+
+        t_stdout = Thread.new {
+          while !p_stdout.eof?  do
+            handle_stdout(p_stdout)
+          end
+          p_stdout.close unless p_stdout.closed?
+        }
+
+        [t_stderr, t_stdout, t_stdin].compact.each { |t| t.join }
+        logger.debug(t_stdin.value)
+        t_stdin.value
+      end
+    end
+
+    def handle_stdout(io)
+      line = io.gets.chomp
+      return unless line
+      if @delegate.respond_to?(:handle_stdout)
+        @delegate.handle_stdout(line, @stdout_line_no)
+      else
+        logger.debug(line)
+      end
+      @stdout_line_no += 1
+    end
+
+    def handle_stderr(io)
+      line = io.gets.chomp
+      return unless line
+      if @delegate.respond_to?(:handle_stderr)
+        @delegate.handle_stderr(line, @stderr_line_no)
+      else
+        logger.debug(line)
+      end
+      @stderr_line_no += 1
+    end
+
+    def handle_failure(status)
+      if @delegate.respond_to?(:handle_failure)
+        @delegate.handle_failure(status)
+      end
+      raise "fail_fast: #{command_args.join(' ')}" if fail_fast
+    end
+
+    private
+
+    def exit_code(status, code = 1)
+      return code unless status
+      status.exitstatus
+    end
+  end
+end

data/lib/masamune/configuration.rb

@@ -0,0 +1,195 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+require 'yaml'
+require 'tilt/erb'
+require 'pp'
+
+require 'active_support/core_ext/hash'
+
+class Masamune::Configuration
+  extend Forwardable
+  include Masamune::HasEnvironment
+
+  attr_accessor :quiet
+  attr_accessor :verbose
+  attr_accessor :debug
+  attr_accessor :no_op
+  attr_accessor :dry_run
+  attr_accessor :lock
+  attr_accessor :retries
+  attr_accessor :backoff
+  attr_accessor :params
+
+  COMMANDS = %w(hive hadoop_streaming hadoop_filesystem elastic_mapreduce s3cmd postgres postgres_admin)
+  COMMANDS.each do |command|
+    attr_accessor command
+    define_method(command) do
+      instance_variable_get("@#{command}").symbolize_keys!
+    end
+  end
+
+  def initialize(environment)
+    self.environment   = environment
+    self.quiet    = false
+    self.verbose  = false
+    self.debug    = false
+    self.no_op    = false
+    self.dry_run  = false
+    self.lock     = nil
+    self.retries  = 3
+    self.backoff  = 5
+    self.params   = HashWithIndifferentAccess.new
+
+    @templates    = Hash.new { |h,k| h[k] = {} }
+
+    COMMANDS.each do |command|
+      instance_variable_set("@#{command}", {})
+    end
+  end
+
+  def load(file)
+    @load_once ||= begin
+      load_yaml_erb_file(file).each_pair do |command, value|
+        if COMMANDS.include?(command)
+          send("#{command}=", value)
+        elsif command == 'paths'
+          load_paths(value)
+        elsif command == 'params'
+          raise ArgumentError, 'params section must only contain key value pairs' unless value.is_a?(Hash)
+          self.params.merge! value
+        end
+      end
+      logger.debug("Loaded configuration #{file}")
+      load_catalog(configuration.postgres.fetch(:schema_files, []) + configuration.hive.fetch(:schema_files, []))
+      true
+    end
+  end
+
+  def load_catalog(paths = [])
+    paths.each do |path|
+      filesystem.glob_sort(path, order: :basename).each do |file|
+        configuration.with_quiet do
+          catalog.load(file)
+        end
+      end
+    end
+  end
+
+  def to_s
+    io = StringIO.new
+    rep = {"path" => filesystem.paths}
+    COMMANDS.each do |command|
+      rep[command] = send(command)
+    end
+    PP.pp(rep, io)
+    io.string
+  end
+
+  def debug=(debug)
+    @debug = debug
+    environment.reload_logger!
+  end
+
+  def bind_template(section, template, input_args = {})
+    free_command = load_template(section, template)[:command].split(/\s+/)
+    [].tap do |bind_command|
+      free_command.each do |free_expr|
+        if free_expr =~ /(%.*)/
+          free_param = $1
+          bind_command << free_expr.gsub!(free_param, bind_param(section, template, free_param, input_args))
+        elsif param = free_expr
+          bind_command << param
+        end
+      end
+    end
+  end
+
+  def as_options
+    opts = []
+    opts << '--quiet'   if quiet
+    opts << '--verbose' if verbose
+    opts << '--debug'   if debug
+    opts << '--no_op'   if no_op
+    opts << '--dry_run' if dry_run
+    opts
+  end
+
+  def_delegators :filesystem, :add_path, :get_path
+
+  def with_quiet(&block)
+    prev_quiet, self.quiet = quiet, true
+    yield
+  ensure
+    self.quiet = prev_quiet
+  end
+
+  def load_yaml_erb_file(file)
+    t = ERB.new(File.read(file))
+    t.filename = file
+    YAML.load(t.result(binding))
+  end
+
+  class << self
+    def default_config_file=(config_file)
+      @default_config_file = config_file
+    end
+
+    def default_config_file
+      @default_config_file ||= File.join(File.expand_path('../../../', __FILE__), 'config', 'masamune.yml.erb')
+    end
+  end
+
+  def default_config_file
+    self.class.default_config_file
+  end
+
+  private
+
+  def load_template(section, template_name)
+    @templates[section][template_name] ||= begin
+      raise ArgumentError, "no configuration section #{section}" unless COMMANDS.include?(section.to_s)
+      raise ArgumentError, 'no template_name' unless template_name
+      templates = send(section).fetch(:templates, {}).symbolize_keys!
+      template = templates[template_name.to_sym] or raise ArgumentError, "no template for #{template_name}"
+      template.symbolize_keys!
+      template.has_key?(:command) or raise ArgumentError, "no command for template #{template_name}"
+      template[:default] ||= {}
+      template[:default] = Hash[ template[:default].collect { |key,val| [key.to_sym, val.to_s] } ]
+      template
+    end
+  end
+
+  def bind_param(section, template, free_param, input_args = {})
+    default = load_template(section, template).fetch(:default, {})
+    param = free_param[/(?<=%).*/].to_sym
+    default.merge(input_args.symbolize_keys || {})[param] or raise ArgumentError, "no param for #{free_param}"
+  end
+
+  def load_paths(paths)
+    paths.each do |value|
+      symbol, path, options = *value.to_a.flatten
+      add_path(symbol, path, options)
+    end
+  end
+end