masamune 0.11.0

data/lib/masamune/commands/hadoop_streaming.rb

@@ -0,0 +1,116 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+require 'active_support/core_ext/array'
+
+module Masamune::Commands
+  class HadoopStreaming < SimpleDelegator
+    def self.default_hadoop_streaming_jar
+      @default_hadoop_streaming_jar ||=
+      case RUBY_PLATFORM
+      when /darwin/
+        Dir.glob('/usr/local/Cellar/hadoop/**/hadoop-streaming-*.jar').first
+      when /linux/
+        '/usr/lib/hadoop-mapreduce/hadoop-streaming.jar'
+      else
+        raise "unknown RUBY_PLATFORM=#{RUBY_PLATFORM}"
+      end
+    end
+
+    DEFAULT_ATTRIBUTES =
+    {
+      :path         => 'hadoop',
+      :options      => [],
+      :jar          => default_hadoop_streaming_jar,
+      :input        => [],
+      :output       => nil,
+      :mapper       => nil,
+      :reducer      => nil,
+      :extra        => [],
+      :upload       => true,
+      :quote        => false
+    }
+
+    attr_reader :input
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.hadoop_streaming).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+      @input = Array.wrap(@input)
+    end
+
+    # TODO ensure jar/ mapper/reduce exists, warn or remove if output exists
+    def command_args
+      args = []
+      args << @path
+      args << ['jar', @jar]
+      args << (@quote ? @extra.map { |arg| quote_arg(arg) } : @extra)
+      args << @options.map(&:to_a)
+      args << ['-input', *@input] if @input
+      args << ['-mapper', @mapper]
+      args << ['-file', @mapper] if @upload
+      args << ['-reducer', @reducer] if @reducer
+      args << ['-file', @reducer] if @upload && @reducer
+      args << ['-output', @output]
+      args.flatten
+    end
+
+    def before_execute
+      @input.reject! do |path|
+        if filesystem.exists?(path)
+          false
+        else
+          logger.debug("Removing missing input #{path} from hadoop_streaming command")
+          true
+        end
+      end
+      @input.map! do |path|
+        if path =~ /part_.*\Z/ || path =~ /\..*\Z/
+          path
+        elsif path =~ %r{/\Z}
+          path + '*'
+        else
+          path + '/*'
+        end
+      end
+      console("hadoop_streaming %s -> %s (%s/%s)" % [@input.join(' '), @output, @mapper, @reducer])
+    end
+
+    def around_execute(&block)
+      Dir.chdir(filesystem.path(:run_dir)) do
+        yield
+      end
+    end
+
+    private
+
+    # FIXME shell quoting is a separate concern
+    def quote_arg(arg)
+      out = arg.dup
+      out.gsub!(%q('\t'), %q('"'\\\\\\\\t'"'))
+      out
+    end
+  end
+end

data/lib/masamune/commands/hive.rb

@@ -0,0 +1,178 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+require 'csv'
+
+require 'masamune/string_format'
+require 'masamune/commands/shell'
+
+module Masamune::Commands
+  class Hive < SimpleDelegator
+    include Masamune::StringFormat
+
+    PROMPT = 'hive>'
+
+    DEFAULT_ATTRIBUTES =
+    {
+      :path           => 'hive',
+      :options        => [],
+      :database       => 'default',
+      :setup_files    => [],
+      :schema_files   => [],
+      :file           => nil,
+      :exec           => nil,
+      :input          => nil,
+      :output         => nil,
+      :print          => false,
+      :block          => nil,
+      :variables      => {},
+      :buffer         => nil,
+      :service        => false,
+      :delimiter      => "\001",
+      :csv            => false,
+      :debug          => false
+    }
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.hive).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    def stdin
+      if @input || @exec
+        @stdin ||= StringIO.new(strip_sql(@input || @exec))
+      end
+    end
+
+    def interactive?
+      !(@exec || @file)
+    end
+
+    def print?
+      @print
+    end
+
+    def command_args
+      args = []
+      args << @path
+      if @service
+        args << ['--service', 'hiveserver']
+      else
+        args << ['--database', @database] if @database && !@service
+        args << @options.map(&:to_a)
+        args << load_setup_files.map(&:to_a)
+        args << command_args_for_file if @file
+      end
+      args.flatten
+    end
+
+    def before_execute
+      if @file
+        console("hive with file #{@file}")
+      end
+
+      if @debug and output = @rendered_file || @file
+        logger.debug("#{output}:\n" + File.read(output))
+      end
+
+      if @exec
+        console("hive exec '#{strip_sql(@exec)}' #{'into ' + @output if @output}")
+      end
+
+      if @output
+        @buffer = Tempfile.new('masamune')
+      end
+    end
+
+    def around_execute(&block)
+      Dir.chdir(filesystem.path(:run_dir)) do
+        yield
+      end
+    end
+
+    def after_execute
+      return unless @buffer
+      @buffer.flush unless @buffer.closed?
+      @buffer.close unless @buffer.closed?
+      return unless @output
+
+      filesystem.move_file_to_file(@buffer.path, @output)
+    end
+
+    # FIXME use temporary tables with delimiters for CSV output format
+    def handle_stdout(line, line_no)
+      if line =~ /\A#{PROMPT}/
+        logger.debug(line)
+      else
+        @block.call(line) if @block
+
+        if @buffer
+          @buffer.puts(@csv ? line.split(@delimiter).map { |row| encode_row(row) }.to_csv : line)
+        else
+          console(line) if print?
+        end
+      end
+    end
+
+    def encode_row(row)
+      row unless row == 'NULL' || row == ''
+    end
+
+    def load_setup_files
+      files = []
+      @setup_files.each do |path|
+        filesystem.glob_sort(path, order: :basename).each do |file|
+          files << file
+        end
+      end
+      files.flatten.compact.map { |file| {'-i' => file} }
+    end
+
+    def command_args_for_file
+      @file =~ /\.erb\Z/ ? command_args_for_template : command_args_for_simple_file
+    end
+
+    def command_args_for_simple_file
+      filesystem.copy_file_to_file(@file, remote_file)
+      ['-f', remote_file].tap do |args|
+        @variables.each do |key, val|
+          args << ['-d', "#{key.to_s}=#{val.to_s}"]
+        end
+      end
+    end
+
+    def command_args_for_template
+      @rendered_file = Masamune::Template.render_to_file(@file, @variables)
+      filesystem.copy_file_to_file(@rendered_file, remote_file)
+      ['-f', remote_file]
+    end
+
+    private
+
+    def remote_file
+      @remote_file ||= File.join(filesystem.mktempdir!(:tmp_dir), filesystem.basename(@file)).gsub(/.erb\z/,'')
+    end
+  end
+end

data/lib/masamune/commands/interactive.rb

@@ -0,0 +1,37 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+module Masamune::Commands
+  class Interactive < SimpleDelegator
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      @interactive = attrs.fetch(:interactive, false)
+    end
+
+    def interactive?
+      @interactive
+    end
+  end
+end

data/lib/masamune/commands/postgres.rb

@@ -0,0 +1,128 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+require 'masamune/string_format'
+
+module Masamune::Commands
+  class Postgres < SimpleDelegator
+    include Masamune::StringFormat
+    include Masamune::Commands::PostgresCommon
+
+    DEFAULT_ATTRIBUTES =
+    {
+      :path           => 'psql',
+      :options        => [],
+      :hostname       => 'localhost',
+      :database       => 'postgres',
+      :username       => 'postgres',
+      :pgpass_file    => nil,
+      :file           => nil,
+      :exec           => nil,
+      :input          => nil,
+      :output         => nil,
+      :print          => false,
+      :block          => nil,
+      :csv            => false,
+      :variables      => {},
+      :tuple_output   => false,
+      :debug          => false
+    }
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.postgres).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    def stdin
+      if @input
+        @stdin ||= StringIO.new(strip_sql(@input))
+      end
+    end
+
+    def interactive?
+      !(@exec || @file)
+    end
+
+    def print?
+      @print
+    end
+
+    def command_args
+      args = []
+      args << @path
+      args << '--host=%s' % @hostname if @hostname
+      args << '--dbname=%s' % @database
+      args << '--username=%s' % @username if @username
+      args << '--no-password'
+      args << @options.map(&:to_a)
+      args << command_args_for_file if @file
+      args << '--output=%s' % @output if @output
+      args << '--no-align' << '--field-separator=,' << '--pset=footer' if @csv
+      args << '--command=%s' % strip_sql(@exec) if @exec
+      args << '--pset=tuples_only' if @tuple_output
+      args.flatten.compact
+    end
+
+    def before_execute
+      console("psql with file #{@file}") if @file
+      if @debug and output = @rendered_file || @file
+        logger.debug("#{output}:\n" + File.read(output))
+      end
+    end
+
+    def handle_stdout(line, line_no)
+      if line =~ /\A#{prompt}/
+        logger.debug(line)
+      else
+        @block.call(line) if @block
+        console(line) if print?
+      end
+    end
+
+    def prompt
+      @database + '=>'
+    end
+
+    private
+
+    def command_args_for_file
+      @file =~ /\.erb\Z/ ? command_args_for_template : command_args_for_simple_file
+    end
+
+    def command_args_for_simple_file
+      ['--file=%s' % @file].tap do |args|
+        @variables.each do |key, val|
+          args << '--set=%s' % "#{key.to_s}='#{val.to_s}'"
+        end
+      end
+    end
+
+    def command_args_for_template
+      @rendered_file = Masamune::Template.render_to_file(@file, @variables)
+      ['--file=%s' % @rendered_file]
+    end
+  end
+end