masamune 0.15.6 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b70cf7812702e987cc6c1f72d7f50cbb3ddde9b
-  data.tar.gz: 2d487a59b8da26e186f2eefab0f7067b5e9f950f
+  metadata.gz: c8bcd874b8f53a052225b41043cfda18de823cec
+  data.tar.gz: 9e4c71e47097a5035be079b92ce9d2698d74c7b4
 SHA512:
-  metadata.gz: 91777fd4da7e0928d678ce4574f91305da2673e8bbbbeed65ce892ac612808552afa68689906f70a54c8f61d4f592b821ecb4ae9e67588bb988a6dc38e2a07b7
-  data.tar.gz: 76a2f706860eae74756e2c4b7a04e596913f3c0189e591d5c5f250681fcada14177be815f4033bede60ba1fa75a70a073d7a3bb91bac6692ab21c1ee04d8df3b
+  metadata.gz: 8e4b65c5424fdf38915854a32299cc7825418f96798c63719619a8cc4bfd7a69d81e286006974b5660a03cb3b020a677b7db838cf5efa0f03d79ff4cac209053
+  data.tar.gz: 8099dca87827675a761f9882025452d0c8bfd24773b09538006747e5b155d1ef9ec2f82a16bb16d5dc7b313aec748081d07541c332be5e633299c0dfeb88000c

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/Rakefile

@@ -1,7 +1,7 @@
 #!/usr/bin/env rake
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/bin/{masamune-elastic-mapreduce → masamune-aws-emr}

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -22,5 +22,5 @@
 #  THE SOFTWARE.
 
 $: << File.expand_path('../../lib/', __FILE__)
-require 'masamune/tasks/elastic_mapreduce_thor'
-Masamune::Tasks::ElasticMapreduceThor.start(ARGV)
+require 'masamune/tasks/aws_emr_thor'
+Masamune::Tasks::AwsEmrThor.start(ARGV)

data/bin/masamune-dump

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/bin/masamune-hive

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/bin/masamune-psql

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/bin/masamune-shell

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/{elastic_mapreduce.rb → aws_emr.rb}

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -21,49 +21,33 @@
 #  THE SOFTWARE.
 
 module Masamune::Actions
-  module ElasticMapreduce
+  module AwsEmr
     extend ActiveSupport::Concern
 
-    def elastic_mapreduce(opts = {})
+    def aws_emr(opts = {})
       opts = opts.to_hash.symbolize_keys
-      opts[:jobflow] = resolve_jobflow(opts[:jobflow]) if opts[:jobflow]
 
-      command = Masamune::Commands::Interactive.new(environment, :interactive => opts.fetch(:interactive, false))
-      command = Masamune::Commands::ElasticMapReduce.new(command, opts)
-      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.elastic_mapreduce.slice(:retries, :backoff).merge(opts))
+      command = Masamune::Commands::AwsEmr.new(environment, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, configuration.aws_emr.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.interactive? ? command.replace : command.execute
     end
 
-    def defined_jobflows
-      @defined_jobflows ||= configuration.elastic_mapreduce.fetch(:jobflows, {}).symbolize_keys
-    end
-
-    def resolve_jobflow(jobflow)
-      return unless jobflow
-      defined_jobflows[jobflow.to_sym] || jobflow
-    end
-
-    def jobflow_required?
-      extra.empty?
-    end
-
-    def validate_jobflow!
-      return unless jobflow_required?
-      jobflow = configuration.elastic_mapreduce[:jobflow]
-      raise ::Thor::RequiredArgumentMissingError, "No value provided for required options '--jobflow'" unless jobflow
-      raise ::Thor::RequiredArgumentMissingError, %Q(Value '#{jobflow}' for '--jobflow' doesn't exist) unless elastic_mapreduce(extra: '--list', jobflow: jobflow, fail_fast: false).success?
+    def validate_cluster_id!
+      cluster_id = configuration.aws_emr[:cluster_id]
+      raise ::Thor::RequiredArgumentMissingError, "No value provided for required options '--cluster-id'" unless cluster_id
+      raise ::Thor::RequiredArgumentMissingError, %Q(AWS EMR cluster '#{cluster_id}' does not exist) unless aws_emr(action: 'describe-cluster', cluster_id: cluster_id, fail_fast: false).success?
     end
 
     included do |base|
-      base.class_option :jobflow, :aliases => '-j', :desc => 'Elastic MapReduce jobflow ID (Hint: elastic-mapreduce --list)' if defined?(base.class_option)
+      base.class_option :cluster_id, :desc => "AWS EMR cluster_id ID (Hint: `masamune-emr-aws list-clusters`)" if defined?(base.class_option)
       base.after_initialize(:early) do |thor, options|
-        next unless thor.configuration.elastic_mapreduce.any?
-        next unless thor.configuration.elastic_mapreduce.fetch(:enabled, true)
-        thor.configuration.elastic_mapreduce[:jobflow] = thor.resolve_jobflow(options[:jobflow] || thor.configuration.elastic_mapreduce[:jobflow])
+        next unless thor.configuration.aws_emr.any?
+        next unless thor.configuration.aws_emr.fetch(:enabled, true)
+        thor.configuration.aws_emr[:cluster_id] = options[:cluster_id] if options[:cluster_id]
         next unless options[:initialize]
-        thor.validate_jobflow!
+        thor.validate_cluster_id!
       end if defined?(base.after_initialize)
     end
   end

data/lib/masamune/actions/data_flow.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/date_parse.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/execute.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/filesystem.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/hadoop_filesystem.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/hadoop_streaming.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -25,17 +25,19 @@ module Masamune::Actions
     def hadoop_streaming(opts = {})
       opts = opts.to_hash.symbolize_keys
 
-      command = if configuration.elastic_mapreduce[:jobflow]
-        Masamune::Commands::HadoopStreaming.new(environment, opts.merge(quote: true, upload: false))
-      else
-        Masamune::Commands::HadoopStreaming.new(environment, opts)
-      end
-
-      command = Masamune::Commands::ElasticMapReduce.new(command, opts.except(:extra)) if configuration.elastic_mapreduce[:jobflow]
+      command = Masamune::Commands::HadoopStreaming.new(environment, aws_emr_options(opts))
+      command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.aws_emr[:cluster_id]
       command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hadoop_streaming.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 
       command.execute
     end
+
+    private
+
+    def aws_emr_options(opts = {})
+      return opts unless configuration.aws_emr[:cluster_id]
+      opts.merge(quote: true, upload: false)
+    end
   end
 end

data/lib/masamune/actions/hive.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -33,7 +33,7 @@ module Masamune::Actions
       opts.merge!(block: block.to_proc) if block_given?
 
       command = Masamune::Commands::Hive.new(environment, opts)
-      command = Masamune::Commands::ElasticMapReduce.new(command, opts.except(:extra)) if configuration.elastic_mapreduce[:jobflow]
+      command = Masamune::Commands::AwsEmr.new(command, opts.except(:extra)) if configuration.aws_emr[:cluster_id]
       command = Masamune::Commands::RetryWithBackoff.new(command, configuration.hive.slice(:retries, :backoff).merge(opts))
       command = Masamune::Commands::Shell.new(command, opts)
 

data/lib/masamune/actions/invoke_parallel.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/postgres.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/postgres_admin.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/s3cmd.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/actions/transform.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/after_initialize_callbacks.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/cached_filesystem.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -36,7 +36,7 @@ module Masamune
     end
 
     def exists?(file)
-      glob(file, max_depth: 0).include?(file) || @cache.any?(file)
+      glob(file, max_depth: 0).include?(file)
     end
 
     def glob(file_or_glob, options = {})

data/lib/masamune/commands.rb

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal

data/lib/masamune/commands/{elastic_mapreduce.rb → aws_emr.rb}

@@ -1,6 +1,6 @@
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#  Copyright (c) 2014-2016, VMware, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -25,22 +25,23 @@ require 'delegate'
 require 'masamune/actions/execute'
 
 module Masamune::Commands
-  class ElasticMapReduce < SimpleDelegator
+  class AwsEmr < SimpleDelegator
     include Masamune::Actions::Execute
 
     DEFAULT_ATTRIBUTES =
     {
-      :path       => 'elastic-mapreduce',
-      :options    => [],
-      :extra      => [],
-      :jobflow    => nil,
-      :input      => nil,
+      :path         => 'aws',
+      :extra        => [],
+      :config_file  => nil,
+      :action       => nil,
+      :cluster_id   => nil,
+      :interactive  => false
     }
 
     def initialize(delegate, attrs = {})
       super delegate
       @delegate = delegate
-      DEFAULT_ATTRIBUTES.merge(configuration.elastic_mapreduce).merge(attrs).each do |name, value|
+      DEFAULT_ATTRIBUTES.merge(configuration.aws_emr).merge(attrs).each do |name, value|
         instance_variable_set("@#{name}", value)
       end
     end
@@ -48,55 +49,55 @@ module Masamune::Commands
     def interactive?
       if @delegate.respond_to?(:interactive?)
         @delegate.interactive?
-      elsif @extra.any?
-        true
       else
-        @input == nil
+        @interactive
       end
     end
 
-    def stdin
-      if @delegate.respond_to?(:stdin)
-        @delegate.stdin
-      elsif @input
-        @stdin ||= StringIO.new(@input)
-      end
-    end
-
-    def elastic_mapreduce_command
+    def aws_emr_command
       args = []
       args << @path
-      args << @options.map(&:to_a)
-      args << ['--jobflow', @jobflow] if @jobflow
-      args.flatten
+      args << 'emr'
+      args << action
+      args << 'cluster-running' if @action == 'wait'
+      args << action_options.map(&:to_a)
+      args << ['--cluster-id', @cluster_id] if @cluster_id
+      args
     end
 
     def ssh_args
       args = []
-      args << elastic_mapreduce_command
-      args << '--ssh'
+      args << aws_emr_command
+      args << '--command'
       args << 'exit'
       args.flatten
     end
 
-    # Use elastic-mapreduce to translate jobflow into raw ssh command
+    # Use `aws emr` to translate cluster_id into raw ssh command
     def ssh_command
       @ssh_command ||= begin
         result = nil
         execute(*ssh_args, fail_fast: true, safe: true) do |line|
+          next if result
           if line =~ /exit\Z/
             result = line.sub(/ exit\Z/, '').split(' ')
           else
-            logger.error(line)
+            logger.debug(line)
           end
         end
         result
       end
     end
 
+    def command_env
+      {}.tap do |env|
+        env['AWS_CONFIG_FILE'] = @config_file if @config_file
+      end
+    end
+
     def command_args
       args = []
-      args << (ssh_command? ? ssh_command : elastic_mapreduce_command)
+      args << (ssh_command? ? ssh_command : aws_emr_command)
       args << @extra
       args << @delegate.command_args if @delegate.respond_to?(:command_args)
       args.flatten
@@ -112,8 +113,16 @@ module Masamune::Commands
 
     private
 
+    def action
+      @action || 'ssh'
+    end
+
+    def action_options
+      configuration.aws_emr.fetch(action.underscore.to_sym, {}).with_indifferent_access.fetch(:options, {}).reject { |key, _| @extra.include?(key.to_s) }
+    end
+
     def ssh_command?
-      @delegate.respond_to?(:command_args) || @input.present?
+      @delegate.respond_to?(:command_args)
     end
   end
 end