masamune 0.11.0

data/lib/masamune/actions/elastic_mapreduce.rb

@@ -0,0 +1,68 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+
+module Masamune::Actions
+  module ElasticMapreduce
+    extend ActiveSupport::Concern
+
+    def elastic_mapreduce(opts = {})
+      opts = opts.to_hash.symbolize_keys
+      opts[:jobflow] = resolve_jobflow(opts[:jobflow]) if opts[:jobflow]
+
+      command = Masamune::Commands::Interactive.new(environment, :interactive => opts.fetch(:interactive, false))
+      command = Masamune::Commands::ElasticMapReduce.new(command, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.interactive? ? command.replace : command.execute
+    end
+
+    def defined_jobflows
+      @defined_jobflows ||= configuration.elastic_mapreduce.fetch(:jobflows, {}).symbolize_keys
+    end
+
+    def resolve_jobflow(jobflow)
+      return unless jobflow
+      defined_jobflows[jobflow.to_sym] || jobflow
+    end
+
+    def jobflow_required?
+      extra.empty?
+    end
+
+    included do |base|
+      base.class_option :jobflow, :aliases => '-j', :desc => 'Elastic MapReduce jobflow ID (Hint: elastic-mapreduce --list)' if defined?(base.class_option)
+      base.after_initialize(:early) do |thor, options|
+        next if thor.configuration.elastic_mapreduce.empty?
+        next unless thor.configuration.elastic_mapreduce.fetch(:enabled, true)
+        jobflow = thor.resolve_jobflow(options[:jobflow] || thor.configuration.elastic_mapreduce[:jobflow])
+        if thor.jobflow_required?
+          raise ::Thor::RequiredArgumentMissingError, "No value provided for required options '--jobflow'" unless jobflow
+          raise ::Thor::RequiredArgumentMissingError, %Q(Value '#{jobflow}' for '--jobflow' doesn't exist) unless thor.elastic_mapreduce(extra: '--list', jobflow: jobflow, fail_fast: false).success?
+        end
+        thor.configuration.elastic_mapreduce[:jobflow] = jobflow
+      end if defined?(base.after_initialize)
+    end
+  end
+end

data/lib/masamune/actions/execute.rb

@@ -0,0 +1,52 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Actions
+  module Execute
+    def execute(*args, &block)
+      opts = args.last.is_a?(Hash) ? args.pop : {}
+      opts = opts.to_hash.symbolize_keys
+
+      klass = Class.new(SimpleDelegator)
+      klass.class_eval do
+        define_method(:command_args) do
+          args
+        end
+      end
+
+      klass.class_eval do
+        define_method(:stdin) do
+          @stdin ||= StringIO.new(opts[:input])
+        end
+      end if opts[:input]
+
+      klass.class_eval do
+        define_method(:handle_stdout) do |line, line_no|
+          block.call(line, line_no)
+        end
+      end if block_given?
+
+      command = Masamune::Commands::Shell.new(klass.new(self), {fail_fast: false}.merge(opts))
+      command.execute
+    end
+  end
+end

data/lib/masamune/actions/filesystem.rb

@@ -0,0 +1,37 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+
+module Masamune::Actions
+  module Filesystem
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def filesystem
+        defined?(environment) ? environment.filesystem : Masamune.environment.filesystem
+      end
+      alias :fs :filesystem
+    end
+    include ClassMethods
+  end
+end

data/lib/masamune/actions/hadoop_filesystem.rb

@@ -0,0 +1,40 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Actions
+  module HadoopFilesystem
+    def hadoop_filesystem(*args, &block)
+      opts = args.last.is_a?(Hash) ? args.pop : {}
+      opts = opts.to_hash.symbolize_keys
+      opts.reverse_merge!(fail_fast: false)
+      opts.merge!(extra: Array.wrap(args))
+      opts.merge!(block: block.to_proc) if block_given?
+
+      command = Masamune::Commands::HadoopFilesystem.new(environment, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.execute
+    end
+    alias hadoop_fs hadoop_filesystem
+  end
+end

data/lib/masamune/actions/hadoop_streaming.rb

@@ -0,0 +1,41 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Actions
+  module HadoopStreaming
+    def hadoop_streaming(opts = {})
+      opts = opts.to_hash.symbolize_keys
+
+      command = if configuration.elastic_mapreduce[:jobflow]
+        Masamune::Commands::HadoopStreaming.new(environment, opts.merge(quote: true, upload: false))
+      else
+        Masamune::Commands::HadoopStreaming.new(environment, opts)
+      end
+
+      command = Masamune::Commands::ElasticMapReduce.new(command, opts.except(:extra)) if configuration.elastic_mapreduce[:jobflow]
+      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.execute
+    end
+  end
+end

data/lib/masamune/actions/hive.rb

@@ -0,0 +1,74 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+
+require 'masamune/transform/define_schema'
+
+module Masamune::Actions
+  module Hive
+    include Masamune::Transform::DefineSchema
+
+    extend ActiveSupport::Concern
+
+    def hive(opts = {}, &block)
+      opts = opts.to_hash.symbolize_keys
+      opts.merge!(block: block.to_proc) if block_given?
+
+      command = Masamune::Commands::Hive.new(environment, opts)
+      command = Masamune::Commands::ElasticMapReduce.new(command, opts.except(:extra)) if configuration.elastic_mapreduce[:jobflow]
+      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.interactive? ? command.replace : command.execute
+    end
+
+    # TODO warn or error if database is not defined
+    def create_hive_database_if_not_exists
+      return if configuration.hive[:database] == 'default'
+      sql = []
+      sql << %Q(CREATE DATABASE IF NOT EXISTS #{configuration.hive[:database]})
+      sql << %Q(LOCATION "#{configuration.hive[:location]}") if configuration.hive[:location]
+      hive(exec: sql.join(' ') + ';', database: nil)
+    end
+
+    def load_hive_schema
+      transform = define_schema(catalog, :hive)
+      hive(file: transform.to_file)
+    rescue => e
+      logger.error(e)
+      logger.error("Could not load schema")
+      logger.error("\n" + transform.to_s)
+      exit
+    end
+
+    included do |base|
+      base.after_initialize do |thor, options|
+        thor.create_hive_database_if_not_exists
+        if options[:dry_run]
+          raise ::Thor::InvocationError, 'Dry run of hive failed' unless thor.hive(exec: 'SHOW TABLES;', safe: true, fail_fast: false).success?
+        end
+        thor.load_hive_schema
+      end if defined?(base.after_initialize)
+    end
+  end
+end

data/lib/masamune/actions/postgres.rb

@@ -0,0 +1,76 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+require 'masamune/transform/define_schema'
+require 'masamune/actions/postgres_admin'
+
+module Masamune::Actions
+  module Postgres
+    extend ActiveSupport::Concern
+
+    include Masamune::Actions::PostgresAdmin
+    include Masamune::Transform::DefineSchema
+
+    def postgres(opts = {}, &block)
+      opts = opts.to_hash.symbolize_keys
+      opts.merge!(block: block.to_proc) if block_given?
+
+      command = Masamune::Commands::Postgres.new(environment, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.interactive? ? command.replace : command.execute
+    end
+
+    def create_postgres_database_if_not_exists
+      unless postgres_helper.database_exists?
+        postgres_admin(action: :create, database: configuration.postgres[:database], safe: true)
+      end if configuration.postgres.has_key?(:database)
+    end
+
+    def load_postgres_setup_files
+      configuration.postgres[:setup_files].each do |file|
+        configuration.with_quiet do
+          postgres(file: file)
+        end
+      end if configuration.postgres.has_key?(:setup_files)
+    end
+
+    def load_postgres_schema
+      transform = define_schema(catalog, :postgres)
+      postgres(file: transform.to_file)
+    rescue => e
+      logger.error(e)
+      logger.error("Could not load schema")
+      logger.error("\n" + transform.to_s)
+      exit
+    end
+
+    included do |base|
+      base.after_initialize do |thor, options|
+        thor.create_postgres_database_if_not_exists
+        thor.load_postgres_setup_files
+        thor.load_postgres_schema
+      end if defined?(base.after_initialize)
+    end
+  end
+end

data/lib/masamune/actions/postgres_admin.rb

@@ -0,0 +1,34 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Actions
+  module PostgresAdmin
+    def postgres_admin(opts = {}, &block)
+      opts = opts.to_hash.symbolize_keys
+
+      command = Masamune::Commands::PostgresAdmin.new(environment, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.execute
+    end
+  end
+end

data/lib/masamune/actions/s3cmd.rb

@@ -0,0 +1,44 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune::Actions
+  module S3Cmd
+    include Masamune::Commands::S3Cmd::ClassMethods
+
+    def s3cmd(*args, &block)
+      opts = args.last.is_a?(Hash) ? args.pop : {}
+      opts = opts.to_hash.symbolize_keys
+      opts.merge!(extra: Array.wrap(args))
+      opts.merge!(block: block.to_proc) if block_given?
+
+      command = Masamune::Commands::S3Cmd.new(environment, opts)
+      command = Masamune::Commands::RetryWithBackoff.new(command, opts)
+      command = Masamune::Commands::Shell.new(command, opts)
+
+      command.execute
+    end
+
+    def s3_sync(src, dst)
+      s3cmd('sync', s3b(src), s3b(dst, :dir => true))
+    end
+  end
+end