masamune 0.11.0

data/lib/masamune/actions/transform.rb

@@ -0,0 +1,89 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+
+require 'masamune/actions/postgres'
+
+require 'masamune/transform/load_dimension'
+require 'masamune/transform/consolidate_dimension'
+require 'masamune/transform/relabel_dimension'
+require 'masamune/transform/load_fact'
+require 'masamune/transform/rollup_fact'
+
+module Masamune::Actions
+  module Transform
+    extend ActiveSupport::Concern
+    extend Forwardable
+
+    include Masamune::Actions::Postgres
+
+    # FIXME should eventually be able to include Transform directly instead of through wrapper
+    class Wrapper
+      extend Masamune::Transform::LoadDimension
+      extend Masamune::Transform::ConsolidateDimension
+      extend Masamune::Transform::RelabelDimension
+      extend Masamune::Transform::LoadFact
+      extend Masamune::Transform::RollupFact
+    end
+
+    FILE_MODE = 0777 - File.umask
+
+    def load_dimension(source_files, source, target)
+      output = Tempfile.new('masamune')
+      FileUtils.chmod(FILE_MODE, output.path)
+
+      if source.respond_to?(:map) and map = source.map(to: target)
+        result = map.apply(source_files, output)
+      else
+        output = source_files
+        result = source
+      end
+
+      transform = Wrapper.load_dimension(output, result, target)
+      logger.debug(File.read(output)) if (source.debug || map.debug)
+      postgres file: transform.to_file, debug: (source.debug || target.debug || map.debug)
+    ensure
+      output.unlink
+    end
+
+    def consolidate_dimension(target)
+      transform = Wrapper.consolidate_dimension(target)
+      postgres file: transform.to_file, debug: target.debug
+    end
+
+    def relabel_dimension(target)
+      transform = Wrapper.relabel_dimension(target)
+      postgres file: transform.to_file, debug: target.debug
+    end
+
+    def load_fact(source_files, source, target, date)
+      transform = Wrapper.load_fact(source_files, source, target, date)
+      postgres file: transform.to_file, debug: (source.debug || target.debug)
+    end
+
+    def rollup_fact(source, target, date)
+      transform = Wrapper.rollup_fact(source, target, date)
+      postgres file: transform.to_file, debug: (source.debug || target.debug)
+    end
+  end
+end

data/lib/masamune/after_initialize_callbacks.rb

@@ -0,0 +1,55 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'active_support/concern'
+
+module Masamune
+  module AfterInitializeCallbacks
+    extend ActiveSupport::Concern
+
+    PRIORITY =
+    {
+      first:    20,
+      early:    10,
+      default:   0,
+      later:   -10,
+      final:   -20
+    }
+
+    module ClassMethods
+      # Callbacks registered with the highest priority are executed first, ties are broken by callback registration order
+      def after_initialize(priority = :default, &block)
+        @after_initialize ||= Hash.new { |h,k| h[k] = [] }
+        @after_initialize[PRIORITY.fetch(priority, 0)] << block
+      end
+
+      def after_initialize_invoke(*a)
+        @after_initialize ||= Hash.new { |h,k| h[k] = [] }
+        @after_initialize.sort.reverse.each { |p, x| x.each { |y| y.call(*a) } }
+      end
+    end
+
+    def after_initialize_invoke(*a)
+      self.class.after_initialize_invoke(self, *a)
+    end
+  end
+end

data/lib/masamune/cached_filesystem.rb

@@ -0,0 +1,110 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune
+  class CachedFilesystem < SimpleDelegator
+    include Masamune::Accumulate
+
+    def initialize(filesystem)
+      super filesystem
+      @filesystem = filesystem
+      clear!
+    end
+
+    def clear!
+      @cache = Hash.new { |h,k| h[k] = Set.new }
+    end
+
+    def exists?(file)
+      @cache.key?(file) || glob(file).include?(file) || @cache.key?(file)
+    end
+
+    def glob(file_or_glob, &block)
+      glob_stat(file_or_glob) do |entry|
+        yield entry.name unless entry.name == dirname(file_or_glob)
+      end
+    end
+    method_accumulate :glob
+
+    def stat(file_or_dir)
+      raise ArgumentError, 'cannot contain wildcard' if file_or_dir.include?('*')
+      result = Set.new
+      glob_stat(file_or_dir) do |entry|
+        result << entry
+      end
+      result += @cache[file_or_dir]
+      return unless result.any?
+      return result.first if result.size == 1
+      max_time = result.map { |stat| stat.try(:mtime) }.compact.max
+      sum_size = result.map { |stat| stat.try(:size) }.compact.reduce(:+)
+      OpenStruct.new(name: file_or_dir, mtime: max_time, size: sum_size)
+    end
+
+    # FIXME cache eviction policy can be more precise
+    [:touch!, :mkdir!, :copy_file_to_file, :copy_file_to_dir, :copy_dir, :remove_file, :remove_dir, :move_file_to_file, :move_file_to_dir, :move_dir, :write].each do |method|
+      define_method(method) do |*args|
+        clear!
+        @filesystem.send(method, *args)
+      end
+    end
+
+    private
+
+    MAX_DEPTH   = 10
+    CACHE_DEPTH = 1
+    EMPTY_SET   = Set.new
+
+    def glob_stat(file_or_glob, options = {}, &block)
+      return if file_or_glob.blank?
+      return if root_path?(file_or_glob)
+      depth = options.fetch(:depth, 0)
+      return if depth > MAX_DEPTH || depth > CACHE_DEPTH
+
+      glob_stat(dirname(file_or_glob), depth: depth + 1, &block)
+
+      dirname = dirname(file_or_glob)
+      unless @cache.key?(dirname)
+        pattern = root_path?(dirname) ? file_or_glob : File.join(dirname, '*')
+        @filesystem.glob_stat(pattern) do |entry|
+          recursive_paths(dirname, entry.name) do |path|
+            @cache[path] << entry
+          end
+        end
+      end
+      @cache[dirname] ||= EMPTY_SET
+
+      file_regexp = glob_to_regexp(file_or_glob)
+      @cache[dirname].each do |entry|
+        yield entry if entry.name =~ file_regexp
+      end if depth == 0
+    end
+
+    def recursive_paths(root, path, options = {}, &block)
+      depth = options.fetch(:depth, 0)
+      return if depth > MAX_DEPTH
+      return if root == path
+      yield path
+      yield dirname(path)
+      recursive_paths(root, dirname(path), depth: depth + 1, &block)
+    end
+  end
+end

data/lib/masamune/commands.rb

@@ -0,0 +1,37 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+module Masamune
+  module Commands
+    require 'masamune/commands/hive'
+    require 'masamune/commands/s3cmd'
+    require 'masamune/commands/hadoop_streaming'
+    require 'masamune/commands/hadoop_filesystem'
+    require 'masamune/commands/elastic_mapreduce'
+    require 'masamune/commands/postgres_common'
+    require 'masamune/commands/postgres'
+    require 'masamune/commands/postgres_admin'
+    require 'masamune/commands/interactive'
+    require 'masamune/commands/shell'
+    require 'masamune/commands/retry_with_backoff'
+  end
+end

data/lib/masamune/commands/elastic_mapreduce.rb

@@ -0,0 +1,119 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+require 'masamune/actions/execute'
+
+module Masamune::Commands
+  class ElasticMapReduce < SimpleDelegator
+    include Masamune::Actions::Execute
+
+    DEFAULT_ATTRIBUTES =
+    {
+      :path       => 'elastic-mapreduce',
+      :options    => [],
+      :extra      => [],
+      :jobflow    => nil,
+      :input      => nil,
+    }
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      @delegate = delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.elastic_mapreduce).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    def interactive?
+      if @delegate.respond_to?(:interactive?)
+        @delegate.interactive?
+      elsif @extra.any?
+        true
+      else
+        @input == nil
+      end
+    end
+
+    def stdin
+      if @delegate.respond_to?(:stdin)
+        @delegate.stdin
+      elsif @input
+        @stdin ||= StringIO.new(@input)
+      end
+    end
+
+    def elastic_mapreduce_command
+      args = []
+      args << @path
+      args << @options.map(&:to_a)
+      args << ['--jobflow', @jobflow] if @jobflow
+      args.flatten
+    end
+
+    def ssh_args
+      args = []
+      args << elastic_mapreduce_command
+      args << '--ssh'
+      args << 'exit'
+      args.flatten
+    end
+
+    # Use elastic-mapreduce to translate jobflow into raw ssh command
+    def ssh_command
+      @ssh_command ||= begin
+        result = nil
+        execute(*ssh_args, fail_fast: true, safe: true) do |line|
+          if line =~ /exit\Z/
+            result = line.sub(/ exit\Z/, '').split(' ')
+          else
+            logger.error(line)
+          end
+        end
+        result
+      end
+    end
+
+    def command_args
+      args = []
+      args << (ssh_command? ? ssh_command : elastic_mapreduce_command)
+      args << @extra
+      args << @delegate.command_args if @delegate.respond_to?(:command_args)
+      args.flatten
+    end
+
+    def handle_stdout(line, line_no)
+      if line_no == 0 && line =~ /\Assh/
+        @delegate.handle_stderr(line, line_no) if @delegate.respond_to?(:handle_stderr)
+      else
+        @delegate.handle_stdout(line, line_no) if @delegate.respond_to?(:handle_stdout)
+      end
+    end
+
+    private
+
+    def ssh_command?
+      @delegate.respond_to?(:command_args) || @input.present?
+    end
+  end
+end

data/lib/masamune/commands/hadoop_filesystem.rb

@@ -0,0 +1,57 @@
+#  The MIT License (MIT)
+#
+#  Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+require 'delegate'
+
+module Masamune::Commands
+  class HadoopFilesystem < SimpleDelegator
+    DEFAULT_ATTRIBUTES =
+    {
+      :path         => 'hadoop',
+      :options      => [],
+      :extra        => [],
+      :block        => nil,
+      :print        => false
+    }
+
+    def initialize(delegate, attrs = {})
+      super delegate
+      DEFAULT_ATTRIBUTES.merge(configuration.hadoop_filesystem).merge(attrs).each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    def command_args
+      args = []
+      args << @path
+      args << 'fs'
+      args << @options.map(&:to_a)
+      args << @extra
+      args.flatten
+    end
+
+    def handle_stdout(line, line_no)
+      @block.call(line) if @block
+      console(line) if @print
+    end
+  end
+end