hasta 0.1.0

data/lib/hasta/emr_job_definition.rb

@@ -0,0 +1,104 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'forwardable'
+require 'json'
+
+require 'hasta/emr_node'
+require 'hasta/env'
+require 'hasta/identity_mapper'
+require 'hasta/mapper'
+require 'hasta/reducer'
+require 'hasta/identity_reducer'
+require 'hasta/s3_data_source'
+require 'hasta/s3_uri'
+require 'hasta/s3_data_sink'
+
+module Hasta
+  # Defines the EMR job that is being tested
+  class EmrJobDefinition
+    extend Forwardable
+
+    def self.load(file_path, id, scheduled_start_time = Time.now)
+      emr_node = JSON.parse(File.read(file_path))['objects'].find { |node|
+        node['type'] == 'EmrActivity' && node['id'] == id
+      }
+
+      raise ArgumentError, "No EmrActivity for id: #{id} in file: #{file_path}" unless emr_node
+      new(EmrNode.from_json(emr_node, scheduled_start_time))
+    end
+
+    def_delegators :emr_node, :id
+
+    def initialize(emr_node)
+      @emr_node = emr_node
+    end
+
+    def input_paths
+      @input_paths ||= emr_node.input_paths.map { |path| S3URI.parse(path) }
+    end
+
+    def output_path
+      @output_path ||= S3URI.parse(emr_node.output_path)
+    end
+
+    def env
+      @env ||= Env.new(
+        emr_node.env,
+        Hash[
+          emr_node.
+            cache_files.
+            reject { |tag, uri| uri.end_with?('.rb') }.
+            map { |tag, uri| ["#{tag.split('.').first.upcase}_FILE_PATH", S3URI.parse(uri)] }
+        ]
+      )
+    end
+
+    def ruby_files
+      @ruby_files ||= emr_node.
+        cache_files.
+        values.
+        select { |uri| uri.end_with?('.rb') }.
+        map { |uri| local_path_to_step_file(S3URI.parse(uri)) }
+    end
+
+    def mapper
+      @mapper ||= parse_mapper(emr_node.mapper)
+    end
+
+    def reducer
+      @reducer ||= parse_reducer(emr_node.reducer)
+    end
+
+    def data_sources
+      @data_sources ||= input_paths.map { |path| S3DataSource.new(path) }
+    end
+
+    def data_sink
+      @data_sink ||= S3DataSink.new(output_path)
+    end
+
+    private
+
+    attr_reader :emr_node
+
+    def local_path_to_step_file(s3_uri)
+      File.join(Hasta.project_root, Hasta.project_steps, s3_uri.basename)
+    end
+
+    def parse_mapper(mapper_command)
+      if %w[cat org.apache.hadoop.mapred.lib.IdentityMapper].include?(mapper_command)
+        IdentityMapper
+      else
+        Mapper.new(local_path_to_step_file(S3URI.parse(mapper_command)))
+      end
+    end
+
+    def parse_reducer(reducer_command)
+      if %w[cat org.apache.hadoop.mapred.lib.IdentityReducer].include?(reducer_command)
+        IdentityReducer
+      else
+        Reducer.new(local_path_to_step_file(S3URI.parse(reducer_command)))
+      end
+    end
+  end
+end

data/lib/hasta/emr_node.rb

@@ -0,0 +1,103 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/interpolate_string'
+
+module Hasta
+  # Models the Amazon Data Pipeline configuration details for the EMR job that is being tested
+  class EmrNode
+    class << self
+      def from_json(json, scheduled_start_time = Time.now)
+        command_line = parse_step_line(json['step'])
+
+        new(
+          :id => json['id'],
+          :input_paths => command_line['input'],
+          :output_path => command_line['output'].first,
+          :mapper => command_line['mapper'].first,
+          :reducer => command_line['reducer'].first,
+          :cache_files => command_line['cacheFile'],
+          :env => command_line['cmdenv'],
+          :scheduled_start_time => scheduled_start_time
+        )
+      end
+
+      private
+
+      # Parses the 'step' attribute of an EMR configuration into a Hash
+      # Sample step line:
+      #   "/home/hadoop/contrib/streaming/hadoop-streaming.jar,
+      #    -input,s3n://data-bucket/input1/,
+      #    -output,s3://data-bucket/output/,
+      #    -mapper,cat,
+      #    -reducer,s3n://steps-bucket/path/to/reducer.rb,
+      #    -cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,
+      #    -cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,
+      #    -cmdenv,API_KEY=123456,
+      #    -cmdenv,ENVIRONMENT_NAME=uat"
+      #
+      # Sample output:
+      #   {
+      #     "input" => ["s3n://data-bucket/input1/"],
+      #     "output"=> ["s3://data-bucket/output/"],
+      #     "mapper => ["cat"],
+      #     "reducer" => ["s3n://steps-bucket/path/to/reducer.rb"],
+      #     "cacheFile" => ["s3://data-bucket/path/to/mappings.yml#mappings.yml",
+      #       "s3://data-bucket/path/to/ignored.yml#ignored.yml"],
+      #     "cmdenv" => ["API_KEY=123456", "ENVIRONMENT_NAME=uat"]
+      #   }
+      #
+      def parse_step_line(step)
+        parsed = Hash.new { |h, k| h[k] = [] }
+        step.
+          split(',-').
+          drop(1).
+          map { |value| i = value.index(','); [value[0...i], value[i+1..-1]] }.
+          each do |switch, arg|
+            parsed[switch] << arg
+          end
+
+        parsed
+      end
+    end
+
+    def initialize(attributes)
+      @attributes = attributes
+    end
+
+    def id
+      attributes[:id]
+    end
+
+    def input_paths
+      @input_path ||= attributes[:input_paths].map { |path| interpolate(path) }
+    end
+
+    def output_path
+      @output_path ||= interpolate(attributes[:output_path])
+    end
+
+    def mapper
+      attributes[:mapper]
+    end
+
+    def reducer
+      attributes[:reducer]
+    end
+
+    def cache_files
+      @cache_files ||= Hash[attributes[:cache_files].map { |value| value.split('#').reverse }]
+    end
+
+    def env
+      @env ||= Hash[attributes[:env].map { |value| value.split('=') }]
+    end
+
+    private
+
+    attr_reader :attributes
+
+    def interpolate(path)
+      InterpolateString.evaluate(path, 'scheduledStartTime' => attributes[:scheduled_start_time])
+    end
+  end
+end

data/lib/hasta/env.rb

@@ -0,0 +1,35 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/local_file_path'
+require 'hasta/s3_data_source'
+
+module Hasta
+  # Constructs the ENV variables required to run a local EMR job
+  class Env
+    attr_reader :variables, :files
+
+    def initialize(
+      variables = {},
+      files = {},
+      combined_storage = Hasta.combined_storage
+      )
+      @variables = variables
+      @files = files
+      @combined_storage = combined_storage
+    end
+
+    def setup
+      file_vars = {}
+      files.each do |key, s3_uri|
+        input_source = S3DataSource.new(s3_uri, combined_storage)
+        file_vars[key] = LocalFilePath.for(combined_storage.write(s3_uri, input_source))
+      end
+
+      variables.merge(file_vars)
+    end
+
+    private
+
+    attr_reader :combined_storage
+  end
+end

data/lib/hasta/execution_context.rb

@@ -0,0 +1,90 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'open3'
+
+require 'hasta/s3_data_sink'
+
+module Hasta
+  # Executes each local EMR job in isolation
+  class ExecutionContext
+    # A Subprocess
+    class Subprocess
+      attr_reader :stdin, :stdout, :stderr
+
+      def initialize(ruby_files, env)
+        @ruby_files = ruby_files
+        @env = env
+      end
+
+      def start(source_file)
+        Open3.popen3(*cmd_line(source_file)) do |stdin, stdout, stderr, wait_thr|
+          @stdin, @stdout, @stderr, @wait_thr = stdin, stdout, stderr, wait_thr
+
+          yield self
+
+          if (exit_code = wait_thr.value.exitstatus) != 0
+            raise ExecutionError, "#{source_file} exited with non-zero status: #{exit_code}"
+          end
+        end
+      end
+
+      private
+
+      attr_reader :source_file, :env, :ruby_files
+
+      def cmd_line(source_file)
+        [env, ruby_exe_path] + load_path + [source_file]
+      end
+
+      def ruby_exe_path
+        File.join(RbConfig::CONFIG['bindir'], RbConfig::CONFIG['ruby_install_name'])
+      end
+
+      def load_path
+        ruby_files.
+          map { |file| File.expand_path(File.dirname(file)) }.
+          uniq.
+          map { |path| ['-I', path] }.
+          flatten
+      end
+    end
+
+    def initialize(ruby_files = [], env = {})
+      @sub_process = Subprocess.new(ruby_files, env)
+    end
+
+    def execute(source_file, data_source, data_sink)
+      sub_process.start(source_file) do |sub_process|
+        [
+          stream_input(data_source, sub_process.stdin),
+          stream_output(sub_process.stdout) { |line| data_sink << line },
+          stream_output(sub_process.stderr) { |line| Hasta.logger.error line },
+        ].each(&:join)
+      end
+
+      data_sink.close
+    end
+
+    private
+
+    attr_reader :sub_process
+
+    def stream_input(data_source, io)
+      Thread.new do
+        data_source.each_line do |line|
+          io.puts line
+        end
+
+        io.close_write
+      end
+    end
+
+    def stream_output(io)
+      Thread.new do
+        StringIO.new(io.read).each_line do |line|
+          yield line.rstrip
+        end
+      end
+    end
+  end
+end

data/lib/hasta/filter.rb

@@ -0,0 +1,40 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'set'
+
+module Hasta
+  # The filter that is used to drop unwanted lines from input files
+  class Filter
+    def self.from_file(file)
+      if lines = File.read(file)
+        Hasta.logger.debug "Loading data filter file: #{File.expand_path(file)}"
+        new(*lines.split("\n").map { |line| Regexp.new(line) })
+      end
+    rescue => ex
+      raise ConfigurationError.new,
+        "Failed to load filter configuration file: #{file} - #{ex.message}"
+    end
+
+    def initialize(*accept_regexes)
+      @accept_regexes = Set.new(accept_regexes)
+    end
+
+    def include?(line)
+      to_proc.call(line)
+    end
+
+    def to_proc
+      @proc ||= Proc.new { |line| !!(accept_regexes.find { |regex| line =~ regex }) }
+    end
+
+    def to_s
+      "#<#{self.class.name}:#{accept_regexes.to_a.inspect}>"
+    end
+
+    private
+
+    def accept_regexes
+      @accept_regexes.to_a.sort_by(&:inspect)
+    end
+  end
+end

data/lib/hasta/filtered_s3_file.rb

@@ -0,0 +1,34 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'delegate'
+require 'digest/md5'
+
+module Hasta
+  # An S3File delegate that drops filtered lines
+  class FilteredS3File < SimpleDelegator
+    def initialize(s3_file, filter)
+      super(s3_file)
+      @filter = filter
+    end
+
+    def body
+      each_line.to_a.join
+    end
+
+    def fingerprint
+      @fingerprint ||= Digest::MD5.hexdigest("#{__getobj__.fingerprint}_#{filter.to_s}")
+    end
+
+    def each_line
+      return enum_for(:each_line) unless block_given?
+
+      __getobj__.each_line do |line|
+        yield line if filter.include?(line)
+      end
+    end
+
+    private
+
+    attr_reader :filter
+  end
+end

data/lib/hasta/identity_mapper.rb

@@ -0,0 +1,17 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/combined_data_source'
+
+module Hasta
+  # Used by any EMR job that required an identity mapper
+  module IdentityMapper
+    def self.map(_, data_sources, data_sink = InMemoryDataSink.new)
+      Hasta.logger.debug "Starting Identity Mapper"
+      CombinedDataSource.new(data_sources).each_line do |line|
+        data_sink << line.rstrip
+      end
+
+      data_sink.close.tap { Hasta.logger.debug "Finished Identity Mapper" }
+    end
+  end
+end

data/lib/hasta/identity_reducer.rb

@@ -0,0 +1,18 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/in_memory_data_sink'
+require 'hasta/sorted_data_source'
+
+module Hasta
+  # Used by any EMR job that requires an identity reducer
+  module IdentityReducer
+    def self.reduce(_, data_source, data_sink = InMemoryDataSink.new)
+      Hasta.logger.debug "Starting Identity Reducer"
+      SortedDataSource.new(data_source).each_line do |line|
+        data_sink << line.rstrip
+      end
+
+      data_sink.close.tap { Hasta.logger.debug "Finished Identity Reducer" }
+    end
+  end
+end