hasta 0.1.0

data/lib/hasta/in_memory_data_sink.rb

@@ -0,0 +1,40 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/in_memory_data_source'
+
+module Hasta
+  # Data sink for writing data in-memory
+  class InMemoryDataSink
+    attr_reader :name
+
+    def initialize(name=nil)
+      @name = name
+    end
+
+    def <<(line)
+      lines << Hasta.tab_separated_line(line)
+    end
+
+    def data_source
+      InMemoryDataSource.new(lines, name)
+    end
+
+    def close
+      self
+    end
+
+    def to_s
+      if name
+        "#<#{self.class.name}:#{name} (#{lines.count} lines)>"
+      else
+        super
+      end
+    end
+
+    private
+
+    def lines
+      @lines ||= []
+    end
+  end
+end

data/lib/hasta/in_memory_data_source.rb

@@ -0,0 +1,35 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'forwardable'
+
+module Hasta
+  # Data source for reading data from memory
+  class InMemoryDataSource
+    attr_reader :name
+
+    def initialize(lines, name=nil)
+      @lines = lines
+      @name = name
+    end
+
+    def each_line
+      return enum_for(:each_line) unless block_given?
+
+      lines.each do |line|
+        yield line
+      end
+    end
+
+    def to_a
+      lines
+    end
+
+    def to_s
+      "#<#{self.class.name}:#{name} size=#{lines.count} lines>"
+    end
+
+    private
+
+    attr_reader :lines
+  end
+end

data/lib/hasta/interpolate_string.rb

@@ -0,0 +1,45 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+module Hasta
+  # Interpolates scheduled start time expressions in S3 path strings
+  class InterpolateString
+    INTERPOLATE_PATTERN = /\#\{format\(@scheduledStartTime,'(.*?)'\)\}/
+
+    PATTERN_CONVERSIONS = {
+      'YYYY' => '%Y',
+      'MM' => '%m',
+      'dd' => '%d',
+      'HH' => '%H',
+      'mm' => '%M',
+      'ss' => '%S',
+    }
+
+    def self.evaluate(pattern, context)
+      new(pattern).evaluate(context)
+    end
+
+    def initialize(pattern)
+      @pattern = pattern
+    end
+
+    def evaluate(context)
+      pattern.gsub(INTERPOLATE_PATTERN) do |match|
+        format(context, Regexp.last_match[1])
+      end
+    end
+
+    private
+
+    attr_reader :pattern
+
+    def format(context, pattern)
+      context['scheduledStartTime'].strftime(convert_pattern(pattern))
+    end
+
+    def convert_pattern(pattern)
+      PATTERN_CONVERSIONS.inject(pattern) { |converted_pattern, (pipeline_pattern, ruby_pattern)|
+        converted_pattern.gsub(pipeline_pattern, ruby_pattern)
+      }
+    end
+  end
+end

data/lib/hasta/local_file_path.rb

@@ -0,0 +1,12 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/s3_uri'
+
+module Hasta
+  # Resolves the local file path for an S3 URI
+  module LocalFilePath
+    def self.for(s3_uri)
+      File.expand_path(File.join(Hasta.local_storage_root, s3_uri.bucket, s3_uri.path))
+    end
+  end
+end

data/lib/hasta/local_storage.rb

@@ -0,0 +1,41 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'fog'
+
+require 'hasta/s3_uri'
+require 'hasta/storage'
+
+module Hasta
+  # The read/write file storage interface to the local representation of the S3 data used
+  # by the local map/reduce jobs
+  class LocalStorage
+    include Storage
+
+    def write(s3_uri, data_source)
+      contents = StringIO.new
+      data_source.each_line do |line|
+        contents << line
+      end
+
+      write_to(
+        s3_uri.file? ? s3_uri : s3_uri.append('part-00000'),
+        contents.string
+      )
+    end
+
+    private
+
+    def write_to(s3_uri, contents)
+      write_bucket = bucket(s3_uri) || create_bucket(s3_uri.bucket)
+      file = create_file(write_bucket, s3_uri.path, contents)
+
+      s3_uri
+    end
+
+    def fog_files(s3_bucket, s3_uri)
+      s3_bucket.files.select { |file|
+        file.key.start_with?(s3_uri.path) && file_s3_uri(file).parent == s3_uri
+      }
+    end
+  end
+end

data/lib/hasta/mapper.rb

@@ -0,0 +1,23 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/combined_data_source'
+require 'hasta/execution_context'
+require 'hasta/in_memory_data_sink'
+
+module Hasta
+  # A wrapper for instantiating a mapper from a definition file and invoking it
+  class Mapper
+    attr_reader :mapper_file
+
+    def initialize(mapper_file)
+      @mapper_file = mapper_file
+    end
+
+    def map(execution_context, data_sources, data_sink = InMemoryDataSink.new('Mapper Output'))
+      Hasta.logger.debug "Starting mapper: #{mapper_file}"
+
+      data_source = CombinedDataSource.new(data_sources)
+      execution_context.execute(mapper_file, data_source, data_sink)
+    end
+  end
+end

data/lib/hasta/reducer.rb

@@ -0,0 +1,29 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/execution_context'
+require 'hasta/in_memory_data_sink'
+require 'hasta/sorted_data_source'
+
+module Hasta
+  # A wrapper for instantiating a reducer from a definition file and invoking it
+  class Reducer
+    attr_reader :reducer_file
+
+    def initialize(reducer_file)
+      @reducer_file = reducer_file
+    end
+
+    def reduce(execution_context, data_source, data_sink = InMemoryDataSink.new("Reducer Output"))
+      Hasta.logger.debug "Starting reducer: #{reducer_file}"
+      execution_context.execute(reducer_file, sorted_data_source(data_source), data_sink)
+
+      data_sink.close
+    end
+
+    private
+
+    def sorted_data_source(data_source)
+      SortedDataSource.new(data_source)
+    end
+  end
+end

data/lib/hasta/resolve_cached_s3_file.rb

@@ -0,0 +1,29 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/cached_s3_file'
+require 'hasta/s3_file_cache'
+
+module Hasta
+  # Retrieves a file from the local cache instead of S3,
+  # or retrieves it from S3 and caches it locally
+  class ResolveCachedS3File
+    def initialize(file_cache, child_resolver)
+      @file_cache = file_cache
+      @child_resolver = child_resolver
+    end
+
+    def resolve(fog_file)
+      resolved = child_resolver.resolve(fog_file)
+      if cached_file = file_cache.get(resolved.fingerprint)
+        Hasta.logger.debug "Retrieved file: #{resolved.s3_uri} from local cache"
+        CachedS3File.new(cached_file, resolved.s3_uri)
+      else
+        file_cache.put(resolved.fingerprint, resolved.body)
+        Hasta.logger.debug "Cached file: #{resolved.s3_uri} locally"
+        resolved
+      end
+    end
+
+    attr_reader :file_cache, :child_resolver
+  end
+end

data/lib/hasta/resolve_filtered_s3_file.rb

@@ -0,0 +1,22 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/filtered_s3_file'
+
+module Hasta
+  # Creates a Hasta filtered S3 file instance given a Fog file
+  class ResolveFilteredS3File
+    def initialize(filter, child_resolver = Hasta::Storage::ResolveS3File)
+      @filter = filter
+      @child_resolver = child_resolver
+    end
+
+    def resolve(fog_file)
+      FilteredS3File.new(child_resolver.resolve(fog_file), filter)
+    end
+
+
+    private
+
+    attr_reader :filter, :child_resolver
+  end
+end

data/lib/hasta/runner.rb

@@ -0,0 +1,32 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/env'
+require 'hasta/execution_context'
+
+module Hasta
+  # Runs a map/reduce job locally
+  class Runner
+    attr_reader :job_name
+
+    def initialize(job_name, mapper, reducer = nil)
+      @job_name = job_name
+      @mapper = mapper
+      @reducer = reducer
+    end
+
+    def run(data_sources, data_sink, ruby_files = [], env = Hasta::Env.new)
+      Hasta.logger.debug "Starting Job: #{job_name}"
+
+      context = ExecutionContext.new(ruby_files, env.setup)
+      if reducer
+        reducer.reduce(context, mapper.map(context, data_sources).data_source, data_sink)
+      else
+        mapper.map(context, data_sources, data_sink)
+      end
+    end
+
+    private
+
+    attr_reader :mapper, :reducer
+  end
+end

data/lib/hasta/s3_data_sink.rb

@@ -0,0 +1,48 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/local_file_path'
+
+module Hasta
+  # Data sink for writing data to S3 storage
+  class S3DataSink
+    attr_reader :s3_uri
+
+    def initialize(s3_uri, combined_storage = Hasta.combined_storage)
+      @s3_uri = s3_uri
+      @combined_storage = combined_storage
+    end
+
+    def <<(line)
+      lines << Hasta.tab_separated_line(line)
+    end
+
+    def close
+      storage_uri = combined_storage.write(s3_uri, contents)
+      Hasta.logger.debug(
+        "Wrote #{lines.count} lines to uri: #{storage_uri} (#{LocalFilePath.for(storage_uri)})"
+      )
+
+      self
+    end
+
+    def data_source
+      S3DataSource.new(s3_uri, combined_storage)
+    end
+
+    def to_s
+      "#<#{self.class.name}:#{s3_uri}>"
+    end
+
+    private
+
+    attr_reader :combined_storage
+
+    def lines
+      @lines ||= []
+    end
+
+    def contents
+      lines.join("\n")
+    end
+  end
+end

data/lib/hasta/s3_data_source.rb

@@ -0,0 +1,41 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'hasta/s3_uri'
+require 'hasta/combined_storage'
+
+module Hasta
+  # Data source for reading data from S3
+  class S3DataSource
+    def initialize(s3_uri, combined_storage = Hasta.combined_storage)
+      @s3_uri = s3_uri
+      @combined_storage = combined_storage
+    end
+
+    def name
+      s3_uri.to_s
+    end
+
+    def each_line
+      return enum_for(:each_line) unless block_given?
+
+      combined_storage.files_for(s3_uri).each do |file|
+        Hasta.logger.debug(
+          "Processing #{file.remote? ? 'remote' : 'local'} #{file.class}: #{file.key}"
+        )
+        file.each_line { |line| yield line }
+      end
+    end
+
+    def to_a
+      each_line.to_a
+    end
+
+    def to_s
+      "#<#{self.class.name}:#{s3_uri}>"
+    end
+
+    private
+
+    attr_reader :s3_uri, :combined_storage
+  end
+end

data/lib/hasta/s3_file.rb

@@ -0,0 +1,56 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+require 'forwardable'
+
+module Hasta
+  # Hasta's interface to the File objects returned by Fog
+  class S3File
+    extend Forwardable
+
+    def_delegators :s3_file, :key, :body
+
+    def self.wrap_files(s3_files)
+      s3_files.map { |s3_file| wrap(s3_file) }
+    end
+
+    def self.wrap(s3_file)
+      if self === s3_file
+        s3_file
+      elsif s3_file.nil?
+        nil
+      else
+        new(s3_file)
+      end
+    end
+
+    def initialize(s3_file)
+      @s3_file = s3_file
+    end
+
+    def s3_uri
+      @s3_uri ||= S3URI.new(s3_file.directory.key, key)
+    end
+
+    def fingerprint
+      @fingerprint ||= if s3_file.respond_to? :etag
+        s3_file.etag
+      else
+        Digest::MD5.hexdigest(body)
+      end
+    end
+
+    def remote?
+      !(Fog::Storage::Local::File === s3_file)
+    end
+
+    def each_line
+      return enum_for(:each_line) unless block_given?
+
+      StringIO.new(s3_file.body).each_line { |line| yield line }
+    end
+
+    private
+
+    attr_reader :s3_file
+  end
+end