exel 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c9430975944c61a336b74ec445fc85836c64827
-  data.tar.gz: c0189c97975d51869f45300c7a08d50a6f41c0f3
+  metadata.gz: 6bb6f49bf87e5bdf9dab0634d6ba7ea975335aed
+  data.tar.gz: 661b46455bab58722f422f341f2470ded9519853
 SHA512:
-  metadata.gz: 5391eead9f34042b4e4e4fb22c5d19871b69a51371691ccea5d869a664d0a2d6f2dca408cb787eb0b1342b21a97250079e4551cd4bca8579b63012f8a39d3ae2
-  data.tar.gz: 46d654603de7860071251878716647dfcce4199ab858cb6d852156dc79f3258d4486e5202e52b543af53141d8dfda7e4e64e553ca9636a8648c7b60c2797ae96
+  metadata.gz: 359c314ec2a54d8e229738a44e0a243636c644fb451e82a013a26f04317bd3964ddefbe982781ae76abb40a3550ce34c457b1663a70f371b57e0a5183e01d195
+  data.tar.gz: c3a4b9799e38d4b4d8f2af6ab62141001b71377faaea6eac83caed5cbe647755f5cbd5c8a1322b1a576f4ab688ced6972c8cfbfe57bff3c39dfed8326d4c2c76

data/.rubocop.yml

@@ -5,6 +5,7 @@ require:
 inherit_from: .rubocop_todo.yml
 
 AllCops:
+  DisplayCopNames: true
   DisplayStyleGuide: true
 
 Metrics/LineLength:

data/.rubocop_todo.yml

@@ -1,32 +1,26 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2015-12-13 18:06:06 -0500 using RuboCop version 0.35.1.
+# on 2016-03-09 23:07:44 -0500 using RuboCop version 0.37.2.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 65
-# Configuration parameters: CustomTransform, IgnoredWords.
-RSpec/ExampleWording:
+# Offense count: 5
+RSpec/AnyInstance:
   Exclude:
-    - 'spec/exel/ast_node_spec.rb'
-    - 'spec/exel/context_spec.rb'
-    - 'spec/exel/deferred_context_value_spec.rb'
-    - 'spec/exel/handlers/s3_handler_spec.rb'
-    - 'spec/exel/instruction_node_spec.rb'
-    - 'spec/exel/instruction_spec.rb'
-    - 'spec/exel/logging_spec.rb'
     - 'spec/exel/processors/split_processor_spec.rb'
-    - 'spec/exel/resource_spec.rb'
-    - 'spec/exel/sequence_node_spec.rb'
+    - 'spec/exel/value_spec.rb'
+    - 'spec/integration/integration_spec.rb'
 
 # Offense count: 1
 RSpec/InstanceVariable:
   Exclude:
     - 'spec/exel/logging_spec.rb'
 
-# Offense count: 18
-# Configuration parameters: Exclude.
+# Offense count: 1
 Style/Documentation:
-  Enabled: false
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/exel/logging.rb'

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in exel.gemspec
 gemspec
+
+gem 'codeclimate-test-reporter', group: :test, require: nil

data/README.md

@@ -1,7 +1,9 @@
 # EXEL
 [![Gem Version](https://badge.fury.io/rb/exel.svg)](https://badge.fury.io/rb/exel)
 [![Code Climate](https://codeclimate.com/github/47colborne/exel/badges/gpa.svg)](https://codeclimate.com/github/47colborne/exel)
+[![Test Coverage](https://codeclimate.com/github/47colborne/exel/badges/coverage.svg)](https://codeclimate.com/github/47colborne/exel/coverage)
 [![Build Status](https://snap-ci.com/47colborne/exel/branch/master/build_image)](https://snap-ci.com/47colborne/exel/branch/master)
+[![Documentation](http://img.shields.io/badge/docs-rdoc.info-blue.svg)](http://www.rubydoc.info/github/47colborne/exel/master)
 
 EXEL is the Elastic eXEcution Language, a simple Ruby DSL for creating processing jobs that can be run on a single machine, or scaled up to run on dozens of machines with no changes to the job itself. To run a job on more than one machine, simply install EXEL async and remote provider gems to integrate with your preferred platforms. The currently implemented providers so far are:
 
@@ -53,13 +55,14 @@ The ```Context``` class has a Hash-like interface and acts as shared storage for
 * Arguments passed to processors in the job DSL
 * Outputs assigned by processors during processing
 
-If you use EXEL with an async provider, such as [exel-sidekiq](https://github.com/47colborne/exel-sidekiq), and a remote provider, such as [exel-s3](https://github.com/47colborne/exel-s3), a context switch will occur when the ```async``` command is executed. Context shifts involve serializing the context and uploading it via the remote provider, then downloading and deserializing it when the async block is eventually run. This allows the processors to pass the results of their process through the sequence of processors in the job, without having to be concerned with when, where, or how those processors will be run.
+If you use EXEL with an async provider, such as [exel-sidekiq](https://github.com/47colborne/exel-sidekiq), and a remote provider, such as [exel-s3](https://github.com/47colborne/exel-s3), a context switch will occur when the ```async``` instruction is executed. Context shifts involve serializing the context and uploading it via the remote provider, then downloading and deserializing it when the async block is eventually run. This allows the processors to pass the results of their process through the sequence of processors in the job, without having to be concerned with when, where, or how those processors will be run.
 
-### Supported Commands
+### Supported Instructions
 
 * ```process``` Execute the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
 * ```split``` Split the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
 * ```async``` Asynchronously run the given block. Uses the configured async provider to execute the block.
+* ```run``` Runs the job specified by the ```:job``` option. The job will run using the current context.
 
 ### Example job
 

data/exel.gemspec

@@ -26,7 +26,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'guard-rubocop', '~> 1'
   spec.add_development_dependency 'terminal-notifier', '~> 1'
   spec.add_development_dependency 'terminal-notifier-guard', '~> 1'
-  spec.add_development_dependency 'rubocop', '~> 0'
+  spec.add_development_dependency 'rubocop', '~> 0.37.0'
   spec.add_development_dependency 'rubocop-rspec', '~> 1'
   spec.add_development_dependency 'rubocop-rspec-focused', '~> 0'
+  spec.add_development_dependency 'pry-byebug'
 end

data/lib/exel/ast_node.rb

@@ -1,4 +1,5 @@
 module EXEL
+  # An abstract class that serves as the parent class of nodes in the AST
   class ASTNode
     attr_reader :instruction, :children
 
@@ -12,7 +13,7 @@ module EXEL
     end
 
     def run(_context)
-      fail "#{self.class} does not implement #process"
+      raise "#{self.class} does not implement #process"
     end
 
     def add_child(node)

data/lib/exel/context.rb

@@ -1,54 +1,67 @@
 require 'tempfile'
 
 module EXEL
-  class Context
-    attr_reader :table
-
+  # The +Context+ is the shared memory of a running job. It acts as the source of input to processors and the place for
+  # them to store their outputs. It can be serialized and deserialized to support remote execution.
+  class Context < Hash
+    # Accepts an optional hash of keys and values to initialize the context with.
     def initialize(initial_context = {})
-      @table = initial_context
+      super()
+      merge!(initial_context)
     end
 
+    # Returns a deep copy of this context. The copy and the original will have no shared object references.
+    #
+    # @return [Context]
     def deep_dup
       Context.deserialize(serialize)
     end
 
+    # Serializes this instance to a local file and uses the remote provider to upload it. Returns a URI indicating where
+    # the serialized context can be downloaded.
+    #
+    # @return [String] A URI such as +s3://bucket/file+, +file:///path/to/file+, etc.
     def serialize
       EXEL::Value.remotize(serialized_context)
     end
 
+    # Given a string representing the URI to a serialized context, downloads and returns the deserialized context
+    #
+    # @return [Context]
+    # rubocop:disable Metrics/MethodLength
     def self.deserialize(uri)
       file = EXEL::Value.localize(uri)
-      context = Marshal.load(file.read)
-      file.close
-      context
-    end
 
-    def [](key)
-      value = EXEL::Value.localize(@table[key])
-      value = get_deferred(value)
-      @table[key] = value
-      value
-    end
+      begin
+        context = Marshal.load(file.read)
+      rescue
+        # temporarily in place for backwards compatibility
 
-    def []=(key, value)
-      @table[key] = value
-    end
+        dir = File.expand_path('..', __FILE__)
 
-    def merge!(hash)
-      @table.merge!(hash)
-      self
-    end
+        EXEL.send(:remove_const, :Context)
+        load File.join(dir, 'old_context.rb')
+
+        context = Context.deserialize(uri)
 
-    def delete(key)
-      @table.delete(key)
+        EXEL.send(:remove_const, :Context)
+        load File.join(dir, 'context.rb')
+      ensure
+        file.close
+      end
+
+      context
     end
+    # rubocop:enable Metrics/MethodLength
 
-    def ==(other)
-      other.is_a?(EXEL::Context) && table == other.table
+    # Returns the value referenced by the given key. If it is a remote value, it will be converted to a local value and
+    # the local value will be returned.
+    def [](key)
+      convert_value!(key, super(key))
     end
 
-    def include?(values)
-      @table.merge(values) == @table
+    def fetch(key)
+      convert_value!(key, super(key))
     end
 
     private
@@ -61,23 +74,13 @@ module EXEL
     end
 
     def remotized_table
-      @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Value.remotize(value) }
-    end
-
-    def get_deferred(value)
-      if deferred?(value)
-        value = value.get(self)
-      elsif value.is_a?(Array)
-        value.map! { |v| get_deferred(v) }
-      elsif value.is_a?(Hash)
-        value.each { |k, v| value[k] = get_deferred(v) }
-      end
-
-      value
+      each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Value.remotize(value) }
     end
 
-    def deferred?(value)
-      value.is_a?(DeferredContextValue)
+    def convert_value!(key, value)
+      value = EXEL::Value.localize(value)
+      value = DeferredContextValue.resolve(value, self)
+      self[key] = value
     end
   end
 end

data/lib/exel/deferred_context_value.rb

@@ -1,16 +1,51 @@
 module EXEL
+  # When +context+ is referenced in a job definition, an instance of +DeferredContextValue+ will be put in its place.
+  # At runtime, the first time a +DeferredContextValue+ is read via {EXEL::Context#[]}, it will be replaced by the value
+  # it was referring to.
+  #
+  # Example:
+  #   process with: MyProcessor, foo: context[:bar]
   class DeferredContextValue
     attr_reader :keys
 
+    class << self
+      # If +value+ is an instance of +DeferredContextValue+, it will be resolved to its actual value in the context. If
+      # it is an +Array+ or +Hash+ all +DeferredContextValue+ instances within it will be resolved. If it is anything
+      # else, it will just be returned.
+      #
+      # @return value, with all +DeferredContextValue+ instances resolved
+      def resolve(value, context)
+        if deferred?(value)
+          value = value.get(context)
+        elsif value.is_a?(Array)
+          value.map! { |v| resolve(v, context) }
+        elsif value.is_a?(Hash)
+          value.each { |k, v| value[k] = resolve(v, context) }
+        end
+
+        value
+      end
+
+      private
+
+      def deferred?(value)
+        value.is_a?(DeferredContextValue)
+      end
+    end
+
     def initialize
       @keys = []
     end
 
+    # Records the keys that will be used to lookup the value from the context at runtime. Supports nested hashes
+    # such as:
+    #   context[:hash1][:hash2][:key]
     def [](key)
       keys << key
       self
     end
 
+    # Given a context, returns the value that this instance was acting as a placeholder for.
     def get(context)
       keys.reduce(context) { |a, e| a[e] }
     end

data/lib/exel/error/job_termination.rb

@@ -1,10 +1,8 @@
 module EXEL
   module Error
-    # Inherit from Exception rather then StandardError
-    # because rescue => e will only catch StandardError
-    # and allow the Exception to propagate to the root
-    # of the job
-    class JobTermination < Exception
+    # If a processor raises a JobTermination exception, the job will immediately stop running without raising anything.
+    # This is useful if you want to stop a job without triggering any kind of retry mechanism, for example.
+    class JobTermination < Exception # Inherit from Exception so it won't be rescued and can propagate to ASTNode#start
     end
   end
 end

data/lib/exel/events.rb

@@ -0,0 +1,26 @@
+module EXEL
+  # Provides methods for registering and triggering event listeners
+  module Events
+    LISTENERS_KEY = :_listeners
+
+    def register_listener(context, event, listener)
+      listeners_for_event(event, context) << listener
+    end
+
+    def trigger(event, data = {})
+      listeners_for_event(event, context).each { |listener| listener.send(event, context, data) }
+    end
+
+    private
+
+    def listeners_for_event(event, context)
+      listeners(context).fetch(event)
+    rescue KeyError
+      listeners(context)[event] = []
+    end
+
+    def listeners(context)
+      context[LISTENERS_KEY] ||= Hash.new([])
+    end
+  end
+end

data/lib/exel/instruction.rb

@@ -1,9 +1,7 @@
 module EXEL
+  # Represents one step to be executed in the processing of a job
   class Instruction
-    attr_reader :name
-
-    def initialize(name, processor_class, args, subtree = nil)
-      @name = name
+    def initialize(processor_class, args, subtree = nil)
       @processor_class = processor_class
       @args = args || {}
       @subtree = subtree

data/lib/exel/instruction_node.rb

@@ -1,6 +1,7 @@
 require_relative './ast_node'
 
 module EXEL
+  # A leaf node in the AST that contains an instruction ({Instruction}, {ListenInstruction}) to be executed
   class InstructionNode < ASTNode
     def run(context)
       @instruction.execute(context)

data/lib/exel/job.rb

@@ -1,18 +1,31 @@
 module EXEL
+  # The +Job+ module provides the main interface for defining and running EXEL jobs
   module Job
     class << self
+      # Registers a new job
+      #
+      # @param job_name [Symbol] A symbol to set as the name of this job. Used to run it later.
+      # @param block A block of code that calls the EXEL DSL methods
       def define(job_name, &block)
-        fail "Job #{job_name.inspect} is already defined" unless registry[job_name].nil?
+        raise "Job #{job_name.inspect} is already defined" unless registry[job_name].nil?
         registry[job_name] = block
       end
 
+      # @return [Hash] A hash of all the defined jobs
       def registry
         @registry ||= {}
       end
 
+      # If given a symbol as the first parameter, it attempts to run a previously registered job using that name.
+      # Alternatively, a string of code can be passed to be parsed and run directly.
+      #
+      # @param dsl_code_or_name [String, Symbol] As a symbol, the name of a registered job. As a string, the EXEL code
+      #   to be run.
+      # @param context [Context, Hash] (Optional) The initial {Context} to be passed to the job.
+      # @raise If no job has been registered with the given name
       def run(dsl_code_or_name, context = {})
-        context = EXEL::Context.new(context) if context.is_a?(Hash)
-        (ast = parse(dsl_code_or_name)) ? ast.start(context) : fail(%(Job "#{dsl_code_or_name}" not found))
+        context = EXEL::Context.new(context) if context.instance_of?(Hash)
+        (ast = parse(dsl_code_or_name)) ? ast.start(context) : raise(%(Job "#{dsl_code_or_name}" not found))
       end
 
       private
@@ -27,6 +40,7 @@ module EXEL
       end
     end
 
+    # Defines the EXEL DSL methods and is used to convert a block of Ruby code into an abstract syntax tree (AST)
     class Parser
       attr_reader :ast
 
@@ -46,19 +60,24 @@ module EXEL
 
       def process(options, &block)
         processor_class = options.delete(:with)
-        add_instruction_node('process', processor_class, block, options)
+        add_instruction_node(processor_class, parse(block), options)
       end
 
       def async(options = {}, &block)
-        add_instruction_node('async', Processors::AsyncProcessor, block, options)
+        add_instruction_node(Processors::AsyncProcessor, parse(block), options)
       end
 
       def split(options = {}, &block)
-        add_instruction_node('split', Processors::SplitProcessor, block, options)
+        add_instruction_node(Processors::SplitProcessor, parse(block), options)
       end
 
       def run(options = {}, &block)
-        add_instruction_node('run', Processors::RunProcessor, block, options)
+        add_instruction_node(Processors::RunProcessor, parse(block), options)
+      end
+
+      def listen(options)
+        instruction = ListenInstruction.new(options.fetch(:for), options.fetch(:with))
+        @ast.add_child(InstructionNode.new(instruction))
       end
 
       def context
@@ -67,9 +86,12 @@ module EXEL
 
       private
 
-      def add_instruction_node(name, processor, block, args = {})
-        sub_tree = block.nil? ? nil : Parser.parse(block)
-        instruction = EXEL::Instruction.new(name, processor, args, sub_tree)
+      def parse(block)
+        block.nil? ? nil : Parser.parse(block)
+      end
+
+      def add_instruction_node(processor, sub_tree, args = {})
+        instruction = EXEL::Instruction.new(processor, args, sub_tree)
         node = sub_tree.nil? ? InstructionNode.new(instruction) : InstructionNode.new(instruction, [sub_tree])
         @ast.add_child(node)
       end

data/lib/exel/listen_instruction.rb

@@ -0,0 +1,17 @@
+require_relative 'events'
+
+module EXEL
+  # Registers an event listener
+  class ListenInstruction
+    include EXEL::Events
+
+    def initialize(event, listener)
+      @event = event
+      @listener = listener
+    end
+
+    def execute(context)
+      register_listener(context, @event, @listener)
+    end
+  end
+end

data/lib/exel/null_instruction.rb

@@ -1,4 +1,5 @@
 module EXEL
+  # An {Instruction} that does nothing when executed
   class NullInstruction
     def execute(context)
     end

data/lib/exel/old_context.rb

@@ -0,0 +1,109 @@
+require 'tempfile'
+
+module EXEL
+  # This is here for one version for backwards compatibility with already serialized contexts, which can't
+  # be deserialized with the new +Context+ class
+  class Context
+    # Internal hash of keys/values in the context. Use {#[]} and {#[]=} to get and set values instead of this.
+    attr_reader :table
+
+    # Accepts an optional hash of keys and values to initialize the context with.
+    def initialize(initial_context = {})
+      @table = initial_context
+    end
+
+    # Returns a deep copy of this context. The copy and the original will have no shared object references.
+    #
+    # @return [Context]
+    def deep_dup
+      Context.deserialize(serialize)
+    end
+
+    # Serializes this instance to a local file and uses the remote provider to upload it. Returns a URI indicating where
+    # the serialized context can be downloaded.
+    #
+    # @return [String] A URI such as +s3://bucket/file+, +file:///path/to/file+, etc.
+    def serialize
+      EXEL::Value.remotize(serialized_context)
+    end
+
+    # Given a string representing the URI to a serialized context, downloads and returns the deserialized context
+    #
+    # @return [Context]
+    def self.deserialize(uri)
+      file = EXEL::Value.localize(uri)
+      context = Marshal.load(file.read)
+      file.close
+      context
+    end
+
+    def foobar
+      puts 'hi'
+    end
+
+    # Returns the value referenced by the given key
+    def [](key)
+      value = EXEL::Value.localize(@table[key])
+      value = get_deferred(value)
+      @table[key] = value
+      value
+    end
+
+    # Stores the given key/value pair
+    def []=(key, value)
+      @table[key] = value
+    end
+
+    # Adds the given key/value pairs to the context, overriding any keys that are already present.
+    #
+    # @return [Context]
+    def merge!(hash)
+      @table.merge!(hash)
+      self
+    end
+
+    # Removes the value referenced by +key+ from the context
+    def delete(key)
+      @table.delete(key)
+    end
+
+    # Two Contexts are equal if they contain the same key/value pairs
+    def ==(other)
+      other.is_a?(EXEL::Context) && table == other.table
+    end
+
+    # Returns true if this instance contains all of the given key/value pairs
+    def include?(hash)
+      @table.merge(hash) == @table
+    end
+
+    private
+
+    def serialized_context
+      file = Tempfile.new(SecureRandom.uuid, encoding: 'ascii-8bit')
+      file.write(Marshal.dump(Context.new(remotized_table)))
+      file.rewind
+      file
+    end
+
+    def remotized_table
+      @table.each_with_object({}) { |(key, value), acc| acc[key] = EXEL::Value.remotize(value) }
+    end
+
+    def get_deferred(value)
+      if deferred?(value)
+        value = value.get(self)
+      elsif value.is_a?(Array)
+        value.map! { |v| get_deferred(v) }
+      elsif value.is_a?(Hash)
+        value.each { |k, v| value[k] = get_deferred(v) }
+      end
+
+      value
+    end
+
+    def deferred?(value)
+      value.is_a?(DeferredContextValue)
+    end
+  end
+end

data/lib/exel/processor_helper.rb

@@ -1,7 +1,6 @@
 module EXEL
+  # Helper methods useful to processors
   module ProcessorHelper
-    # Helper Methods
-
     def tag(*tags)
       tags.map { |t| "[#{t}]" }.join('')
     end
@@ -14,8 +13,6 @@ module EXEL
       format('%.2f MB', file.size.to_f / 1_024_000)
     end
 
-    # Logging Helpers
-
     def log_prefix_with(prefix)
       @log_prefix = (@context[:log_prefix] || '') + prefix
     end

data/lib/exel/processors/async_processor.rb

@@ -2,6 +2,7 @@ require_relative '../processor_helper'
 
 module EXEL
   module Processors
+    # Implements the +async+ instruction by using the configured async provider to run a block asynchronously.
     class AsyncProcessor
       include EXEL::ProcessorHelper
       attr_reader :provider

data/lib/exel/processors/run_processor.rb

@@ -2,13 +2,16 @@ require_relative '../processor_helper'
 
 module EXEL
   module Processors
+    # Implements the +run+ instruction.
     class RunProcessor
       include EXEL::ProcessorHelper
 
+      # Requires +context[:job]+ to contain the name of the job to be run.
       def initialize(context)
         @context = context
       end
 
+      # Runs the specified job with the current context
       def process(_block = nil)
         log_process "running job #{@context[:job]}" do
           EXEL::Job.run(@context[:job], @context)