exel 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ddd9afec5ea8298052cc0ee3aee03dedb9c1e085
-  data.tar.gz: 51c89e0ccad9b2be2dbc8961c52d0c80fad5fe24
+  metadata.gz: 11c8c9e8fb6b5a93a48d76c078b9a257c6996454
+  data.tar.gz: ae102849b1ac9bde80efee667eba44f09dd1fe3a
 SHA512:
-  metadata.gz: 81c0b0b05b1d5fbd01de78c75e342e7a4beb557d9c4bc70ff5aed5835e50c108e6f3326b6fc095e15375fc38a0872decaa6df63d2f627cbcaa26a6852a12c144
-  data.tar.gz: 3628008c5f8479ef6dab87a5229bc024117b49a3ba668a16e451f1dbdf0b216b323bd8ac80a4154eda090ac0db1c322e805ef6861a07f21c881224577e068979
+  metadata.gz: 85a10b1419674989f942ba83a588257e7ad4130def0e4e9d79d7c3f3ee62d9a78f2b5b7befd3f6f9e0f2e46bbc71a545760b2136a284e8c014417029bcbdceca
+  data.tar.gz: a3cac79e65e8d06058f64ee4778b2d8182528b52d89942f1b5674aa65c8904cc754a0d019719b992ced510f3e237a6eeafb005e07348364be0487cc29da18f5f

data/.rubocop.yml

@@ -1,22 +1,23 @@
 require:
-  - rubocop-rspec
   - rubocop/rspec/focused
 
 inherit_from: .rubocop_todo.yml
 
 AllCops:
+  TargetRubyVersion: 2.3
   DisplayCopNames: true
   DisplayStyleGuide: true
 
 Metrics/LineLength:
   Max: 120
 
+Style/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+  IndentationWidth: 4
+
 Style/SpaceInsideHashLiteralBraces:
   EnforcedStyle: no_space
 
-RSpec/DescribedClass:
-  Enabled: false
-
 Metrics/ModuleLength:
   Exclude:
     - 'spec/**/*'

data/.rubocop_todo.yml

@@ -1,23 +1,11 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2016-03-09 23:07:44 -0500 using RuboCop version 0.37.2.
+# on 2016-10-14 22:09:13 -0400 using RuboCop version 0.39.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 5
-RSpec/AnyInstance:
-  Exclude:
-    - 'spec/exel/processors/split_processor_spec.rb'
-    - 'spec/exel/value_spec.rb'
-    - 'spec/integration/integration_spec.rb'
-
-# Offense count: 1
-RSpec/InstanceVariable:
-  Exclude:
-    - 'spec/exel/logging_spec.rb'
-
 # Offense count: 1
 Style/Documentation:
   Exclude:

data/Gemfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in exel.gemspec

data/Guardfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 group :rspec_rubocop, halt_on_fail: true do
   guard :rspec, cmd: 'bundle exec rspec' do
     require 'guard/rspec/dsl'

data/README.md

@@ -35,15 +35,17 @@ Or install it yourself as:
 
 A processor can be any class that provides the following interface:
 
-    class MyProcessor
-        def initialize(context)
-            # typically context is assigned to @context here
-        end
-        
-        def process(block)
-            # do your work here
-        end
-    end
+```ruby
+class MyProcessor
+  def initialize(context)
+    # typically context is assigned to @context here
+  end
+
+  def process(block)
+    # do your work here
+  end
+end
+```
 
 Processors are initialized immediately before ```#process``` is called, allowing them to set up any state that they need from the context. The ```#process``` method is where your processing logic will be implemented. Processors should be focused on performing one particular aspect of the processing that you want to accomplish, allowing your job to be composed of a sequence of small processing steps. If a block was given in the call to ```process``` in the job DSL, it will be passed as the argument to ```#process``` and can be run with: ```block.run(@context)```
 
@@ -59,35 +61,98 @@ If you use EXEL with an async provider, such as [exel-sidekiq](https://github.co
 
 ### Supported Instructions
 
-* ```process``` Execute the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
-* ```split``` Split the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
-* ```async``` Asynchronously run the given block. Uses the configured async provider to execute the block.
+* ```process``` Executes the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
+* ```split``` Splits the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
+* ```async``` Asynchronously runs the given block. Uses the configured async provider to execute the block.
 * ```run``` Runs the job specified by the ```:job``` option. The job will run using the current context.
+* ```listen``` Registers an event listener. See the [Events](#events) section below for more detail.
 
 ### Example job
 
-    EXEL::Job.define :example_job do
-        # Download a large CSV data file
-        process with: FTPDownloader, host: ftp.example.com, path: context[:file_path]
-        
-        # split it into smaller 1000 line files
-        split do
-            # for each file asynchronously run the following sequence of processors
-            async do  
-                process with: RecordLoader # convert each row of data into your domain model
-                process with: SomeProcessor # apply some additional processing to each record
-                process with: RecordSaver # write this batch of records to your database
-                process with: ExternalServiceProcessor # interact with some service, ex: updating a search index
-            end
-        end
+```ruby
+EXEL::Job.define :example_job do
+  # Download a large CSV data file
+  process with: FTPDownloader, host: ftp.example.com, path: context[:file_path]
+
+  # split it into smaller 1000 line files
+  split do
+    # for each file asynchronously run the following sequence of processors
+    async do  
+      process with: RecordLoader # convert each row of data into your domain model
+      process with: SomeProcessor # apply some additional processing to each record
+      process with: RecordSaver # write this batch of records to your database
+      process with: ExternalServiceProcessor # interact with some service, ex: updating a search index
     end
+  end
+end
+```
 
 Elsewhere in your application, you could run this job as follows:
 
-    def run_example_job(file_path)
-        context = EXEL::Context.new(file_path: file_path, user: 'username')
-        EXEL::Job.run(:example_job, context)
-    end
+```ruby
+def run_example_job(file_path)
+  # context can also be passed as a Hash
+  context = EXEL::Context.new(file_path: file_path, user: 'username')
+  EXEL::Job.run(:example_job, context)
+end
+```
+
+### Events
+
+Event listeners can be registered using the ```listen``` instruction:
+
+```ruby
+listen for: :my_event, with: MyEventListener
+```
+
+The event listener must implement a method with the same name as the event which accepts two arguments: the context and any data passed when the event was triggered:
+
+```ruby
+class MyEventListener
+  def self.my_event(context, data)
+    # handle event
+  end
+end
+```
+
+To trigger an event, include the ```EXEL::Events``` module and call #trigger with the event name and data:
+
+```ruby
+include EXEL::Events
+
+def process(_block)
+  # trigger event and optionally pass data to the event listener
+  trigger :my_event, foo: 'bar'
+end
+```
+
+### Middleware
+
+Middleware is code configured to run around each processor execution. It is modelled after [Rack](https://github.com/rack/rack) and [Sidekiq](https://github.com/mperham/sidekiq). Custom middleware can be added as follows:
+
+```ruby
+EXEL.configure do |config|
+  config.middleware.add(MyMiddleware)
+  config.middleware.add(AnotherMiddleware, 'constructor arg')
+end
+```
+  
+Middleware can be any class that implements a ```call``` method that includes a call to ```yield```:
+
+```ruby
+class MyMiddleware
+  def call(processor_class, context, args)
+    puts 'before process'
+    
+    # must yield so other middleware and processor will run
+    yield
+    
+    puts 'after process'
+  end
+end
+```
+  
+The ```call``` method will be passed the class of the processor that will be executed, the current context, and any args that were passed to the processor in the job definition.
 
 ## Contributing
 

data/Rakefile

@@ -1 +1,2 @@
+# frozen_string_literal: true
 require 'bundler/gem_tasks'

data/exel.gemspec

@@ -1,4 +1,5 @@
 # coding: utf-8
+# frozen_string_literal: true
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'exel/version'
@@ -26,8 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'guard-rubocop', '~> 1'
   spec.add_development_dependency 'terminal-notifier', '~> 1'
   spec.add_development_dependency 'terminal-notifier-guard', '~> 1'
-  spec.add_development_dependency 'rubocop', '~> 0.37.0'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1'
+  spec.add_development_dependency 'rubocop', '~> 0.39.0'
   spec.add_development_dependency 'rubocop-rspec-focused', '~> 0'
   spec.add_development_dependency 'pry-byebug'
 end

data/lib/exel.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'exel/version'
 require 'exel/logging'
 require 'ostruct'
@@ -18,7 +19,7 @@ module EXEL
 
   # @return The current configuration
   def self.configuration
-    @config ||= OpenStruct.new
+    @config ||= OpenStruct.new(middleware: Middleware::Chain.new)
   end
 
   # Yields the configuration object to the given block. Configuration can include:
@@ -46,6 +47,10 @@ module EXEL
     configuration.remote_provider || Providers::LocalFileProvider
   end
 
+  def self.middleware
+    configuration.middleware
+  end
+
   root = File.expand_path('../..', __FILE__)
   Dir[File.join(root, 'lib/exel/**/*.rb')].each { |file| require file }
 end

data/lib/exel/ast_node.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module EXEL
   # An abstract class that serves as the parent class of nodes in the AST
   class ASTNode
@@ -9,7 +10,9 @@ module EXEL
     end
 
     def start(context)
-      fail_silently { run(context) }
+      run(context)
+    rescue EXEL::Error::JobTermination => e
+      EXEL.logger.send(e.cmd, "JobTerminationError: #{e.message.chomp}")
     end
 
     def run(_context)
@@ -19,13 +22,5 @@ module EXEL
     def add_child(node)
       @children << node
     end
-
-    private
-
-    def fail_silently(&_block)
-      yield if block_given?
-    rescue EXEL::Error::JobTermination => e
-      EXEL.logger.error "JobTerminationError: #{e.message.chomp}"
-    end
   end
 end

data/lib/exel/context.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'tempfile'
 
 module EXEL

data/lib/exel/deferred_context_value.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module EXEL
   # When +context+ is referenced in a job definition, an instance of +DeferredContextValue+ will be put in its place.
   # At runtime, the first time a +DeferredContextValue+ is read via {EXEL::Context#[]}, it will be replaced by the value

data/lib/exel/error/job_termination.rb

@@ -1,8 +1,17 @@
+# frozen_string_literal: true
 module EXEL
   module Error
     # If a processor raises a JobTermination exception, the job will immediately stop running without raising anything.
     # This is useful if you want to stop a job without triggering any kind of retry mechanism, for example.
     class JobTermination < Exception # Inherit from Exception so it won't be rescued and can propagate to ASTNode#start
+      attr_reader :cmd
+
+      CMDS = [:info, :warn, :error].freeze
+
+      def initialize(message = nil, cmd = :error)
+        super(message)
+        @cmd = CMDS.include?(cmd) ? cmd : :error
+      end
     end
   end
 end

data/lib/exel/events.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module EXEL
   # Provides methods for registering and triggering event listeners
   module Events

data/lib/exel/instruction.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module EXEL
   # Represents one step to be executed in the processing of a job
   class Instruction
@@ -9,7 +10,8 @@ module EXEL
 
     def execute(context)
       context.merge!(@args)
-      @processor_class.new(context).process(@subtree)
+      processor = @processor_class.new(context)
+      EXEL.middleware.invoke(@processor_class, context, @args) { processor.process(@subtree) }
     end
   end
 end

data/lib/exel/instruction_node.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require_relative './ast_node'
 
 module EXEL

data/lib/exel/job.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module EXEL
   # The +Job+ module provides the main interface for defining and running EXEL jobs
   module Job

data/lib/exel/listen_instruction.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require_relative 'events'
 
 module EXEL

data/lib/exel/logging.rb

@@ -1,9 +1,17 @@
+# frozen_string_literal: true
 require 'logger'
 
 module EXEL
   module Logging
     DEFAULT_LEVEL = :info
 
+    # Formats log messages with timestamp, severity and Logging prefix (if set via {Logging.with_prefix})
+    class PrefixFormatter < Logger::Formatter
+      def call(severity, time, _program_name, message)
+        "#{time.utc} severity=#{severity}, #{Logging.prefix}#{message}\n"
+      end
+    end
+
     def self.logger
       @logger || initialize_logger
     end
@@ -11,6 +19,7 @@ module EXEL
     def self.initialize_logger
       @logger = Logger.new(log_filename)
       @logger.level = log_level
+      @logger.formatter = PrefixFormatter.new
       @logger
     end
 
@@ -24,7 +33,19 @@ module EXEL
     end
 
     def self.logger=(logger)
-      @logger = logger || Logger.new('/dev/null')
+      @logger = logger ? LoggerWrapper.new(logger) : Logger.new('/dev/null')
+    end
+
+    # Sets a prefix to be added to any messages sent to the EXEL logger in the given block.
+    def self.with_prefix(prefix)
+      Thread.current[:exel_log_prefix] = prefix
+      yield
+    ensure
+      Thread.current[:exel_log_prefix] = nil
+    end
+
+    def self.prefix
+      Thread.current[:exel_log_prefix]
     end
   end
 end

data/lib/exel/logging/logger_wrapper.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+module EXEL
+  module Logging
+    # Wraps calls to a logger to add {Logging} prefix to log messages
+    class LoggerWrapper < SimpleDelegator
+      LOG_LEVELS = %i(debug info warn error fatal unknown).freeze
+
+      LOG_LEVELS.each do |level|
+        define_method level do |progname = nil, &block|
+          prefix_block = nil
+
+          if block
+            prefix_block = proc { "#{Logging.prefix}#{block.call}" }
+          else
+            progname = "#{Logging.prefix}#{progname}"
+          end
+
+          __getobj__.send(level, progname, &prefix_block)
+        end
+      end
+
+      def add(severity, message = nil, progname = nil)
+        message = yield if message.nil? && block_given?
+        __getobj__.add(severity, "#{Logging.prefix}#{message}", progname)
+      end
+    end
+  end
+end