exel 1.2.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ddd9afec5ea8298052cc0ee3aee03dedb9c1e085
-  data.tar.gz: 51c89e0ccad9b2be2dbc8961c52d0c80fad5fe24
+SHA256:
+  metadata.gz: 546280a360b5092ca3cca1b0e0ae670560c60fb87e0c143b04405f9a5a3de9f9
+  data.tar.gz: 1b4b4f78ae5797c82f12abdd61057f915d1f06f53e775b4b1f502b06e1bea463
 SHA512:
-  metadata.gz: 81c0b0b05b1d5fbd01de78c75e342e7a4beb557d9c4bc70ff5aed5835e50c108e6f3326b6fc095e15375fc38a0872decaa6df63d2f627cbcaa26a6852a12c144
-  data.tar.gz: 3628008c5f8479ef6dab87a5229bc024117b49a3ba668a16e451f1dbdf0b216b323bd8ac80a4154eda090ac0db1c322e805ef6861a07f21c881224577e068979
+  metadata.gz: 0117f598e372c74606421159b405fdd4aa4fa1be3d2d658741ed4163b8c223cc3f75ce68ab0e67adbd495cd53a72b47b8411cf74135b067f2c430f00a3ff70da
+  data.tar.gz: 7a40ed7071070e4799681b750595c78b07326d72859f3824b9e411d94a782147fc94101b0c9466795270b41126a0ad0bbdb3a137185db7ae0ef825a90f3f5f5d

data/.codeclimate.yml

@@ -4,13 +4,13 @@ engines:
     enabled: true
     config:
       languages:
-      - ruby
+        - ruby
   fixme:
     enabled: true
   rubocop:
-    enabled: true
+    enabled: false
 ratings:
   paths:
-  - "**.rb"
+    - "**.rb"
 exclude_paths:
-- spec/**/*
+  - spec/**/*

data/.gitignore

@@ -3,7 +3,6 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage
@@ -22,4 +21,4 @@ tmp
 mkmf.log
 .idea
 *.iml
-.ruby-*
+.ruby-*

data/.rubocop.yml

@@ -1,22 +1,31 @@
-require:
-  - rubocop-rspec
-  - rubocop/rspec/focused
-
-inherit_from: .rubocop_todo.yml
+inherit_from:
+  - .rubocop_airbnb.yml
+  - .rubocop_todo.yml
 
 AllCops:
+  TargetRubyVersion: 2.7
   DisplayCopNames: true
   DisplayStyleGuide: true
 
-Metrics/LineLength:
-  Max: 120
+Airbnb/OptArgParameters:
+  Exclude:
+    - "lib/exel/processor_helper.rb"
+    - "lib/exel/logging/logger_wrapper.rb"
+    - "lib/exel/processors/*.rb"
+    - "lib/exel/error/job_termination.rb"
 
-Style/SpaceInsideHashLiteralBraces:
-  EnforcedStyle: no_space
+Layout/DotPosition:
+  EnforcedStyle: leading
 
-RSpec/DescribedClass:
-  Enabled: false
+Layout/IndentFirstArrayElement:
+  IndentationWidth: 4
 
-Metrics/ModuleLength:
-  Exclude:
-    - 'spec/**/*'
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+  IndentationWidth: 4
+
+Layout/SpaceInsideHashLiteralBraces:
+  EnforcedStyle: no_space
+
+Metrics/LineLength:
+  Max: 120

data/.rubocop_airbnb.yml

@@ -0,0 +1,2 @@
+require:
+  - rubocop-airbnb

data/.rubocop_todo.yml

@@ -1,23 +1,11 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2016-03-09 23:07:44 -0500 using RuboCop version 0.37.2.
+# on 2016-10-14 22:09:13 -0400 using RuboCop version 0.39.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 5
-RSpec/AnyInstance:
-  Exclude:
-    - 'spec/exel/processors/split_processor_spec.rb'
-    - 'spec/exel/value_spec.rb'
-    - 'spec/integration/integration_spec.rb'
-
-# Offense count: 1
-RSpec/InstanceVariable:
-  Exclude:
-    - 'spec/exel/logging_spec.rb'
-
 # Offense count: 1
 Style/Documentation:
   Exclude:

data/.travis.yml

@@ -0,0 +1,26 @@
+env:
+  global:
+    - CC_TEST_REPORTER_ID=29a10c062e6416be84441296b1ec7b212f9c01a252e78b3e3df02cd9fb076abe
+
+language: ruby
+
+before_script:
+  - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+  - chmod +x ./cc-test-reporter
+  - ./cc-test-reporter before-build
+
+script: bundle exec rspec
+
+after_script:
+  - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
+
+rvm:
+  - 2.6
+  - 2.7
+
+notifications:
+  email:
+    recipients:
+      - dev@yroo.com
+    on_success: change
+    on_failure: always

data/Gemfile

@@ -1,6 +1,6 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in exel.gemspec
 gemspec
-
-gem 'codeclimate-test-reporter', group: :test, require: nil

data/Gemfile.lock

@@ -0,0 +1,118 @@
+PATH
+  remote: .
+  specs:
+    exel (1.5.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    byebug (11.1.3)
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    docile (1.3.2)
+    ffi (1.12.2)
+    formatador (0.2.5)
+    guard (2.16.2)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    guard-rubocop (1.3.0)
+      guard (~> 2.0)
+      rubocop (~> 0.20)
+    jaro_winkler (1.5.4)
+    listen (3.2.1)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    lumberjack (1.2.4)
+    method_source (1.0.0)
+    nenv (0.3.0)
+    notiffany (0.1.3)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parallel (1.19.1)
+    parser (2.7.1.1)
+      ast (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    rack (2.2.2)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.76.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-airbnb (3.0.2)
+      rubocop (~> 0.76.0)
+      rubocop-performance (~> 1.5.0)
+      rubocop-rails (~> 2.3.2)
+      rubocop-rspec (~> 1.30.0)
+    rubocop-performance (1.5.2)
+      rubocop (>= 0.71.0)
+    rubocop-rails (2.3.2)
+      rack (>= 1.1)
+      rubocop (>= 0.72.0)
+    rubocop-rspec (1.30.1)
+      rubocop (>= 0.60.0)
+    ruby-progressbar (1.10.1)
+    shellany (0.0.1)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
+    terminal-notifier (1.6.3)
+    terminal-notifier-guard (1.7.0)
+    thor (1.0.1)
+    unicode-display_width (1.6.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  exel!
+  guard (~> 2)
+  guard-rspec (~> 4)
+  guard-rubocop (~> 1)
+  pry-byebug
+  rake (~> 13)
+  rspec (~> 3)
+  rubocop-airbnb (~> 3.0)
+  simplecov (~> 0.17)
+  terminal-notifier (~> 1.6.0)
+  terminal-notifier-guard (~> 1.7.0)
+
+BUNDLED WITH
+   2.1.4

data/Guardfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 group :rspec_rubocop, halt_on_fail: true do
   guard :rspec, cmd: 'bundle exec rspec' do
     require 'guard/rspec/dsl'

data/README.md

@@ -2,7 +2,7 @@
 [![Gem Version](https://badge.fury.io/rb/exel.svg)](https://badge.fury.io/rb/exel)
 [![Code Climate](https://codeclimate.com/github/47colborne/exel/badges/gpa.svg)](https://codeclimate.com/github/47colborne/exel)
 [![Test Coverage](https://codeclimate.com/github/47colborne/exel/badges/coverage.svg)](https://codeclimate.com/github/47colborne/exel/coverage)
-[![Build Status](https://snap-ci.com/47colborne/exel/branch/master/build_image)](https://snap-ci.com/47colborne/exel/branch/master)
+[![Build Status](https://travis-ci.org/47colborne/exel.svg?branch=master)](https://travis-ci.org/47colborne/exel)
 [![Documentation](http://img.shields.io/badge/docs-rdoc.info-blue.svg)](http://www.rubydoc.info/github/47colborne/exel/master)
 
 EXEL is the Elastic eXEcution Language, a simple Ruby DSL for creating processing jobs that can be run on a single machine, or scaled up to run on dozens of machines with no changes to the job itself. To run a job on more than one machine, simply install EXEL async and remote provider gems to integrate with your preferred platforms. The currently implemented providers so far are:
@@ -35,15 +35,17 @@ Or install it yourself as:
 
 A processor can be any class that provides the following interface:
 
-    class MyProcessor
-        def initialize(context)
-            # typically context is assigned to @context here
-        end
-        
-        def process(block)
-            # do your work here
-        end
-    end
+```ruby
+class MyProcessor
+  def initialize(context)
+    # typically context is assigned to @context here
+  end
+
+  def process(block)
+    # do your work here
+  end
+end
+```
 
 Processors are initialized immediately before ```#process``` is called, allowing them to set up any state that they need from the context. The ```#process``` method is where your processing logic will be implemented. Processors should be focused on performing one particular aspect of the processing that you want to accomplish, allowing your job to be composed of a sequence of small processing steps. If a block was given in the call to ```process``` in the job DSL, it will be passed as the argument to ```#process``` and can be run with: ```block.run(@context)```
 
@@ -59,35 +61,98 @@ If you use EXEL with an async provider, such as [exel-sidekiq](https://github.co
 
 ### Supported Instructions
 
-* ```process``` Execute the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
-* ```split``` Split the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
-* ```async``` Asynchronously run the given block. Uses the configured async provider to execute the block.
+* ```process``` Executes the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
+* ```split``` Splits the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
+* ```async``` Asynchronously runs the given block. Uses the configured async provider to execute the block.
 * ```run``` Runs the job specified by the ```:job``` option. The job will run using the current context.
+* ```listen``` Registers an event listener. See the [Events](#events) section below for more detail.
 
 ### Example job
 
-    EXEL::Job.define :example_job do
-        # Download a large CSV data file
-        process with: FTPDownloader, host: ftp.example.com, path: context[:file_path]
-        
-        # split it into smaller 1000 line files
-        split do
-            # for each file asynchronously run the following sequence of processors
-            async do  
-                process with: RecordLoader # convert each row of data into your domain model
-                process with: SomeProcessor # apply some additional processing to each record
-                process with: RecordSaver # write this batch of records to your database
-                process with: ExternalServiceProcessor # interact with some service, ex: updating a search index
-            end
-        end
+```ruby
+EXEL::Job.define :example_job do
+  # Download a large CSV data file
+  process with: FTPDownloader, host: ftp.example.com, path: context[:file_path]
+
+  # split it into smaller 1000 line files
+  split do
+    # for each file asynchronously run the following sequence of processors
+    async do  
+      process with: RecordLoader # convert each row of data into your domain model
+      process with: SomeProcessor # apply some additional processing to each record
+      process with: RecordSaver # write this batch of records to your database
+      process with: ExternalServiceProcessor # interact with some service, ex: updating a search index
     end
+  end
+end
+```
 
 Elsewhere in your application, you could run this job as follows:
 
-    def run_example_job(file_path)
-        context = EXEL::Context.new(file_path: file_path, user: 'username')
-        EXEL::Job.run(:example_job, context)
-    end
+```ruby
+def run_example_job(file_path)
+  # context can also be passed as a Hash
+  context = EXEL::Context.new(file_path: file_path, user: 'username')
+  EXEL::Job.run(:example_job, context)
+end
+```
+
+### Events
+
+Event listeners can be registered using the ```listen``` instruction:
+
+```ruby
+listen for: :my_event, with: MyEventListener
+```
+
+The event listener must implement a method with the same name as the event which accepts two arguments: the context and any data passed when the event was triggered:
+
+```ruby
+class MyEventListener
+  def self.my_event(context, data)
+    # handle event
+  end
+end
+```
+
+To trigger an event, include the ```EXEL::Events``` module and call #trigger with the event name and data:
+
+```ruby
+include EXEL::Events
+
+def process(_block)
+  # trigger event and optionally pass data to the event listener
+  trigger :my_event, foo: 'bar'
+end
+```
+
+### Middleware
+
+Middleware is code configured to run around each processor execution. It is modelled after [Rack](https://github.com/rack/rack) and [Sidekiq](https://github.com/mperham/sidekiq). Custom middleware can be added as follows:
+
+```ruby
+EXEL.configure do |config|
+  config.middleware.add(MyMiddleware)
+  config.middleware.add(AnotherMiddleware, 'constructor arg')
+end
+```
+  
+Middleware can be any class that implements a ```call``` method that includes a call to ```yield```:
+
+```ruby
+class MyMiddleware
+  def call(processor_class, context, args)
+    puts 'before process'
+    
+    # must yield so other middleware and processor will run
+    yield
+    
+    puts 'after process'
+  end
+end
+```
+  
+The ```call``` method will be passed the class of the processor that will be executed, the current context, and any args that were passed to the processor in the job definition.
 
 ## Contributing
 

data/Rakefile

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'

data/exel.gemspec

@@ -1,4 +1,6 @@
 # coding: utf-8
+# frozen_string_literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'exel/version'
@@ -18,16 +20,14 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.6'
-  spec.add_development_dependency 'rake', '~> 10'
+  spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'
+  spec.add_development_dependency 'simplecov', '~> 0.17'
   spec.add_development_dependency 'guard', '~> 2'
   spec.add_development_dependency 'guard-rspec', '~> 4'
   spec.add_development_dependency 'guard-rubocop', '~> 1'
-  spec.add_development_dependency 'terminal-notifier', '~> 1'
-  spec.add_development_dependency 'terminal-notifier-guard', '~> 1'
-  spec.add_development_dependency 'rubocop', '~> 0.37.0'
-  spec.add_development_dependency 'rubocop-rspec', '~> 1'
-  spec.add_development_dependency 'rubocop-rspec-focused', '~> 0'
+  spec.add_development_dependency 'terminal-notifier', '~> 1.6.0'
+  spec.add_development_dependency 'terminal-notifier-guard', '~> 1.7.0'
+  spec.add_development_dependency 'rubocop-airbnb', '~> 3.0'
   spec.add_development_dependency 'pry-byebug'
 end