exel 1.2.1 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -4
- data/.rubocop_todo.yml +1 -13
- data/Gemfile +1 -0
- data/Guardfile +1 -0
- data/README.md +95 -30
- data/Rakefile +1 -0
- data/exel.gemspec +2 -2
- data/lib/exel.rb +6 -1
- data/lib/exel/ast_node.rb +4 -9
- data/lib/exel/context.rb +1 -0
- data/lib/exel/deferred_context_value.rb +1 -0
- data/lib/exel/error/job_termination.rb +9 -0
- data/lib/exel/events.rb +1 -0
- data/lib/exel/instruction.rb +3 -1
- data/lib/exel/instruction_node.rb +1 -0
- data/lib/exel/job.rb +1 -0
- data/lib/exel/listen_instruction.rb +1 -0
- data/lib/exel/logging.rb +22 -1
- data/lib/exel/logging/logger_wrapper.rb +28 -0
- data/lib/exel/logging_helper.rb +35 -0
- data/lib/exel/middleware/chain.rb +66 -0
- data/lib/exel/middleware/logging.rb +30 -0
- data/lib/exel/null_instruction.rb +1 -0
- data/lib/exel/processor_helper.rb +7 -0
- data/lib/exel/processors/async_processor.rb +2 -8
- data/lib/exel/processors/run_processor.rb +2 -7
- data/lib/exel/processors/split_processor.rb +5 -8
- data/lib/exel/providers/local_file_provider.rb +1 -0
- data/lib/exel/providers/threaded_async_provider.rb +1 -0
- data/lib/exel/sequence_node.rb +1 -0
- data/lib/exel/value.rb +1 -0
- data/lib/exel/version.rb +2 -1
- data/spec/exel/ast_node_spec.rb +24 -3
- data/spec/exel/context_spec.rb +1 -0
- data/spec/exel/deferred_context_value_spec.rb +1 -0
- data/spec/exel/events_spec.rb +1 -0
- data/spec/exel/instruction_node_spec.rb +1 -0
- data/spec/exel/instruction_spec.rb +6 -0
- data/spec/exel/job_spec.rb +1 -0
- data/spec/exel/listen_instruction_spec.rb +1 -0
- data/spec/exel/logging/logger_wrapper_spec.rb +95 -0
- data/spec/exel/logging_helper_spec.rb +25 -0
- data/spec/exel/logging_spec.rb +36 -3
- data/spec/exel/middleware/chain_spec.rb +67 -0
- data/spec/exel/middleware/logging_spec.rb +33 -0
- data/spec/exel/middleware_spec.rb +69 -0
- data/spec/exel/null_instruction_spec.rb +1 -0
- data/spec/exel/processors/async_processor_spec.rb +1 -0
- data/spec/exel/processors/run_processor_spec.rb +1 -0
- data/spec/exel/processors/split_processor_spec.rb +2 -7
- data/spec/exel/providers/local_file_provider_spec.rb +1 -0
- data/spec/exel/providers/threaded_async_provider_spec.rb +1 -0
- data/spec/exel/sequence_node_spec.rb +1 -0
- data/spec/exel/value_spec.rb +1 -0
- data/spec/exel_spec.rb +1 -0
- data/spec/integration/integration_spec.rb +1 -0
- data/spec/spec_helper.rb +1 -0
- data/spec/support/integration_test_classes.rb +1 -0
- metadata +18 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11c8c9e8fb6b5a93a48d76c078b9a257c6996454
|
4
|
+
data.tar.gz: ae102849b1ac9bde80efee667eba44f09dd1fe3a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85a10b1419674989f942ba83a588257e7ad4130def0e4e9d79d7c3f3ee62d9a78f2b5b7befd3f6f9e0f2e46bbc71a545760b2136a284e8c014417029bcbdceca
|
7
|
+
data.tar.gz: a3cac79e65e8d06058f64ee4778b2d8182528b52d89942f1b5674aa65c8904cc754a0d019719b992ced510f3e237a6eeafb005e07348364be0487cc29da18f5f
|
data/.rubocop.yml
CHANGED
@@ -1,22 +1,23 @@
|
|
1
1
|
require:
|
2
|
-
- rubocop-rspec
|
3
2
|
- rubocop/rspec/focused
|
4
3
|
|
5
4
|
inherit_from: .rubocop_todo.yml
|
6
5
|
|
7
6
|
AllCops:
|
7
|
+
TargetRubyVersion: 2.3
|
8
8
|
DisplayCopNames: true
|
9
9
|
DisplayStyleGuide: true
|
10
10
|
|
11
11
|
Metrics/LineLength:
|
12
12
|
Max: 120
|
13
13
|
|
14
|
+
Style/MultilineMethodCallIndentation:
|
15
|
+
EnforcedStyle: indented
|
16
|
+
IndentationWidth: 4
|
17
|
+
|
14
18
|
Style/SpaceInsideHashLiteralBraces:
|
15
19
|
EnforcedStyle: no_space
|
16
20
|
|
17
|
-
RSpec/DescribedClass:
|
18
|
-
Enabled: false
|
19
|
-
|
20
21
|
Metrics/ModuleLength:
|
21
22
|
Exclude:
|
22
23
|
- 'spec/**/*'
|
data/.rubocop_todo.yml
CHANGED
@@ -1,23 +1,11 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on 2016-
|
3
|
+
# on 2016-10-14 22:09:13 -0400 using RuboCop version 0.39.0.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
8
8
|
|
9
|
-
# Offense count: 5
|
10
|
-
RSpec/AnyInstance:
|
11
|
-
Exclude:
|
12
|
-
- 'spec/exel/processors/split_processor_spec.rb'
|
13
|
-
- 'spec/exel/value_spec.rb'
|
14
|
-
- 'spec/integration/integration_spec.rb'
|
15
|
-
|
16
|
-
# Offense count: 1
|
17
|
-
RSpec/InstanceVariable:
|
18
|
-
Exclude:
|
19
|
-
- 'spec/exel/logging_spec.rb'
|
20
|
-
|
21
9
|
# Offense count: 1
|
22
10
|
Style/Documentation:
|
23
11
|
Exclude:
|
data/Gemfile
CHANGED
data/Guardfile
CHANGED
data/README.md
CHANGED
@@ -35,15 +35,17 @@ Or install it yourself as:
|
|
35
35
|
|
36
36
|
A processor can be any class that provides the following interface:
|
37
37
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
38
|
+
```ruby
|
39
|
+
class MyProcessor
|
40
|
+
def initialize(context)
|
41
|
+
# typically context is assigned to @context here
|
42
|
+
end
|
43
|
+
|
44
|
+
def process(block)
|
45
|
+
# do your work here
|
46
|
+
end
|
47
|
+
end
|
48
|
+
```
|
47
49
|
|
48
50
|
Processors are initialized immediately before ```#process``` is called, allowing them to set up any state that they need from the context. The ```#process``` method is where your processing logic will be implemented. Processors should be focused on performing one particular aspect of the processing that you want to accomplish, allowing your job to be composed of a sequence of small processing steps. If a block was given in the call to ```process``` in the job DSL, it will be passed as the argument to ```#process``` and can be run with: ```block.run(@context)```
|
49
51
|
|
@@ -59,35 +61,98 @@ If you use EXEL with an async provider, such as [exel-sidekiq](https://github.co
|
|
59
61
|
|
60
62
|
### Supported Instructions
|
61
63
|
|
62
|
-
* ```process```
|
63
|
-
* ```split```
|
64
|
-
* ```async``` Asynchronously
|
64
|
+
* ```process``` Executes the given processor class (specified by the ```:with``` option), given the current context and any additional arguments provided
|
65
|
+
* ```split``` Splits the input data into 1000 line chunks and run the given block for each chunk. Assumes that the input data is a CSV formatted file referenced by ```context[:resource]```. When each block is run, ```context[:resource]``` will reference to the chunk file.
|
66
|
+
* ```async``` Asynchronously runs the given block. Uses the configured async provider to execute the block.
|
65
67
|
* ```run``` Runs the job specified by the ```:job``` option. The job will run using the current context.
|
68
|
+
* ```listen``` Registers an event listener. See the [Events](#events) section below for more detail.
|
66
69
|
|
67
70
|
### Example job
|
68
71
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
end
|
72
|
+
```ruby
|
73
|
+
EXEL::Job.define :example_job do
|
74
|
+
# Download a large CSV data file
|
75
|
+
process with: FTPDownloader, host: ftp.example.com, path: context[:file_path]
|
76
|
+
|
77
|
+
# split it into smaller 1000 line files
|
78
|
+
split do
|
79
|
+
# for each file asynchronously run the following sequence of processors
|
80
|
+
async do
|
81
|
+
process with: RecordLoader # convert each row of data into your domain model
|
82
|
+
process with: SomeProcessor # apply some additional processing to each record
|
83
|
+
process with: RecordSaver # write this batch of records to your database
|
84
|
+
process with: ExternalServiceProcessor # interact with some service, ex: updating a search index
|
83
85
|
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
```
|
84
89
|
|
85
90
|
Elsewhere in your application, you could run this job as follows:
|
86
91
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
92
|
+
```ruby
|
93
|
+
def run_example_job(file_path)
|
94
|
+
# context can also be passed as a Hash
|
95
|
+
context = EXEL::Context.new(file_path: file_path, user: 'username')
|
96
|
+
EXEL::Job.run(:example_job, context)
|
97
|
+
end
|
98
|
+
```
|
99
|
+
|
100
|
+
### Events
|
101
|
+
|
102
|
+
Event listeners can be registered using the ```listen``` instruction:
|
103
|
+
|
104
|
+
```ruby
|
105
|
+
listen for: :my_event, with: MyEventListener
|
106
|
+
```
|
107
|
+
|
108
|
+
The event listener must implement a method with the same name as the event which accepts two arguments: the context and any data passed when the event was triggered:
|
109
|
+
|
110
|
+
```ruby
|
111
|
+
class MyEventListener
|
112
|
+
def self.my_event(context, data)
|
113
|
+
# handle event
|
114
|
+
end
|
115
|
+
end
|
116
|
+
```
|
117
|
+
|
118
|
+
To trigger an event, include the ```EXEL::Events``` module and call #trigger with the event name and data:
|
119
|
+
|
120
|
+
```ruby
|
121
|
+
include EXEL::Events
|
122
|
+
|
123
|
+
def process(_block)
|
124
|
+
# trigger event and optionally pass data to the event listener
|
125
|
+
trigger :my_event, foo: 'bar'
|
126
|
+
end
|
127
|
+
```
|
128
|
+
|
129
|
+
### Middleware
|
130
|
+
|
131
|
+
Middleware is code configured to run around each processor execution. It is modelled after [Rack](https://github.com/rack/rack) and [Sidekiq](https://github.com/mperham/sidekiq). Custom middleware can be added as follows:
|
132
|
+
|
133
|
+
```ruby
|
134
|
+
EXEL.configure do |config|
|
135
|
+
config.middleware.add(MyMiddleware)
|
136
|
+
config.middleware.add(AnotherMiddleware, 'constructor arg')
|
137
|
+
end
|
138
|
+
```
|
139
|
+
|
140
|
+
Middleware can be any class that implements a ```call``` method that includes a call to ```yield```:
|
141
|
+
|
142
|
+
```ruby
|
143
|
+
class MyMiddleware
|
144
|
+
def call(processor_class, context, args)
|
145
|
+
puts 'before process'
|
146
|
+
|
147
|
+
# must yield so other middleware and processor will run
|
148
|
+
yield
|
149
|
+
|
150
|
+
puts 'after process'
|
151
|
+
end
|
152
|
+
end
|
153
|
+
```
|
154
|
+
|
155
|
+
The ```call``` method will be passed the class of the processor that will be executed, the current context, and any args that were passed to the processor in the job definition.
|
91
156
|
|
92
157
|
## Contributing
|
93
158
|
|
data/Rakefile
CHANGED
data/exel.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
# frozen_string_literal: true
|
2
3
|
lib = File.expand_path('../lib', __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'exel/version'
|
@@ -26,8 +27,7 @@ Gem::Specification.new do |spec|
|
|
26
27
|
spec.add_development_dependency 'guard-rubocop', '~> 1'
|
27
28
|
spec.add_development_dependency 'terminal-notifier', '~> 1'
|
28
29
|
spec.add_development_dependency 'terminal-notifier-guard', '~> 1'
|
29
|
-
spec.add_development_dependency 'rubocop', '~> 0.
|
30
|
-
spec.add_development_dependency 'rubocop-rspec', '~> 1'
|
30
|
+
spec.add_development_dependency 'rubocop', '~> 0.39.0'
|
31
31
|
spec.add_development_dependency 'rubocop-rspec-focused', '~> 0'
|
32
32
|
spec.add_development_dependency 'pry-byebug'
|
33
33
|
end
|
data/lib/exel.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'exel/version'
|
2
3
|
require 'exel/logging'
|
3
4
|
require 'ostruct'
|
@@ -18,7 +19,7 @@ module EXEL
|
|
18
19
|
|
19
20
|
# @return The current configuration
|
20
21
|
def self.configuration
|
21
|
-
@config ||= OpenStruct.new
|
22
|
+
@config ||= OpenStruct.new(middleware: Middleware::Chain.new)
|
22
23
|
end
|
23
24
|
|
24
25
|
# Yields the configuration object to the given block. Configuration can include:
|
@@ -46,6 +47,10 @@ module EXEL
|
|
46
47
|
configuration.remote_provider || Providers::LocalFileProvider
|
47
48
|
end
|
48
49
|
|
50
|
+
def self.middleware
|
51
|
+
configuration.middleware
|
52
|
+
end
|
53
|
+
|
49
54
|
root = File.expand_path('../..', __FILE__)
|
50
55
|
Dir[File.join(root, 'lib/exel/**/*.rb')].each { |file| require file }
|
51
56
|
end
|
data/lib/exel/ast_node.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module EXEL
|
2
3
|
# An abstract class that serves as the parent class of nodes in the AST
|
3
4
|
class ASTNode
|
@@ -9,7 +10,9 @@ module EXEL
|
|
9
10
|
end
|
10
11
|
|
11
12
|
def start(context)
|
12
|
-
|
13
|
+
run(context)
|
14
|
+
rescue EXEL::Error::JobTermination => e
|
15
|
+
EXEL.logger.send(e.cmd, "JobTerminationError: #{e.message.chomp}")
|
13
16
|
end
|
14
17
|
|
15
18
|
def run(_context)
|
@@ -19,13 +22,5 @@ module EXEL
|
|
19
22
|
def add_child(node)
|
20
23
|
@children << node
|
21
24
|
end
|
22
|
-
|
23
|
-
private
|
24
|
-
|
25
|
-
def fail_silently(&_block)
|
26
|
-
yield if block_given?
|
27
|
-
rescue EXEL::Error::JobTermination => e
|
28
|
-
EXEL.logger.error "JobTerminationError: #{e.message.chomp}"
|
29
|
-
end
|
30
25
|
end
|
31
26
|
end
|
data/lib/exel/context.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module EXEL
|
2
3
|
# When +context+ is referenced in a job definition, an instance of +DeferredContextValue+ will be put in its place.
|
3
4
|
# At runtime, the first time a +DeferredContextValue+ is read via {EXEL::Context#[]}, it will be replaced by the value
|
@@ -1,8 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module EXEL
|
2
3
|
module Error
|
3
4
|
# If a processor raises a JobTermination exception, the job will immediately stop running without raising anything.
|
4
5
|
# This is useful if you want to stop a job without triggering any kind of retry mechanism, for example.
|
5
6
|
class JobTermination < Exception # Inherit from Exception so it won't be rescued and can propagate to ASTNode#start
|
7
|
+
attr_reader :cmd
|
8
|
+
|
9
|
+
CMDS = [:info, :warn, :error].freeze
|
10
|
+
|
11
|
+
def initialize(message = nil, cmd = :error)
|
12
|
+
super(message)
|
13
|
+
@cmd = CMDS.include?(cmd) ? cmd : :error
|
14
|
+
end
|
6
15
|
end
|
7
16
|
end
|
8
17
|
end
|
data/lib/exel/events.rb
CHANGED
data/lib/exel/instruction.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
module EXEL
|
2
3
|
# Represents one step to be executed in the processing of a job
|
3
4
|
class Instruction
|
@@ -9,7 +10,8 @@ module EXEL
|
|
9
10
|
|
10
11
|
def execute(context)
|
11
12
|
context.merge!(@args)
|
12
|
-
@processor_class.new(context)
|
13
|
+
processor = @processor_class.new(context)
|
14
|
+
EXEL.middleware.invoke(@processor_class, context, @args) { processor.process(@subtree) }
|
13
15
|
end
|
14
16
|
end
|
15
17
|
end
|
data/lib/exel/job.rb
CHANGED
data/lib/exel/logging.rb
CHANGED
@@ -1,9 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'logger'
|
2
3
|
|
3
4
|
module EXEL
|
4
5
|
module Logging
|
5
6
|
DEFAULT_LEVEL = :info
|
6
7
|
|
8
|
+
# Formats log messages with timestamp, severity and Logging prefix (if set via {Logging.with_prefix})
|
9
|
+
class PrefixFormatter < Logger::Formatter
|
10
|
+
def call(severity, time, _program_name, message)
|
11
|
+
"#{time.utc} severity=#{severity}, #{Logging.prefix}#{message}\n"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
7
15
|
def self.logger
|
8
16
|
@logger || initialize_logger
|
9
17
|
end
|
@@ -11,6 +19,7 @@ module EXEL
|
|
11
19
|
def self.initialize_logger
|
12
20
|
@logger = Logger.new(log_filename)
|
13
21
|
@logger.level = log_level
|
22
|
+
@logger.formatter = PrefixFormatter.new
|
14
23
|
@logger
|
15
24
|
end
|
16
25
|
|
@@ -24,7 +33,19 @@ module EXEL
|
|
24
33
|
end
|
25
34
|
|
26
35
|
def self.logger=(logger)
|
27
|
-
@logger = logger
|
36
|
+
@logger = logger ? LoggerWrapper.new(logger) : Logger.new('/dev/null')
|
37
|
+
end
|
38
|
+
|
39
|
+
# Sets a prefix to be added to any messages sent to the EXEL logger in the given block.
|
40
|
+
def self.with_prefix(prefix)
|
41
|
+
Thread.current[:exel_log_prefix] = prefix
|
42
|
+
yield
|
43
|
+
ensure
|
44
|
+
Thread.current[:exel_log_prefix] = nil
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.prefix
|
48
|
+
Thread.current[:exel_log_prefix]
|
28
49
|
end
|
29
50
|
end
|
30
51
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module EXEL
|
3
|
+
module Logging
|
4
|
+
# Wraps calls to a logger to add {Logging} prefix to log messages
|
5
|
+
class LoggerWrapper < SimpleDelegator
|
6
|
+
LOG_LEVELS = %i(debug info warn error fatal unknown).freeze
|
7
|
+
|
8
|
+
LOG_LEVELS.each do |level|
|
9
|
+
define_method level do |progname = nil, &block|
|
10
|
+
prefix_block = nil
|
11
|
+
|
12
|
+
if block
|
13
|
+
prefix_block = proc { "#{Logging.prefix}#{block.call}" }
|
14
|
+
else
|
15
|
+
progname = "#{Logging.prefix}#{progname}"
|
16
|
+
end
|
17
|
+
|
18
|
+
__getobj__.send(level, progname, &prefix_block)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def add(severity, message = nil, progname = nil)
|
23
|
+
message = yield if message.nil? && block_given?
|
24
|
+
__getobj__.add(severity, "#{Logging.prefix}#{message}", progname)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|