conflow 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 572a2e10d542d2914de25fc748c1c153938e59b081aaef7829208395a86e462b
4
- data.tar.gz: 99a32aa33c314eeabfa4feed3b386072ba2146139b552b257c59cd4b693fcca0
3
+ metadata.gz: e4e50a7f447dad66d1522e881c5253e3478a806a38a08478660fc0a831d59a5d
4
+ data.tar.gz: ae859355013408d204a12532dc480efdef8933dd70bc37ab87e60a06e17f0612
5
5
  SHA512:
6
- metadata.gz: 8538390ead7bcc20bcea935aee1ec4d4ec7359eba95a89a8a5ddd14a3fbec25b69f9900d4548a4fd6370ae94eff13471ac794fe39a593d64a16ae0f2bf0abdf3
7
- data.tar.gz: 53f5221e6bb9c0bd7dffdfdae3f0cb955f170e3aaa22a281a0e7f682952d9ad946de7daa93b0899935a5e77864f8178e5219e766182f74cbddfc16ea0d12881a
6
+ metadata.gz: 6b004258305c37c987b6edee1482ba0e00e2f9e02ca3ef6d5354288de59f7e6f7d8f2c75235cc1a51faf678e8a639459d6cb925f03276a10fc307b86b877a964
7
+ data.tar.gz: b9b87d6d3579b7301e19b1e74ff7ff292e06e2df16f1810c4e0a2de6ab537597fb2aa42b9c1da780248161f1e3b8e667cb97be88a80fcfd7268a2afe23ab21f8
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.4.0
2
+
3
+ - Removed hooks, added promises. If you were looking only on Changelog, you never even knew about hooks - it can stay that way :)
4
+
1
5
  ## 0.2.0
2
6
 
3
7
  - Flow and Jobs are now removed after they are finished
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- conflow (0.3.0)
4
+ conflow (0.4.0)
5
5
  redis (~> 4.0)
6
6
 
7
7
  GEM
data/README.md CHANGED
@@ -97,24 +97,25 @@ end
97
97
 
98
98
  ![Created graph](https://camo.githubusercontent.com/0b1ee59994323900906264ea50fbc9169e4d21dd/68747470733a2f2f63686172742e676f6f676c65617069732e636f6d2f63686172743f63686c3d646967726170682b472b2537422530442530412b2b72616e6b6469722533444c522533422530442530412b2b25323253544152542532322b2d2533452b25323246697273744a6f622532322530442530412b2b25323253544152542532322b2d2533452b253232496e646570656e64656e744a6f622532322530442530412b2b25323246697273744a6f622532322b2d2533452b2532325365636f6e644a6f622532322530442530412b2b253232496e646570656e64656e744a6f622532322b2d2533452b2532325365636f6e644a6f622532322530442530412b2b2532325365636f6e644a6f622532322b2d2533452b25323246696e69736855702532322530442530412b2b25323246696e69736855702532322b2d2533452b253232454e442532322530442530412537442530442530412b266368743d6776)
99
99
 
100
- ### Hooks
100
+ #### Promises
101
101
 
102
- If you want to modify your flow dynamically depending on results of your jobs, you can use hooks.
103
-
104
- Hooks are methods defined on the `Flow` objects which will be called with result of the job (value returned by `Worker`, see [Performing jobs](https://github.com/fanfilmu/conflow#performing-jobs)).
102
+ In order to use other Job's result as parameter of another job, use Futures:
105
103
 
106
104
  ```ruby
107
105
  class MyFlow < ApplicationFlow
108
106
  def configure
109
- run CollectEmails, hook: :send_notifications
110
- end
111
-
112
- def send_notifications(emails)
113
- emails.each { |email| run SendNotification, params: { email: email } }
107
+ first = run FirstJob
108
+ run SecondJob, params: { object_id: first.outcome[:id] }
114
109
  end
115
110
  end
116
111
  ```
117
112
 
113
+ Note that `SecondJob` will automatically depend on `FirstJob`. When `FirstJob` finishes, it is expected to return hash: `{ id: "<some object>" }`.
114
+
115
+ Returned object must be serializable with JSON in order to be properly persisted and handled by Redis script which resolves promises.
116
+
117
+ If `FirstJob` returns `{ id: 14 }`, `SecondJob` will be run with `{ object_id: 14 }` parameter.
118
+
118
119
  ### Performing jobs
119
120
 
120
121
  To perform job, use `Conflow::Worker` mixin. It adds `#perform` method, which accepts two arguments: IDs of the flow and the job.
data/lib/conflow.rb CHANGED
@@ -10,6 +10,7 @@ require "conflow/redis"
10
10
  require "conflow/redis/connection_wrapper"
11
11
  require "conflow/redis/field"
12
12
  require "conflow/redis/value_field"
13
+ require "conflow/redis/raw_value_field"
13
14
  require "conflow/redis/hash_field"
14
15
  require "conflow/redis/array_field"
15
16
  require "conflow/redis/sorted_set_field"
@@ -22,9 +23,14 @@ require "conflow/redis/findable"
22
23
  require "conflow/redis/script"
23
24
  require "conflow/redis/add_job_script"
24
25
  require "conflow/redis/complete_job_script"
26
+ require "conflow/redis/resolve_promises_script"
25
27
  require "conflow/redis/queue_jobs_script"
26
28
 
29
+ require "conflow/error"
30
+ require "conflow/future"
31
+ require "conflow/promise"
27
32
  require "conflow/job"
33
+ require "conflow/flow/job_builder"
28
34
  require "conflow/flow/job_handler"
29
35
  require "conflow/flow"
30
36
  require "conflow/worker"
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Conflow
4
+ # Base class for Conflow errors
5
+ class Error < StandardError; end
6
+ end
data/lib/conflow/flow.rb CHANGED
@@ -43,6 +43,8 @@ module Conflow
43
43
  field :indegree, :sorted_set
44
44
 
45
45
  # Create new flow with given parameters
46
+ # @param args [Array<Object>] any parameters that will be passed to {#configure} method
47
+ # @return [Conflow::Job] job object representing created job
46
48
  # @example Simple configurable flow
47
49
  # class MyFlow < Conflow::Flow
48
50
  # def configure(id:, strict:)
@@ -66,6 +68,8 @@ module Conflow
66
68
  # @abstract
67
69
  # Override this method in order to contain your flow definition inside the class.
68
70
  # This method will be called if flow is created using {.create} method.
71
+ # @param args [Array<Object>] any arguments needed to start a flow
72
+ # @see create
69
73
  def configure(*args); end
70
74
  end
71
75
  end
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Conflow
4
+ class Flow < Conflow::Redis::Field
5
+ # Handles creating jobs
6
+ # @api private
7
+ class JobBuilder
8
+ # Holds mapping between worker classes and {Conflow::Job} objects
9
+ attr_reader :context
10
+
11
+ # Initialize {JobBuilder} with new empty context
12
+ def initialize
13
+ @context = {}
14
+ end
15
+
16
+ # Create new job and resolve it's dependencies.
17
+ # @see Conflow::Flow::JobHandler#run
18
+ # @return [Conflow::Job, Array<Conflow::Job>]
19
+ def call(worker_class, params, dependencies)
20
+ job = initialize_job(worker_class)
21
+
22
+ promises, params = extract_promises(job, params)
23
+ dependencies = build_dependencies(promises, dependencies)
24
+ assign_job_attributes(job, promises, params)
25
+
26
+ [job, dependencies]
27
+ end
28
+
29
+ private
30
+
31
+ def initialize_job(worker_class)
32
+ Conflow::Job.new.tap do |job|
33
+ job.class_name = worker_class.name
34
+ context[worker_class] = job
35
+ end
36
+ end
37
+
38
+ def assign_job_attributes(job, promises, params)
39
+ job.params = params if params.any?
40
+ job.promise_ids.push(*promises.map(&:id)) if promises.any?
41
+ end
42
+
43
+ def extract_promises(job, params)
44
+ params = params.dup
45
+
46
+ promises = params.map do |key, value|
47
+ next unless value.is_a?(Conflow::Future)
48
+
49
+ params.delete(key)
50
+ value.build_promise(job, key)
51
+ end.compact
52
+
53
+ [promises, params]
54
+ end
55
+
56
+ def build_dependencies(promises, dependencies)
57
+ promise_jobs = promises.map { |promise| Conflow::Job.new(promise.job_id.value) }
58
+ dependencies = prepare_dependencies(dependencies)
59
+
60
+ (promise_jobs + dependencies).compact.uniq
61
+ end
62
+
63
+ def prepare_dependencies(dependencies)
64
+ case dependencies
65
+ when Enumerable then dependencies.map(&method(:prepare_dependency))
66
+ else [prepare_dependency(dependencies)]
67
+ end
68
+ end
69
+
70
+ def prepare_dependency(dependency)
71
+ case dependency
72
+ when Conflow::Job then dependency
73
+ when Class then context[dependency]
74
+ when String, Numeric then Conflow::Job.new(dependency)
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -9,24 +9,30 @@ module Conflow
9
9
  # Parameters of the job. They will be passed to {Conflow::Worker#perform} block. Defalts to empty hash.
10
10
  # @param after [Conflow::Job|Class|Integer|Array<Conflow::Job,Class,Integer>]
11
11
  # Dependencies of the job. Can be one or more objects of the following classes: {Conflow::Job}, Class, Integer
12
- # @param hook [Symbol] method to be called on {Conflow::Flow} instance after job is performed.
13
- # The hook method should accept result of the job (value returned by {Conflow::Worker#perform})
14
12
  # @return [Conflow::Job] enqueued job
15
- def run(job_class, params: {}, after: [], hook: nil)
16
- build_job(job_class, params, hook).tap do |job|
17
- job_classes[job_class] = job
18
- after = prepare_dependencies(after).compact
13
+ def run(job_class, params: {}, after: [])
14
+ job, dependencies = job_builder.call(job_class, params, after)
19
15
 
20
- call_script(Conflow::Redis::AddJobScript, job, after: after)
21
- queue_available_jobs
22
- end
16
+ call_script(Conflow::Redis::AddJobScript, job, after: dependencies)
17
+ queue_available_jobs
18
+
19
+ job
23
20
  end
24
21
 
25
- # Finishes job, changes its status, runs hook if it's present and queues new available jobs
22
+ # Starts the job - resolves it's promises. It's called by {Worker} before it yields parameters
23
+ # @api private
24
+ # @see Worker
25
+ # @param job [Conflow::Job] job that needs to resolve it's promises
26
+ # @return [void]
27
+ def start(job)
28
+ call_script(Conflow::Redis::ResolvePromisesScript, job)
29
+ end
30
+
31
+ # Finishes job, changes its status, assigns result of the job and queues new available jobs
26
32
  # @param job [Conflow::Job] job to be marked as finished
27
- # @param result [Object] result of the job to be passed to hook
33
+ # @param result [Object] result of the job
28
34
  def finish(job, result = nil)
29
- send(job.hook.to_s, result) unless job.hook.nil?
35
+ job.result = result if result
30
36
  call_script(Conflow::Redis::CompleteJobScript, job)
31
37
  queue_available_jobs
32
38
  destroy! if finished?
@@ -40,35 +46,12 @@ module Conflow
40
46
  end
41
47
  end
42
48
 
43
- def build_job(job_class, params, hook)
44
- Conflow::Job.new.tap do |job|
45
- job.params = params if params.any?
46
- job.hook = hook if hook
47
- job.class_name = job_class.name
48
- end
49
- end
50
-
51
49
  def call_script(script, *args)
52
50
  script.call(self, *args)
53
51
  end
54
52
 
55
- def prepare_dependencies(dependencies)
56
- case dependencies
57
- when Enumerable then dependencies.map(&method(:prepare_dependency))
58
- else [prepare_dependency(dependencies)]
59
- end
60
- end
61
-
62
- def prepare_dependency(dependency)
63
- case dependency
64
- when Conflow::Job then dependency
65
- when Class then job_classes[dependency]
66
- when String, Numeric then Conflow::Job.new(dependency)
67
- end
68
- end
69
-
70
- def job_classes
71
- @job_classes ||= {}
53
+ def job_builder
54
+ @job_builder ||= JobBuilder.new
72
55
  end
73
56
  end
74
57
  end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Conflow
4
+ # Raised when future was built with nested key
5
+ # @example
6
+ # job.outcome[:parent][:child] #=> raises InvalidNestedPromise
7
+ class InvalidNestedFuture < Error
8
+ # Sets custom message for the error
9
+ def initialize
10
+ super "Futures don't allow extracting nested fields"
11
+ end
12
+ end
13
+
14
+ # Struct like objects that represent value to be returned by {Job}
15
+ class Future
16
+ # Job which result is promised
17
+ attr_reader :job
18
+ # Key in the result hash to which this future proxies
19
+ attr_reader :result_key
20
+
21
+ # @param job [Conflow::Job] Job which result is promised
22
+ # @param result_key [String, Symbol] key under which promised value exists
23
+ # @api private
24
+ # @see Conflow::Job#outcome
25
+ def initialize(job, result_key = nil)
26
+ @job = job
27
+ @result_key = result_key
28
+ end
29
+
30
+ # Returns new {Future} with assigned key, if possible
31
+ # @param key [Symbol, String] Key in result hash of the job that holds promised value
32
+ # @return [Future] future object that can be used as parameter
33
+ # @example
34
+ # job = run AJob #=> returns { email: "test@example.com" }
35
+ # job.outcome[:email] # when resolved, will return "test@example.com"
36
+ def [](key)
37
+ raise InvalidNestedFuture if result_key
38
+ Future.new(job, key)
39
+ end
40
+
41
+ # Builds promise from this future
42
+ # @api private
43
+ # @param depending_job [Conflow::Job] job which will use new promise
44
+ # @param param_key [Symbol, String] key in result hash that holds promised value
45
+ # @return [Conflow::Promise] promise object
46
+ def build_promise(depending_job, param_key)
47
+ Promise.new.tap do |promise|
48
+ promise.assign_attributes(job_id: job.id, hash_field: param_key, result_key: result_key)
49
+ depending_job.promise_ids << promise.id
50
+ end
51
+ end
52
+ end
53
+ end
data/lib/conflow/job.rb CHANGED
@@ -5,8 +5,6 @@ module Conflow
5
5
  # @!attribute [rw] status
6
6
  # Status of the job
7
7
  # @return [Integer] 0 - pending, 1 - finished
8
- # @!attribute [rw] hook
9
- # @return [String, nil] name of the method on related flow to be called once job is finished
10
8
  # @!attribute [rw] class_name
11
9
  # @return [String] class name of the worker class
12
10
  # @!attribute [rw] params
@@ -16,10 +14,11 @@ module Conflow
16
14
  include Conflow::Redis::Identifier
17
15
 
18
16
  has_many :successors, Conflow::Job
17
+ has_many :promises, Conflow::Promise
19
18
  field :params, :hash
19
+ field :result, :hash
20
20
  field :class_name, :value
21
21
  field :status, :value # 0 - pending, 1 - finished
22
- field :hook, :value
23
22
 
24
23
  # Returns instance of Job. It sets status to 0 (pending) for new jobs
25
24
  def initialize(*)
@@ -33,5 +32,17 @@ module Conflow
33
32
  def worker_type
34
33
  Object.const_get(class_name.to_s)
35
34
  end
35
+
36
+ # Returns promise of this job's result. It assumes result of the job will be a Hash.
37
+ # @note Passing a {Promise} as a job parameter automatically sets the job
38
+ # which produces the result as dependency of the new job
39
+ # @return [Conflow::Future] future object (basis of {Promise})
40
+ # @example Running job which depends on result of another
41
+ # job = run MyJob, params: { key: 400 }
42
+ # run OtherJob, params: { value: job.outcome[:result] }
43
+ # # now OtherJob will depend on MyJob and it will use it's :result result as it's own :value parameter
44
+ def outcome
45
+ Future.new(self)
46
+ end
36
47
  end
37
48
  end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Conflow
4
+ # Promises are stubs for returned values from jobs. They will be resolved once the job is done.
5
+ class Promise < Conflow::Redis::Field
6
+ include Conflow::Redis::Model
7
+ include Conflow::Redis::Identifier
8
+
9
+ # @!attribute [rw] job_id
10
+ # ID of the job that promised result
11
+ # @return [Conflow::Redis::ValueField] ID of {Conflow::Job}
12
+ field :job_id, :raw_value
13
+ # @!attribute [rw] key
14
+ # Key of job's result which is promised
15
+ # @return [Conflow::Redis::ValueField] Name of the key
16
+ field :result_key, :raw_value
17
+ # @!attribute [rw] hash_field
18
+ # Redis field name in the result hash
19
+ # @return [Conflow::Redis::ValueField] Name of the field
20
+ field :hash_field, :raw_value
21
+ end
22
+ end
@@ -28,6 +28,13 @@ module Conflow
28
28
  end
29
29
  end
30
30
 
31
+ # Convienience method for assigning multiple fields in model
32
+ def assign_attributes(**attributes)
33
+ attributes.each do |attribute, value|
34
+ send("#{attribute}=", value)
35
+ end
36
+ end
37
+
31
38
  private
32
39
 
33
40
  def collect_relation_keys(relation)
@@ -42,7 +49,8 @@ module Conflow
42
49
  array: Conflow::Redis::ArrayField,
43
50
  value: Conflow::Redis::ValueField,
44
51
  sorted_set: Conflow::Redis::SortedSetField,
45
- set: Conflow::Redis::SetField
52
+ set: Conflow::Redis::SetField,
53
+ raw_value: Conflow::Redis::RawValueField
46
54
  }.freeze
47
55
 
48
56
  # Copies field and relations information
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Conflow
4
+ module Redis
5
+ # Represents single value (Redis String). Values are not serialized
6
+ # @api private
7
+ class RawValueField < Field
8
+ # @param value [Object] new value to be saved
9
+ # @return [String] Redis response
10
+ def overwrite(value)
11
+ command :set, [key, value]
12
+ end
13
+
14
+ # @return [String, nil] String representation of value
15
+ def to_s
16
+ value&.to_s
17
+ end; alias to_str to_s
18
+
19
+ # @param other [Object] Object to compare value with. Handles Strings, Symbols and other RawValueFields
20
+ # @return [Boolean] true if equal
21
+ def ==(other)
22
+ case other
23
+ when String, Symbol then value == other.to_s
24
+ when RawValueField then key == other.key || to_s == other.to_s
25
+ else super
26
+ end
27
+ end
28
+
29
+ # @return [Object] Value present in Redis
30
+ def value
31
+ command(:get, [key])
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Conflow
4
+ module Redis
5
+ # Resolves job's promises
6
+ # @api private
7
+ class ResolvePromisesScript < Script
8
+ self.script = <<~LUA
9
+ local pairs = {}
10
+
11
+ for i=1, #ARGV, 3 do
12
+ local result_field = redis.call('get', ARGV[i])
13
+ local field_name = redis.call('get', ARGV[i + 1])
14
+ local result_hash_key = ARGV[i + 2]
15
+
16
+ table.insert(pairs, field_name)
17
+ table.insert(pairs, redis.call('hget', result_hash_key, result_field) or '""')
18
+ end
19
+
20
+ return redis.call('hmset', KEYS[1], unpack(pairs))
21
+ LUA
22
+
23
+ class << self
24
+ # @param job [Conflow::Job] Job which needs promises to be resolved
25
+ def call(_flow, job)
26
+ promises = extract_keys(job.promises)
27
+ super([job.params.key], promises) if promises.any?
28
+ end
29
+
30
+ private
31
+
32
+ def extract_keys(promises)
33
+ promises.flat_map do |promise|
34
+ [promise.result_key.key, promise.hash_field.key, Conflow::Job.new(promise.job_id.value).result.key]
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -19,23 +19,16 @@ module Conflow
19
19
  command :set, [key, JSON.dump(value), nx: true]
20
20
  end
21
21
 
22
- # @param other [Object] Object to compare value with. Handles Strings, Numerics,
23
- # Symbols and other {ValueField} objects
22
+ # @param other [Object] Object to compare value with. Handles Strings, Numerics, and other {ValueField} objects
24
23
  # @return [Boolean] true if equal
25
24
  def ==(other)
26
25
  case other
27
26
  when String, Numeric then value == other
28
- when Symbol then value.to_sym == other
29
27
  when ValueField then key == other.key || to_s == other.to_s
30
28
  else super
31
29
  end
32
30
  end
33
31
 
34
- # @return [Boolean] true if object does not exist in Redis, else otherwise
35
- def nil?
36
- value.nil?
37
- end
38
-
39
32
  # @return [String, nil] String representation of value
40
33
  def to_s
41
34
  value&.to_s
@@ -2,5 +2,5 @@
2
2
 
3
3
  module Conflow
4
4
  # Current version of the gem
5
- VERSION = "0.3.0"
5
+ VERSION = "0.4.0"
6
6
  end
@@ -26,6 +26,8 @@ module Conflow
26
26
  job = Conflow::Job.new(job_id)
27
27
  flow = Conflow::Flow.find(flow_id)
28
28
 
29
+ flow.start(job)
30
+
29
31
  yield(job.worker_type, job.params).tap { |result| flow.finish(job, result) }
30
32
  end
31
33
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: conflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michał Begejowicz
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-03-15 00:00:00.000000000 Z
11
+ date: 2018-03-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -145,9 +145,13 @@ files:
145
145
  - bin/setup
146
146
  - conflow.gemspec
147
147
  - lib/conflow.rb
148
+ - lib/conflow/error.rb
148
149
  - lib/conflow/flow.rb
150
+ - lib/conflow/flow/job_builder.rb
149
151
  - lib/conflow/flow/job_handler.rb
152
+ - lib/conflow/future.rb
150
153
  - lib/conflow/job.rb
154
+ - lib/conflow/promise.rb
151
155
  - lib/conflow/redis.rb
152
156
  - lib/conflow/redis/add_job_script.rb
153
157
  - lib/conflow/redis/array_field.rb
@@ -160,6 +164,8 @@ files:
160
164
  - lib/conflow/redis/identifier.rb
161
165
  - lib/conflow/redis/model.rb
162
166
  - lib/conflow/redis/queue_jobs_script.rb
167
+ - lib/conflow/redis/raw_value_field.rb
168
+ - lib/conflow/redis/resolve_promises_script.rb
163
169
  - lib/conflow/redis/script.rb
164
170
  - lib/conflow/redis/set_field.rb
165
171
  - lib/conflow/redis/sorted_set_field.rb