chained_job 0.2.0 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 21a653ae3a0eaaf596d741a3208b4aee1a7d297700e1d506c7005bf7cd529618
4
- data.tar.gz: 03ab75c95d0610a9f192d3f6f7bfc7eeac516f641b4bbc32730ac45d1710f3bc
3
+ metadata.gz: dd4890f548ef2ee47ff48180b593134f61b83fedc9017733346ab15d518d2c8d
4
+ data.tar.gz: c315bdee23b062ba3834bd23f8fcf18f07b8c7ca111dff9082d1bd251a5071e7
5
5
  SHA512:
6
- metadata.gz: fb6be468a09e6824fb2dd21fd8aa8029790a63c145d9f5048b5c29dea2632e85e6b6ebcfb29558e7504982a4ad48261140e6b96b3b8e4ef55726d58a18ed58c0
7
- data.tar.gz: 75a1aefe58b764b69d05514bfa47209ec4ff9d3f743379301841d132e15fc60c34f6254e0fae7dfacbca393c973f4ad2809dffcfb5a7041a42e62d3155a95857
6
+ metadata.gz: 0205f904ea84911f3258ac4798f52077c9e61e6850165db93b01b461ecbbf734d385093741211583aabddffcdda4e2cd4bc560ec10033dde9d5467f0232a28f5
7
+ data.tar.gz: 1e65c4a589d8db54ab53a76d80da2049a619945ed4a75428edf8aba28309a3e90b9ec222bb916fcd43842c945c38bf7fa9b881e9a5e2adefc90be92562fcf4eb
@@ -12,6 +12,8 @@ module ChainedJob
12
12
  :arguments_queue_expiration,
13
13
  :around_start_chains,
14
14
  :around_chain_process,
15
+ :around_array_of_job_arguments,
16
+ :after_worker_finished,
15
17
  :debug,
16
18
  :logger,
17
19
  :redis,
@@ -26,6 +28,8 @@ module ChainedJob
26
28
 
27
29
  self.around_start_chains = ->(_options, &block) { block.call }
28
30
  self.around_chain_process = ->(_options, &block) { block.call }
31
+ self.around_array_of_job_arguments = ->(_options, &block) { block.call }
32
+ self.after_worker_finished = ->(_options) {}
29
33
 
30
34
  self.debug = true
31
35
  end
@@ -13,10 +13,15 @@ module ChainedJob
13
13
  if worker_id
14
14
  ChainedJob::Process.run(self, worker_id, tag)
15
15
  else
16
- ChainedJob::StartChains.run(self.class, array_of_job_arguments, parallelism)
16
+ ChainedJob::StartChains.run(self.class, arguments_array, parallelism)
17
17
  end
18
18
  end
19
19
 
20
+ def arguments_array
21
+ options = { job_class: self.class }
22
+ ChainedJob.config.around_array_of_job_arguments.call(options) { array_of_job_arguments }
23
+ end
24
+
20
25
  def array_of_job_arguments
21
26
  raise NoMethodError, 'undefined method array_of_job_arguments'
22
27
  end
@@ -18,7 +18,7 @@ module ChainedJob
18
18
 
19
19
  def run
20
20
  with_hooks do
21
- return log_finished_worker unless argument
21
+ return finished_worker unless argument
22
22
 
23
23
  job_instance.process(argument)
24
24
  job_instance.class.perform_later(worker_id, job_tag)
@@ -32,7 +32,13 @@ module ChainedJob
32
32
  end
33
33
 
34
34
  def options
35
- { job_class: job_instance.class, worker_id: worker_id }
35
+ @options ||= { job_class: job_instance.class, worker_id: worker_id }
36
+ end
37
+
38
+ def finished_worker
39
+ log_finished_worker
40
+
41
+ ChainedJob.config.after_worker_finished&.call(options)
36
42
  end
37
43
 
38
44
  def log_finished_worker
@@ -42,7 +48,20 @@ module ChainedJob
42
48
  end
43
49
 
44
50
  def argument
45
- @argument ||= ChainedJob.redis.lpop(redis_key)
51
+ @argument ||= deserialized_argument
52
+ end
53
+
54
+ def deserialized_argument
55
+ Marshal.load(serialized_argument)
56
+ rescue TypeError, ArgumentError
57
+ # backward compatibility
58
+ serialized_argument
59
+ end
60
+
61
+ def serialized_argument
62
+ return @serialized_argument if defined?(@serialized_argument)
63
+
64
+ @serialized_argument = ChainedJob.redis.lpop(redis_key)
46
65
  end
47
66
 
48
67
  def redis_key
@@ -21,6 +21,8 @@ module ChainedJob
21
21
  # rubocop:disable Metrics/AbcSize
22
22
  def run
23
23
  with_hooks do
24
+ log_chained_job_cleanup
25
+
24
26
  ChainedJob::CleanUpQueue.run(job_class)
25
27
 
26
28
  next unless array_of_job_arguments.count.positive?
@@ -58,5 +60,11 @@ module ChainedJob
58
60
  "processing #{array_of_job_arguments.count} items"
59
61
  )
60
62
  end
63
+
64
+ def log_chained_job_cleanup
65
+ ChainedJob.logger.info(
66
+ "#{job_class}:#{job_tag} cleanup"
67
+ )
68
+ end
61
69
  end
62
70
  end
@@ -20,7 +20,7 @@ module ChainedJob
20
20
  set_tag_list
21
21
 
22
22
  array_of_job_arguments.each_slice(config.arguments_batch_size) do |sublist|
23
- ChainedJob.redis.rpush(redis_key, sublist)
23
+ ChainedJob.redis.rpush(redis_key, serialize(sublist))
24
24
  end
25
25
 
26
26
  ChainedJob.redis.expire(redis_key, config.arguments_queue_expiration)
@@ -44,6 +44,10 @@ module ChainedJob
44
44
  @job_key ||= Helpers.job_key(job_class)
45
45
  end
46
46
 
47
+ def serialize(arguments)
48
+ arguments.map { |argument| Marshal.dump(argument) }
49
+ end
50
+
47
51
  def config
48
52
  ChainedJob.config
49
53
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ChainedJob
4
- VERSION = '0.2.0'
4
+ VERSION = '0.5.0'
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: chained_job
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mantas Kūjalis
@@ -9,22 +9,22 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-07-22 00:00:00.000000000 Z
12
+ date: 2021-04-22 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  requirements:
18
- - - "~>"
18
+ - - ">="
19
19
  - !ruby/object:Gem::Version
20
- version: '1.17'
20
+ version: '0'
21
21
  type: :development
22
22
  prerelease: false
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
- - - "~>"
25
+ - - ">="
26
26
  - !ruby/object:Gem::Version
27
- version: '1.17'
27
+ version: '0'
28
28
  - !ruby/object:Gem::Dependency
29
29
  name: minitest
30
30
  requirement: !ruby/object:Gem::Requirement
@@ -53,25 +53,10 @@ dependencies:
53
53
  - - "~>"
54
54
  - !ruby/object:Gem::Version
55
55
  version: '12.0'
56
- - !ruby/object:Gem::Dependency
57
- name: rubocop-vinted
58
- requirement: !ruby/object:Gem::Requirement
59
- requirements:
60
- - - "~>"
61
- - !ruby/object:Gem::Version
62
- version: '0.3'
63
- type: :development
64
- prerelease: false
65
- version_requirements: !ruby/object:Gem::Requirement
66
- requirements:
67
- - - "~>"
68
- - !ruby/object:Gem::Version
69
- version: '0.3'
70
56
  description: Chained job allows you to define an array of queued jobs that should
71
57
  be run in sequence after the main job has been executed successfully.
72
58
  email:
73
- - mantas.kujalis@vinted.com
74
- - titas@vinted.com
59
+ - backend@vinted.com
75
60
  executables: []
76
61
  extensions: []
77
62
  extra_rdoc_files: []
@@ -104,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
104
89
  - !ruby/object:Gem::Version
105
90
  version: '0'
106
91
  requirements: []
107
- rubygems_version: 3.0.3
92
+ rubygems_version: 3.1.4
108
93
  signing_key:
109
94
  specification_version: 4
110
95
  summary: Chained job helper