procrastinator 0.9.0 → 1.0.0.pre.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: procrastinator
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.0
4
+ version: 1.0.0.pre.rc2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Robin Miller
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-07-26 00:00:00.000000000 Z
11
+ date: 2022-09-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -16,84 +16,98 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.11'
19
+ version: '2.3'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.11'
26
+ version: '2.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: fakefs
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0.10'
33
+ version: '1.8'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0.10'
40
+ version: '1.8'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '12.3'
47
+ version: '13.0'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '12.3'
54
+ version: '13.0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: rspec
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '3.0'
61
+ version: '3.9'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '3.0'
68
+ version: '3.9'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: rubocop
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '0.58'
75
+ version: '1.12'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '0.58'
82
+ version: '1.12'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rubocop-performance
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.10'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.10'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: simplecov
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
101
  - - "~>"
88
102
  - !ruby/object:Gem::Version
89
- version: 0.16.1
103
+ version: 0.18.0
90
104
  type: :development
91
105
  prerelease: false
92
106
  version_requirements: !ruby/object:Gem::Requirement
93
107
  requirements:
94
108
  - - "~>"
95
109
  - !ruby/object:Gem::Version
96
- version: 0.16.1
110
+ version: 0.18.0
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: timecop
99
113
  requirement: !ruby/object:Gem::Requirement
@@ -108,7 +122,7 @@ dependencies:
108
122
  - - "~>"
109
123
  - !ruby/object:Gem::Version
110
124
  version: '0.9'
111
- description: A straightforward, customizable Ruby job queue with zero dependencies.
125
+ description: A flexible pure Ruby job queue. Tasks are reschedulable after failures.
112
126
  email:
113
127
  - robin@tenjin.ca
114
128
  executables: []
@@ -124,26 +138,30 @@ files:
124
138
  - Gemfile
125
139
  - LICENSE.txt
126
140
  - README.md
141
+ - RELEASE_NOTES.md
127
142
  - Rakefile
128
143
  - bin/console
129
144
  - bin/setup
130
145
  - lib/procrastinator.rb
131
146
  - lib/procrastinator/config.rb
132
- - lib/procrastinator/loaders/csv_loader.rb
147
+ - lib/procrastinator/logged_task.rb
133
148
  - lib/procrastinator/queue.rb
134
- - lib/procrastinator/queue_manager.rb
135
149
  - lib/procrastinator/queue_worker.rb
150
+ - lib/procrastinator/rake/daemon_tasks.rb
151
+ - lib/procrastinator/rake/tasks.rb
136
152
  - lib/procrastinator/scheduler.rb
137
153
  - lib/procrastinator/task.rb
138
154
  - lib/procrastinator/task_meta_data.rb
139
- - lib/procrastinator/task_worker.rb
155
+ - lib/procrastinator/task_store/file_transaction.rb
156
+ - lib/procrastinator/task_store/simple_comma_store.rb
157
+ - lib/procrastinator/test/mocks.rb
140
158
  - lib/procrastinator/version.rb
141
- - lib/rake/procrastinator_task.rb
142
159
  - procrastinator.gemspec
143
160
  homepage: https://github.com/TenjinInc/procrastinator
144
161
  licenses:
145
162
  - MIT
146
- metadata: {}
163
+ metadata:
164
+ rubygems_mfa_required: 'true'
147
165
  post_install_message:
148
166
  rdoc_options: []
149
167
  require_paths:
@@ -152,16 +170,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
152
170
  requirements:
153
171
  - - ">="
154
172
  - !ruby/object:Gem::Version
155
- version: '2.3'
173
+ version: '2.4'
156
174
  required_rubygems_version: !ruby/object:Gem::Requirement
157
175
  requirements:
158
- - - ">="
176
+ - - ">"
159
177
  - !ruby/object:Gem::Version
160
- version: '0'
178
+ version: 1.3.1
161
179
  requirements: []
162
- rubyforge_project:
163
- rubygems_version: 2.6.13
180
+ rubygems_version: 3.1.2
164
181
  signing_key:
165
182
  specification_version: 4
166
- summary: For apps that put work off until later
183
+ summary: For apps to put off work until later
167
184
  test_files: []
@@ -1,107 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'csv'
4
- require 'pathname'
5
-
6
- module Procrastinator
7
- module Loader
8
- # Simple Task I/O object that writes task information (ie. TaskMetaData attributes) to a CSV file.
9
- #
10
- # @author Robin Miller
11
- class CSVLoader
12
- # ordered
13
- HEADERS = [:id, :queue, :run_at, :initial_run_at, :expire_at,
14
- :attempts, :last_fail_at, :last_error, :data].freeze
15
-
16
- DEFAULT_FILE = 'procrastinator-tasks.csv'
17
-
18
- def initialize(file_path = DEFAULT_FILE)
19
- @path = Pathname.new(file_path)
20
-
21
- if @path.directory? || @path.to_s.end_with?('/')
22
- @path += DEFAULT_FILE
23
- elsif @path.extname.empty?
24
- @path = Pathname.new("#{ file_path }.csv")
25
- end
26
- end
27
-
28
- def read
29
- data = CSV.table(@path.to_s, force_quotes: false).to_a
30
-
31
- headers = data.shift
32
-
33
- data.collect do |d|
34
- hash = Hash[headers.zip(d)]
35
-
36
- hash[:data] = hash[:data].gsub('""', '"')
37
-
38
- hash
39
- end
40
- end
41
-
42
- def create(queue:, run_at:, initial_run_at:, expire_at:, data: '')
43
- existing_data = begin
44
- read
45
- rescue Errno::ENOENT
46
- []
47
- end
48
-
49
- max_id = existing_data.collect { |task| task[:id] }.max || 0
50
-
51
- new_data = {
52
- id: max_id + 1,
53
- queue: queue,
54
- run_at: run_at,
55
- initial_run_at: initial_run_at,
56
- expire_at: expire_at,
57
- attempts: 0,
58
- data: data
59
- }
60
-
61
- write(existing_data + [new_data])
62
- end
63
-
64
- def update(id, data)
65
- existing_data = begin
66
- read
67
- rescue Errno::ENOENT
68
- []
69
- end
70
-
71
- task_data = existing_data.find do |task|
72
- task[:id] == id
73
- end
74
-
75
- task_data.merge!(data)
76
-
77
- write(existing_data)
78
- end
79
-
80
- def delete(id)
81
- existing_data = begin
82
- read
83
- rescue Errno::ENOENT
84
- []
85
- end
86
-
87
- existing_data.delete_if do |task|
88
- task[:id] == id
89
- end
90
-
91
- write(existing_data)
92
- end
93
-
94
- def write(data)
95
- lines = data.collect do |d|
96
- CSV.generate_line(d, headers: HEADERS, force_quotes: true)
97
- end
98
-
99
- @path.dirname.mkpath
100
- @path.open('w') do |f|
101
- f.puts HEADERS.join(',')
102
- f.puts lines.join
103
- end
104
- end
105
- end
106
- end
107
- end
@@ -1,201 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Procrastinator
4
- # Spawns and manages work queue subprocesses.
5
- #
6
- # This is where all of the multi-process logic should be kept to.
7
- #
8
- # @author Robin Miller
9
- #
10
- # @!attribute [r] :workers
11
- # @return [Hash] Maps the constructed QueueWorkers to their process ID.
12
- class QueueManager
13
- attr_reader :workers
14
-
15
- def initialize(config)
16
- @workers = {}
17
- @config = config
18
- @logger = start_log
19
- end
20
-
21
- # Shuts down any remaining old queue workers and spawns a new one for each queue defined in the config
22
- #
23
- # @return [Scheduler] a scheduler object that can be used to interact with the queues
24
- def spawn_workers
25
- scheduler = Scheduler.new(@config, self)
26
-
27
- kill_old_workers
28
-
29
- if ENV['PROCRASTINATOR_STOP']
30
- @logger.warn('Cannot spawn queue workers because environment variable PROCRASTINATOR_STOP is set')
31
- else
32
- @config.queues.each do |queue|
33
- spawn_worker(queue, scheduler: scheduler)
34
- end
35
- end
36
-
37
- scheduler
38
- end
39
-
40
- # Produces a new QueueWorker for the given queue.
41
- #
42
- # If Test Mode is disabled in the config, then it will also fork a new independent process for that worker
43
- # to work in.
44
- #
45
- # @param queue [Queue] the queue to build a worker for
46
- # @param scheduler [Scheduler] an optional scheduler instance to pass to the worker
47
- def spawn_worker(queue, scheduler: nil)
48
- worker = QueueWorker.new(queue: queue,
49
- config: @config,
50
- scheduler: scheduler)
51
- if @config.test_mode?
52
- @workers[worker] = Process.pid
53
- else
54
- check_for_name(worker.long_name)
55
-
56
- pid = fork
57
-
58
- if pid
59
- # === PARENT PROCESS ===
60
- Process.detach(pid)
61
- @workers[worker] = pid
62
- else
63
- deamonize(worker.long_name)
64
-
65
- worker.work
66
- shutdown_worker
67
- end
68
- end
69
- end
70
-
71
- def act(*queue_names)
72
- unless @config.test_mode?
73
- raise <<~ERR
74
- Procrastinator.act called outside Test Mode.
75
- Either use Procrastinator.spawn_workers or call #enable_test_mode in Procrastinator.setup.
76
- ERR
77
- end
78
-
79
- workers = @workers.keys
80
-
81
- if queue_names.empty?
82
- workers.each(&:act)
83
- else
84
- queue_names.each do |name|
85
- workers.find { |worker| worker.name == name }.act
86
- end
87
- end
88
- end
89
-
90
- private
91
-
92
- def start_log
93
- directory = @config.log_dir
94
-
95
- return unless directory
96
-
97
- log_path = directory + 'queue-manager.log'
98
-
99
- directory.mkpath
100
- File.open(log_path.to_path, 'a+') { |f| f.write '' }
101
-
102
- logger = Logger.new(log_path.to_path)
103
-
104
- logger.level = @config.log_level
105
-
106
- # @logger.info(['',
107
- # '===================================',
108
- # "Started worker process, #{long_name}, to work off queue #{@queue.name}.",
109
- # "Worker pid=#{Process.pid}; parent pid=#{Process.ppid}.",
110
- # '==================================='].join("\n"))
111
-
112
- logger
113
- end
114
-
115
- # Methods exclusive to the child process
116
- module ChildMethods
117
- def deamonize(name)
118
- Process.daemon(true)
119
- Process.setsid
120
- srand
121
- Process.setproctitle(name)
122
- close_io
123
-
124
- write_pid_file(Process.pid, name)
125
-
126
- @config.run_process_block
127
- end
128
-
129
- # Make sure all input/output streams are closed
130
- def close_io
131
- stds = [$stdin, $stdout, $stderr]
132
-
133
- # Part 1: close all IO objects (except for $stdin/$stdout/$stderr)
134
- ObjectSpace.each_object(IO) do |io|
135
- next if stds.include?(io)
136
-
137
- begin
138
- io.close
139
- rescue IOError
140
- next
141
- end
142
- end
143
-
144
- # Part 2: redirect STD connections
145
- stds.each do |io|
146
- io.reopen '/dev/null'
147
- end
148
-
149
- # TODO: redirect OUT or ERR to logger?
150
- end
151
-
152
- # Wrapping #exit to allow for tests to easily stub out this behaviour.
153
- # If #exit isn't prevented, the test framework will break,
154
- # but #exit can't be directly stubbed either (because it's a required Kernel method)
155
- def shutdown_worker
156
- exit
157
- end
158
- end
159
-
160
- # Methods exclusive to the main/parent process
161
- module ParentMethods
162
- def kill_old_workers
163
- @config.pid_dir.mkpath
164
-
165
- @config.pid_dir.each_child do |file|
166
- pid = file.read.to_i
167
-
168
- begin
169
- Process.kill('KILL', pid)
170
- @logger.info("Killing old worker process pid: #{ pid }")
171
- rescue Errno::ESRCH
172
- @logger.info("Expected old worker process pid=#{ pid }, but none was found")
173
- end
174
-
175
- file.delete
176
- end
177
- end
178
-
179
- def write_pid_file(pid, filename)
180
- @config.pid_dir.mkpath
181
-
182
- pid_file = @config.pid_dir + "#{ filename }.pid"
183
-
184
- File.open(pid_file.to_path, 'w') do |f|
185
- f.print(pid)
186
- end
187
- end
188
-
189
- def check_for_name(name)
190
- # better to use backticks so we can get the info and not spam user's stdout
191
- warn <<~WARNING unless `pgrep -f #{ name }`.empty?
192
- Warning: there is another process named "#{ name }". Use #each_process(prefix: '') in
193
- Procrastinator setup if you want to help yourself distinguish them.
194
- WARNING
195
- end
196
- end
197
-
198
- include ChildMethods
199
- include ParentMethods
200
- end
201
- end
@@ -1,100 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'yaml'
4
- require 'ostruct'
5
- require 'timeout'
6
-
7
- module Procrastinator
8
- # Works on a given task by creating a new instance of the queue's task class and running the appropriate hooks.
9
- #
10
- # The behaviour outside of the actual user-defined task is guided by the provided metadata.
11
- #
12
- # @author Robin Miller
13
- #
14
- # @see TaskMetaData
15
- class TaskWorker
16
- extend Forwardable
17
-
18
- def_delegators :@metadata,
19
- :id, :run_at, :initial_run_at, :expire_at,
20
- :attempts, :last_fail_at, :last_error,
21
- :data,
22
- :to_h, :successful?
23
-
24
- def initialize(metadata:,
25
- queue:,
26
- logger: Logger.new(StringIO.new),
27
- context: nil,
28
- scheduler: nil)
29
- @queue = queue
30
-
31
- @metadata = metadata
32
- @task = queue.task_class.new
33
-
34
- @task.data = @metadata.data if @task.respond_to?(:data=)
35
- @task.context = context if @task.respond_to?(:context=)
36
- @task.logger = logger if @task.respond_to?(:logger=)
37
- @task.scheduler = scheduler if @task.respond_to?(:scheduler=)
38
-
39
- @logger = logger
40
- @context = context
41
-
42
- raise MalformedTaskError, "task #{ @task.class } does not support #run method" unless @task.respond_to? :run
43
- end
44
-
45
- def work
46
- @metadata.add_attempt
47
-
48
- begin
49
- @metadata.verify_expiry!
50
-
51
- result = Timeout.timeout(@queue.timeout) do
52
- @task.run
53
- end
54
-
55
- @logger&.debug("Task completed: #{ @task.class } [#{ @metadata.serialized_data }]")
56
-
57
- @metadata.clear_fails
58
-
59
- try_hook(:success, result)
60
- rescue StandardError => error
61
- if @metadata.final_fail?(@queue)
62
- handle_final_failure(error)
63
- else
64
- handle_failure(error)
65
- end
66
- end
67
- end
68
-
69
- private
70
-
71
- def try_hook(method, *params)
72
- @task.send(method, *params) if @task.respond_to? method
73
- rescue StandardError => e
74
- warn "#{ method.to_s.capitalize } hook error: #{ e.message }"
75
- end
76
-
77
- def handle_failure(error)
78
- @metadata.fail(%[Task failed: #{ error.message }\n#{ error.backtrace.join("\n") }])
79
- @logger&.debug("Task failed: #{ @queue.name } with #{ @metadata.serialized_data }")
80
-
81
- @metadata.reschedule
82
-
83
- try_hook(:fail, error)
84
- end
85
-
86
- def handle_final_failure(error)
87
- trace = error.backtrace.join("\n")
88
- msg = "#{ @metadata.expired? ? 'Task expired' : 'Task failed too many times' }: #{ trace }"
89
-
90
- @metadata.fail(msg, final: true)
91
-
92
- @logger&.debug("Task failed permanently: #{ YAML.dump(@task) }")
93
-
94
- try_hook(:final_fail, error)
95
- end
96
- end
97
-
98
- class MalformedTaskError < StandardError
99
- end
100
- end
@@ -1,34 +0,0 @@
1
- require 'rake'
2
- require 'pathname'
3
-
4
- namespace :procrastinator do
5
- desc 'Halt all Procrastinator processes'
6
- task :stop, [:pid_dir] do |task, args|
7
- pid_dir = args[:pid_dir] || Procrastinator::Config::DEFAULT_PID_DIRECTORY
8
-
9
- if !pid_dir.exist? || pid_dir.empty?
10
- raise <<~ERR
11
- Default PID directory does not exist or is empty. Run:
12
- rake procrastinator:stop[directory]
13
- with the directory to search for pid files.
14
- ERR
15
- end
16
-
17
- pid_dir.each_child do |file|
18
- pid = file.read.to_i
19
-
20
- begin
21
- name = `ps -p #{pid} -o command`
22
- print "Halting worker process #{name} (pid: #{ pid })... "
23
- Process.kill('KILL', pid)
24
- puts 'halted'
25
- rescue Errno::ESRCH
26
- warn "Expected worker process pid=#{ pid }, but none was found. Continuing."
27
- end
28
-
29
- file.delete
30
- end
31
-
32
- puts '<= procrastinator:stop executed'
33
- end
34
- end