fractor 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-main-ci-rubocop-yml +552 -0
- data/.rubocop.yml +14 -8
- data/.rubocop_todo.yml +162 -46
- data/README.adoc +1364 -376
- data/examples/auto_detection/auto_detection.rb +9 -9
- data/examples/continuous_chat_common/message_protocol.rb +53 -0
- data/examples/continuous_chat_fractor/README.adoc +217 -0
- data/examples/continuous_chat_fractor/chat_client.rb +303 -0
- data/examples/continuous_chat_fractor/chat_common.rb +83 -0
- data/examples/continuous_chat_fractor/chat_server.rb +167 -0
- data/examples/continuous_chat_fractor/simulate.rb +345 -0
- data/examples/continuous_chat_server/README.adoc +135 -0
- data/examples/continuous_chat_server/chat_client.rb +303 -0
- data/examples/continuous_chat_server/chat_server.rb +359 -0
- data/examples/continuous_chat_server/simulate.rb +343 -0
- data/examples/hierarchical_hasher/hierarchical_hasher.rb +12 -8
- data/examples/multi_work_type/multi_work_type.rb +30 -29
- data/examples/pipeline_processing/pipeline_processing.rb +15 -15
- data/examples/producer_subscriber/producer_subscriber.rb +20 -16
- data/examples/scatter_gather/scatter_gather.rb +29 -28
- data/examples/simple/sample.rb +5 -5
- data/examples/specialized_workers/specialized_workers.rb +44 -37
- data/lib/fractor/continuous_server.rb +188 -0
- data/lib/fractor/result_aggregator.rb +1 -1
- data/lib/fractor/supervisor.rb +277 -104
- data/lib/fractor/version.rb +1 -1
- data/lib/fractor/work_queue.rb +68 -0
- data/lib/fractor/work_result.rb +1 -1
- data/lib/fractor/worker.rb +2 -1
- data/lib/fractor/wrapped_ractor.rb +12 -2
- data/lib/fractor.rb +2 -0
- metadata +15 -2
data/lib/fractor/supervisor.rb
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "etc"
|
|
4
|
+
require "timeout"
|
|
4
5
|
|
|
5
6
|
module Fractor
|
|
7
|
+
# Custom exception for shutdown signal handling
|
|
8
|
+
class ShutdownSignal < StandardError; end
|
|
9
|
+
|
|
6
10
|
# Supervises multiple WrappedRactors, distributes work, and aggregates results.
|
|
7
11
|
class Supervisor
|
|
8
12
|
attr_reader :work_queue, :workers, :results, :worker_pools
|
|
@@ -17,12 +21,15 @@ module Fractor
|
|
|
17
21
|
worker_class = pool_config[:worker_class]
|
|
18
22
|
num_workers = pool_config[:num_workers] || detect_num_workers
|
|
19
23
|
|
|
20
|
-
|
|
24
|
+
unless worker_class < Fractor::Worker
|
|
25
|
+
raise ArgumentError,
|
|
26
|
+
"#{worker_class} must inherit from Fractor::Worker"
|
|
27
|
+
end
|
|
21
28
|
|
|
22
29
|
{
|
|
23
30
|
worker_class: worker_class,
|
|
24
31
|
num_workers: num_workers,
|
|
25
|
-
workers: [] # Will hold the WrappedRactor instances
|
|
32
|
+
workers: [], # Will hold the WrappedRactor instances
|
|
26
33
|
}
|
|
27
34
|
end
|
|
28
35
|
|
|
@@ -34,12 +41,18 @@ module Fractor
|
|
|
34
41
|
@continuous_mode = continuous_mode
|
|
35
42
|
@running = false
|
|
36
43
|
@work_callbacks = []
|
|
44
|
+
@wakeup_ractor = nil # Control ractor for unblocking select
|
|
45
|
+
@timer_thread = nil # Timer thread for periodic wakeup
|
|
46
|
+
@idle_workers = [] # Track workers waiting for work
|
|
37
47
|
end
|
|
38
48
|
|
|
39
49
|
# Adds a single work item to the queue.
|
|
40
50
|
# The item must be an instance of Fractor::Work or a subclass.
|
|
41
51
|
def add_work_item(work)
|
|
42
|
-
|
|
52
|
+
unless work.is_a?(Fractor::Work)
|
|
53
|
+
raise ArgumentError,
|
|
54
|
+
"#{work.class} must be an instance of Fractor::Work"
|
|
55
|
+
end
|
|
43
56
|
|
|
44
57
|
@work_queue << work
|
|
45
58
|
@total_work_count += 1
|
|
@@ -64,6 +77,23 @@ module Fractor
|
|
|
64
77
|
|
|
65
78
|
# Starts the worker Ractors for all worker pools.
|
|
66
79
|
def start_workers
|
|
80
|
+
# Create a wakeup Ractor for unblocking Ractor.select
|
|
81
|
+
@wakeup_ractor = Ractor.new do
|
|
82
|
+
puts "Wakeup Ractor started" if ENV["FRACTOR_DEBUG"]
|
|
83
|
+
loop do
|
|
84
|
+
msg = Ractor.receive
|
|
85
|
+
puts "Wakeup Ractor received: #{msg.inspect}" if ENV["FRACTOR_DEBUG"]
|
|
86
|
+
if %i[wakeup shutdown].include?(msg)
|
|
87
|
+
Ractor.yield({ type: :wakeup, message: msg })
|
|
88
|
+
break if msg == :shutdown
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
puts "Wakeup Ractor shutting down" if ENV["FRACTOR_DEBUG"]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Add wakeup ractor to the map with a special marker
|
|
95
|
+
@ractors_map[@wakeup_ractor] = :wakeup
|
|
96
|
+
|
|
67
97
|
@worker_pools.each do |pool|
|
|
68
98
|
worker_class = pool[:worker_class]
|
|
69
99
|
num_workers = pool[:num_workers]
|
|
@@ -85,37 +115,65 @@ module Fractor
|
|
|
85
115
|
puts "Workers started: #{@workers.size} active across #{@worker_pools.size} pools."
|
|
86
116
|
end
|
|
87
117
|
|
|
88
|
-
# Sets up
|
|
118
|
+
# Sets up signal handlers for graceful shutdown.
|
|
119
|
+
# Handles SIGINT (Ctrl+C), SIGTERM (systemd/docker), and platform-specific status signals.
|
|
89
120
|
def setup_signal_handler
|
|
90
|
-
#
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
# Trap INT signal (Ctrl+C)
|
|
94
|
-
Signal.trap("INT") do
|
|
95
|
-
puts "\nCtrl+C received. Initiating immediate shutdown..." if ENV["FRACTOR_DEBUG"]
|
|
121
|
+
# Universal signals (work on all platforms)
|
|
122
|
+
Signal.trap("INT") { handle_shutdown("SIGINT") }
|
|
123
|
+
Signal.trap("TERM") { handle_shutdown("SIGTERM") }
|
|
96
124
|
|
|
97
|
-
|
|
98
|
-
|
|
125
|
+
# Platform-specific status monitoring
|
|
126
|
+
setup_status_signal
|
|
127
|
+
end
|
|
99
128
|
|
|
100
|
-
|
|
129
|
+
# Handles shutdown signal by mode (continuous vs batch)
|
|
130
|
+
def handle_shutdown(signal_name)
|
|
131
|
+
if @continuous_mode
|
|
132
|
+
puts "\n#{signal_name} received. Initiating graceful shutdown..." if ENV["FRACTOR_DEBUG"]
|
|
133
|
+
stop
|
|
134
|
+
else
|
|
135
|
+
puts "\n#{signal_name} received. Initiating immediate shutdown..." if ENV["FRACTOR_DEBUG"]
|
|
136
|
+
Thread.current.raise(ShutdownSignal, "Interrupted by #{signal_name}")
|
|
137
|
+
end
|
|
138
|
+
rescue Exception => e
|
|
139
|
+
puts "Error in signal handler: #{e.class}: #{e.message}" if ENV["FRACTOR_DEBUG"]
|
|
140
|
+
puts e.backtrace.join("\n") if ENV["FRACTOR_DEBUG"]
|
|
141
|
+
exit!(1)
|
|
142
|
+
end
|
|
101
143
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
144
|
+
# Sets up platform-specific status monitoring signal
|
|
145
|
+
def setup_status_signal
|
|
146
|
+
if Gem.win_platform?
|
|
147
|
+
# Windows: Try SIGBREAK (Ctrl+Break) if available
|
|
148
|
+
begin
|
|
149
|
+
Signal.trap("BREAK") { print_status }
|
|
150
|
+
rescue ArgumentError
|
|
151
|
+
# SIGBREAK not supported on this Ruby version/platform
|
|
152
|
+
# Status monitoring unavailable on Windows
|
|
153
|
+
end
|
|
154
|
+
else
|
|
155
|
+
# Unix/Linux/macOS: Use SIGUSR1
|
|
156
|
+
begin
|
|
157
|
+
Signal.trap("USR1") { print_status }
|
|
158
|
+
rescue ArgumentError
|
|
159
|
+
# SIGUSR1 not supported on this platform
|
|
108
160
|
end
|
|
109
|
-
|
|
110
|
-
puts "Exiting now." if ENV["FRACTOR_DEBUG"]
|
|
111
|
-
exit!(1) # Use exit! to exit immediately without running at_exit handlers
|
|
112
|
-
rescue Exception => e
|
|
113
|
-
puts "Error in signal handler: #{e.class}: #{e.message}" if ENV["FRACTOR_DEBUG"]
|
|
114
|
-
puts e.backtrace.join("\n") if ENV["FRACTOR_DEBUG"]
|
|
115
|
-
exit!(1)
|
|
116
161
|
end
|
|
117
162
|
end
|
|
118
163
|
|
|
164
|
+
# Prints current supervisor status
|
|
165
|
+
def print_status
|
|
166
|
+
puts "\n=== Fractor Supervisor Status ==="
|
|
167
|
+
puts "Mode: #{@continuous_mode ? 'Continuous' : 'Batch'}"
|
|
168
|
+
puts "Running: #{@running}"
|
|
169
|
+
puts "Workers: #{@workers.size}"
|
|
170
|
+
puts "Idle workers: #{@idle_workers.size}"
|
|
171
|
+
puts "Queue size: #{@work_queue.size}"
|
|
172
|
+
puts "Results: #{@results.results.size}"
|
|
173
|
+
puts "Errors: #{@results.errors.size}"
|
|
174
|
+
puts "================================\n"
|
|
175
|
+
end
|
|
176
|
+
|
|
119
177
|
# Runs the main processing loop.
|
|
120
178
|
def run
|
|
121
179
|
setup_signal_handler
|
|
@@ -124,92 +182,174 @@ module Fractor
|
|
|
124
182
|
@running = true
|
|
125
183
|
processed_count = 0
|
|
126
184
|
|
|
127
|
-
#
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
185
|
+
# Start timer thread for continuous mode to periodically check work sources
|
|
186
|
+
if @continuous_mode && !@work_callbacks.empty?
|
|
187
|
+
@timer_thread = Thread.new do
|
|
188
|
+
while @running
|
|
189
|
+
sleep(0.1) # Check work sources every 100ms
|
|
190
|
+
if @wakeup_ractor && @running
|
|
191
|
+
begin
|
|
192
|
+
@wakeup_ractor.send(:wakeup)
|
|
193
|
+
rescue StandardError => e
|
|
194
|
+
puts "Timer thread error sending wakeup: #{e.message}" if ENV["FRACTOR_DEBUG"]
|
|
195
|
+
break
|
|
196
|
+
end
|
|
197
|
+
end
|
|
136
198
|
end
|
|
199
|
+
puts "Timer thread shutting down" if ENV["FRACTOR_DEBUG"]
|
|
137
200
|
end
|
|
201
|
+
end
|
|
138
202
|
|
|
139
|
-
|
|
140
|
-
|
|
203
|
+
begin
|
|
204
|
+
# Main loop: Process events until conditions are met for termination
|
|
205
|
+
while @running && (@continuous_mode || processed_count < @total_work_count)
|
|
206
|
+
processed_count = @results.results.size + @results.errors.size
|
|
141
207
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
208
|
+
if ENV["FRACTOR_DEBUG"]
|
|
209
|
+
if @continuous_mode
|
|
210
|
+
puts "Continuous mode: Waiting for Ractor results. Processed: #{processed_count}, Queue size: #{@work_queue.size}"
|
|
211
|
+
else
|
|
212
|
+
puts "Waiting for Ractor results. Processed: #{processed_count}/#{@total_work_count}, Queue size: #{@work_queue.size}"
|
|
213
|
+
end
|
|
147
214
|
end
|
|
148
|
-
end
|
|
149
215
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
216
|
+
# Get active Ractor objects from the map keys
|
|
217
|
+
active_ractors = @ractors_map.keys
|
|
218
|
+
|
|
219
|
+
# Check for new work from callbacks if in continuous mode
|
|
220
|
+
if @continuous_mode && !@work_callbacks.empty?
|
|
221
|
+
@work_callbacks.each do |callback|
|
|
222
|
+
new_work = callback.call
|
|
223
|
+
if new_work && !new_work.empty?
|
|
224
|
+
add_work_items(new_work)
|
|
225
|
+
puts "Work source provided #{new_work.size} new items" if ENV["FRACTOR_DEBUG"]
|
|
226
|
+
|
|
227
|
+
# Try to send work to idle workers first
|
|
228
|
+
while !@work_queue.empty? && !@idle_workers.empty?
|
|
229
|
+
worker = @idle_workers.shift
|
|
230
|
+
if send_next_work_if_available(worker)
|
|
231
|
+
puts "Sent work to idle worker #{worker.name}" if ENV["FRACTOR_DEBUG"]
|
|
232
|
+
else
|
|
233
|
+
# Worker couldn't accept work, don't re-add to idle list
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|
|
155
239
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
240
|
+
# Break if no active workers and queue is empty, but work remains (indicates potential issue)
|
|
241
|
+
if active_ractors.empty? && @work_queue.empty? && !@continuous_mode && processed_count < @total_work_count
|
|
242
|
+
puts "Warning: No active workers and queue is empty, but not all work is processed. Exiting loop." if ENV["FRACTOR_DEBUG"]
|
|
243
|
+
break
|
|
244
|
+
end
|
|
159
245
|
|
|
160
|
-
|
|
161
|
-
|
|
246
|
+
# In continuous mode, just wait if no active ractors but keep running
|
|
247
|
+
if active_ractors.empty?
|
|
248
|
+
break unless @continuous_mode
|
|
162
249
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
ready_ractor_obj, message = Ractor.select(*active_ractors)
|
|
250
|
+
sleep(0.1) # Small delay to avoid CPU spinning
|
|
251
|
+
next
|
|
252
|
+
end
|
|
167
253
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
254
|
+
# Ractor.select blocks until a message is available from any active Ractor
|
|
255
|
+
# The wakeup ractor ensures we can unblock this call when needed
|
|
256
|
+
ready_ractor_obj, message = Ractor.select(*active_ractors)
|
|
257
|
+
|
|
258
|
+
# Check if this is the wakeup ractor
|
|
259
|
+
if ready_ractor_obj == @wakeup_ractor
|
|
260
|
+
puts "Wakeup signal received: #{message[:message]}" if ENV["FRACTOR_DEBUG"]
|
|
261
|
+
# Remove wakeup ractor from map if shutting down
|
|
262
|
+
if message[:message] == :shutdown
|
|
263
|
+
@ractors_map.delete(@wakeup_ractor)
|
|
264
|
+
@wakeup_ractor = nil
|
|
265
|
+
end
|
|
266
|
+
# Continue loop to check @running flag
|
|
267
|
+
next
|
|
268
|
+
end
|
|
174
269
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
puts "Ractor initialized: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
|
|
181
|
-
# Send work immediately upon initialization if available
|
|
182
|
-
send_next_work_if_available(wrapped_ractor)
|
|
183
|
-
when :result
|
|
184
|
-
# The message[:result] should be a WorkResult object
|
|
185
|
-
work_result = message[:result]
|
|
186
|
-
puts "Completed work: #{work_result.inspect} in Ractor: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
|
|
187
|
-
@results.add_result(work_result)
|
|
188
|
-
if ENV["FRACTOR_DEBUG"]
|
|
189
|
-
puts "Result processed. Total processed: #{@results.results.size + @results.errors.size}"
|
|
190
|
-
puts "Aggregated Results: #{@results.inspect}" unless @continuous_mode
|
|
270
|
+
# Find the corresponding WrappedRactor instance
|
|
271
|
+
wrapped_ractor = @ractors_map[ready_ractor_obj]
|
|
272
|
+
unless wrapped_ractor
|
|
273
|
+
puts "Warning: Received message from unknown Ractor: #{ready_ractor_obj}. Ignoring." if ENV["FRACTOR_DEBUG"]
|
|
274
|
+
next
|
|
191
275
|
end
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
#
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
276
|
+
|
|
277
|
+
puts "Selected Ractor: #{wrapped_ractor.name}, Message Type: #{message[:type]}" if ENV["FRACTOR_DEBUG"]
|
|
278
|
+
|
|
279
|
+
# Process the received message
|
|
280
|
+
case message[:type]
|
|
281
|
+
when :initialize
|
|
282
|
+
puts "Ractor initialized: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
|
|
283
|
+
# Send work immediately upon initialization if available
|
|
284
|
+
if send_next_work_if_available(wrapped_ractor)
|
|
285
|
+
# Work was sent
|
|
286
|
+
else
|
|
287
|
+
# No work available, mark worker as idle
|
|
288
|
+
@idle_workers << wrapped_ractor unless @idle_workers.include?(wrapped_ractor)
|
|
289
|
+
puts "Worker #{wrapped_ractor.name} marked as idle" if ENV["FRACTOR_DEBUG"]
|
|
290
|
+
end
|
|
291
|
+
when :shutdown
|
|
292
|
+
puts "Ractor #{wrapped_ractor.name} acknowledged shutdown" if ENV["FRACTOR_DEBUG"]
|
|
293
|
+
# Remove from active ractors
|
|
294
|
+
@ractors_map.delete(ready_ractor_obj)
|
|
295
|
+
when :result
|
|
296
|
+
# The message[:result] should be a WorkResult object
|
|
297
|
+
work_result = message[:result]
|
|
298
|
+
puts "Completed work: #{work_result.inspect} in Ractor: #{message[:processor]}" if ENV["FRACTOR_DEBUG"]
|
|
299
|
+
@results.add_result(work_result)
|
|
300
|
+
if ENV["FRACTOR_DEBUG"]
|
|
301
|
+
puts "Result processed. Total processed: #{@results.results.size + @results.errors.size}"
|
|
302
|
+
puts "Aggregated Results: #{@results.inspect}" unless @continuous_mode
|
|
303
|
+
end
|
|
304
|
+
# Send next piece of work
|
|
305
|
+
if send_next_work_if_available(wrapped_ractor)
|
|
306
|
+
# Work was sent
|
|
307
|
+
else
|
|
308
|
+
# No work available, mark worker as idle
|
|
309
|
+
@idle_workers << wrapped_ractor unless @idle_workers.include?(wrapped_ractor)
|
|
310
|
+
puts "Worker #{wrapped_ractor.name} marked as idle after completing work" if ENV["FRACTOR_DEBUG"]
|
|
311
|
+
end
|
|
312
|
+
when :error
|
|
313
|
+
# The message[:result] should be a WorkResult object containing the error
|
|
314
|
+
error_result = message[:result]
|
|
315
|
+
puts "Error processing work #{error_result.work&.inspect} in Ractor: #{message[:processor]}: #{error_result.error}" if ENV["FRACTOR_DEBUG"]
|
|
316
|
+
@results.add_result(error_result) # Add error to aggregator
|
|
317
|
+
if ENV["FRACTOR_DEBUG"]
|
|
318
|
+
puts "Error handled. Total processed: #{@results.results.size + @results.errors.size}"
|
|
319
|
+
puts "Aggregated Results (including errors): #{@results.inspect}" unless @continuous_mode
|
|
320
|
+
end
|
|
321
|
+
# Send next piece of work even after an error
|
|
322
|
+
if send_next_work_if_available(wrapped_ractor)
|
|
323
|
+
# Work was sent
|
|
324
|
+
else
|
|
325
|
+
# No work available, mark worker as idle
|
|
326
|
+
@idle_workers << wrapped_ractor unless @idle_workers.include?(wrapped_ractor)
|
|
327
|
+
puts "Worker #{wrapped_ractor.name} marked as idle after error" if ENV["FRACTOR_DEBUG"]
|
|
328
|
+
end
|
|
329
|
+
else
|
|
330
|
+
puts "Unknown message type received: #{message[:type]} from #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
|
|
202
331
|
end
|
|
203
|
-
#
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
332
|
+
# Update processed count for the loop condition
|
|
333
|
+
processed_count = @results.results.size + @results.errors.size
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
puts "Main loop finished." if ENV["FRACTOR_DEBUG"]
|
|
337
|
+
rescue ShutdownSignal => e
|
|
338
|
+
puts "Shutdown signal caught: #{e.message}" if ENV["FRACTOR_DEBUG"]
|
|
339
|
+
puts "Sending shutdown message to all Ractors..." if ENV["FRACTOR_DEBUG"]
|
|
340
|
+
|
|
341
|
+
# Send shutdown message to each worker Ractor
|
|
342
|
+
@workers.each do |w|
|
|
343
|
+
w.send(:shutdown)
|
|
344
|
+
puts "Sent shutdown to Ractor: #{w.name}" if ENV["FRACTOR_DEBUG"]
|
|
345
|
+
rescue StandardError => send_error
|
|
346
|
+
puts "Error sending shutdown to Ractor #{w.name}: #{send_error.message}" if ENV["FRACTOR_DEBUG"]
|
|
207
347
|
end
|
|
208
|
-
|
|
209
|
-
|
|
348
|
+
|
|
349
|
+
puts "Exiting due to shutdown signal." if ENV["FRACTOR_DEBUG"]
|
|
350
|
+
exit!(1) # Force exit immediately
|
|
210
351
|
end
|
|
211
352
|
|
|
212
|
-
puts "Main loop finished." if ENV["FRACTOR_DEBUG"]
|
|
213
353
|
return if @continuous_mode
|
|
214
354
|
|
|
215
355
|
return unless ENV["FRACTOR_DEBUG"]
|
|
@@ -221,6 +361,32 @@ module Fractor
|
|
|
221
361
|
def stop
|
|
222
362
|
@running = false
|
|
223
363
|
puts "Stopping supervisor..." if ENV["FRACTOR_DEBUG"]
|
|
364
|
+
|
|
365
|
+
# Wait for timer thread to finish if it exists
|
|
366
|
+
if @timer_thread&.alive?
|
|
367
|
+
@timer_thread.join(1) # Wait up to 1 second
|
|
368
|
+
puts "Timer thread stopped" if ENV["FRACTOR_DEBUG"]
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# Signal the wakeup ractor first to unblock Ractor.select
|
|
372
|
+
if @wakeup_ractor
|
|
373
|
+
begin
|
|
374
|
+
@wakeup_ractor.send(:shutdown)
|
|
375
|
+
puts "Sent shutdown signal to wakeup ractor" if ENV["FRACTOR_DEBUG"]
|
|
376
|
+
rescue StandardError => e
|
|
377
|
+
puts "Error sending shutdown to wakeup ractor: #{e.message}" if ENV["FRACTOR_DEBUG"]
|
|
378
|
+
end
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
# Send shutdown signal to all workers
|
|
382
|
+
@workers.each do |w|
|
|
383
|
+
begin
|
|
384
|
+
w.send(:shutdown)
|
|
385
|
+
rescue StandardError
|
|
386
|
+
nil
|
|
387
|
+
end
|
|
388
|
+
puts "Sent shutdown signal to #{w.name}" if ENV["FRACTOR_DEBUG"]
|
|
389
|
+
end
|
|
224
390
|
end
|
|
225
391
|
|
|
226
392
|
private
|
|
@@ -237,16 +403,11 @@ module Fractor
|
|
|
237
403
|
end
|
|
238
404
|
|
|
239
405
|
# Helper method to send the next available work item to a specific Ractor.
|
|
406
|
+
# Returns true if work was sent, false otherwise.
|
|
240
407
|
def send_next_work_if_available(wrapped_ractor)
|
|
241
408
|
# Ensure the wrapped_ractor instance is valid and its underlying ractor is not closed
|
|
242
409
|
if wrapped_ractor && !wrapped_ractor.closed?
|
|
243
|
-
if
|
|
244
|
-
work_item = @work_queue.pop # Now directly a Work object
|
|
245
|
-
|
|
246
|
-
puts "Sending next work #{work_item.inspect} to Ractor: #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
|
|
247
|
-
wrapped_ractor.send(work_item) # Send the Work object
|
|
248
|
-
puts "Work sent to #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
|
|
249
|
-
else
|
|
410
|
+
if @work_queue.empty?
|
|
250
411
|
puts "Work queue empty. Not sending new work to Ractor #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
|
|
251
412
|
# In continuous mode, don't close workers as more work may come
|
|
252
413
|
unless @continuous_mode
|
|
@@ -257,11 +418,23 @@ module Fractor
|
|
|
257
418
|
# puts "Closed idle Ractor: #{wrapped_ractor.name}"
|
|
258
419
|
# end
|
|
259
420
|
end
|
|
421
|
+
false
|
|
422
|
+
else
|
|
423
|
+
work_item = @work_queue.pop # Now directly a Work object
|
|
424
|
+
|
|
425
|
+
puts "Sending next work #{work_item.inspect} to Ractor: #{wrapped_ractor.name}" if ENV["FRACTOR_DEBUG"]
|
|
426
|
+
wrapped_ractor.send(work_item) # Send the Work object
|
|
427
|
+
puts "Work sent to #{wrapped_ractor.name}." if ENV["FRACTOR_DEBUG"]
|
|
428
|
+
|
|
429
|
+
# Remove from idle workers list since it's now busy
|
|
430
|
+
@idle_workers.delete(wrapped_ractor)
|
|
431
|
+
true
|
|
260
432
|
end
|
|
261
433
|
else
|
|
262
|
-
puts "Attempted to send work to an invalid or closed Ractor: #{wrapped_ractor&.name ||
|
|
434
|
+
puts "Attempted to send work to an invalid or closed Ractor: #{wrapped_ractor&.name || 'unknown'}." if ENV["FRACTOR_DEBUG"]
|
|
263
435
|
# Remove from map if found but closed
|
|
264
436
|
@ractors_map.delete(wrapped_ractor.ractor) if wrapped_ractor && @ractors_map.key?(wrapped_ractor.ractor)
|
|
437
|
+
false
|
|
265
438
|
end
|
|
266
439
|
end
|
|
267
440
|
end
|
data/lib/fractor/version.rb
CHANGED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Fractor
|
|
4
|
+
# Thread-safe work queue for continuous mode applications.
|
|
5
|
+
# Provides a simple interface for adding work items and retrieving them
|
|
6
|
+
# in batches, with automatic integration with Fractor::Supervisor.
|
|
7
|
+
class WorkQueue
|
|
8
|
+
attr_reader :queue
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@queue = Thread::Queue.new
|
|
12
|
+
@mutex = Mutex.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Add a work item to the queue (thread-safe)
|
|
16
|
+
# @param work_item [Fractor::Work] The work item to add
|
|
17
|
+
# @return [void]
|
|
18
|
+
def <<(work_item)
|
|
19
|
+
unless work_item.is_a?(Fractor::Work)
|
|
20
|
+
raise ArgumentError,
|
|
21
|
+
"#{work_item.class} must be an instance of Fractor::Work"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
@queue << work_item
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Retrieve multiple work items from the queue in a single operation
|
|
28
|
+
# @param max_items [Integer] Maximum number of items to retrieve
|
|
29
|
+
# @return [Array<Fractor::Work>] Array of work items (may be empty)
|
|
30
|
+
def pop_batch(max_items = 10)
|
|
31
|
+
items = []
|
|
32
|
+
max_items.times do
|
|
33
|
+
break if @queue.empty?
|
|
34
|
+
|
|
35
|
+
begin
|
|
36
|
+
items << @queue.pop(true)
|
|
37
|
+
rescue ThreadError
|
|
38
|
+
# Queue became empty between check and pop
|
|
39
|
+
break
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
items
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Check if the queue is empty
|
|
46
|
+
# @return [Boolean] true if the queue is empty
|
|
47
|
+
def empty?
|
|
48
|
+
@queue.empty?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Get the current size of the queue
|
|
52
|
+
# @return [Integer] Number of items in the queue
|
|
53
|
+
def size
|
|
54
|
+
@queue.size
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Register this work queue as a work source with a supervisor
|
|
58
|
+
# @param supervisor [Fractor::Supervisor] The supervisor to register with
|
|
59
|
+
# @param batch_size [Integer] Number of items to retrieve per poll
|
|
60
|
+
# @return [void]
|
|
61
|
+
def register_with_supervisor(supervisor, batch_size: 10)
|
|
62
|
+
supervisor.register_work_source do
|
|
63
|
+
items = pop_batch(batch_size)
|
|
64
|
+
items.empty? ? nil : items
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
data/lib/fractor/work_result.rb
CHANGED
data/lib/fractor/worker.rb
CHANGED
|
@@ -35,6 +35,8 @@ module Fractor
|
|
|
35
35
|
# Handle shutdown message
|
|
36
36
|
if work == :shutdown
|
|
37
37
|
puts "Received shutdown message in Ractor #{name}, terminating..." if ENV["FRACTOR_DEBUG"]
|
|
38
|
+
# Yield a shutdown acknowledgment before terminating
|
|
39
|
+
Ractor.yield({ type: :shutdown, processor: name })
|
|
38
40
|
break
|
|
39
41
|
end
|
|
40
42
|
|
|
@@ -44,15 +46,23 @@ module Fractor
|
|
|
44
46
|
# Process the work using the instantiated worker
|
|
45
47
|
result = worker.process(work)
|
|
46
48
|
puts "Sending result #{result.inspect} from Ractor #{name}" if ENV["FRACTOR_DEBUG"]
|
|
49
|
+
# Wrap the result in a WorkResult object if not already wrapped
|
|
50
|
+
work_result = if result.is_a?(Fractor::WorkResult)
|
|
51
|
+
result
|
|
52
|
+
else
|
|
53
|
+
Fractor::WorkResult.new(result: result, work: work)
|
|
54
|
+
end
|
|
47
55
|
# Yield the result back
|
|
48
|
-
Ractor.yield({ type: :result, result:
|
|
56
|
+
Ractor.yield({ type: :result, result: work_result,
|
|
57
|
+
processor: name })
|
|
49
58
|
rescue StandardError => e
|
|
50
59
|
# Handle errors during processing
|
|
51
60
|
puts "Error processing work #{work.inspect} in Ractor #{name}: #{e.message}\n#{e.backtrace.join("\n")}" if ENV["FRACTOR_DEBUG"]
|
|
52
61
|
# Yield an error message back
|
|
53
62
|
# Ensure the original work object is included in the error result
|
|
54
63
|
error_result = Fractor::WorkResult.new(error: e.message, work: work)
|
|
55
|
-
Ractor.yield({ type: :error, result: error_result,
|
|
64
|
+
Ractor.yield({ type: :error, result: error_result,
|
|
65
|
+
processor: name })
|
|
56
66
|
end
|
|
57
67
|
end
|
|
58
68
|
rescue Ractor::ClosedError
|
data/lib/fractor.rb
CHANGED
|
@@ -8,6 +8,8 @@ require_relative "fractor/work_result"
|
|
|
8
8
|
require_relative "fractor/result_aggregator"
|
|
9
9
|
require_relative "fractor/wrapped_ractor"
|
|
10
10
|
require_relative "fractor/supervisor"
|
|
11
|
+
require_relative "fractor/work_queue"
|
|
12
|
+
require_relative "fractor/continuous_server"
|
|
11
13
|
|
|
12
14
|
# Fractor: Function-driven Ractors framework
|
|
13
15
|
module Fractor
|