fractor 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +227 -102
  3. data/README.adoc +113 -1940
  4. data/docs/.lycheeignore +16 -0
  5. data/docs/Gemfile +24 -0
  6. data/docs/README.md +157 -0
  7. data/docs/_config.yml +151 -0
  8. data/docs/_features/error-handling.adoc +1192 -0
  9. data/docs/_features/index.adoc +80 -0
  10. data/docs/_features/monitoring.adoc +589 -0
  11. data/docs/_features/signal-handling.adoc +202 -0
  12. data/docs/_features/workflows.adoc +1235 -0
  13. data/docs/_guides/continuous-mode.adoc +736 -0
  14. data/docs/_guides/cookbook.adoc +1133 -0
  15. data/docs/_guides/index.adoc +55 -0
  16. data/docs/_guides/pipeline-mode.adoc +730 -0
  17. data/docs/_guides/troubleshooting.adoc +358 -0
  18. data/docs/_pages/architecture.adoc +1390 -0
  19. data/docs/_pages/core-concepts.adoc +1392 -0
  20. data/docs/_pages/design-principles.adoc +862 -0
  21. data/docs/_pages/getting-started.adoc +290 -0
  22. data/docs/_pages/installation.adoc +143 -0
  23. data/docs/_reference/api.adoc +1080 -0
  24. data/docs/_reference/error-reporting.adoc +670 -0
  25. data/docs/_reference/examples.adoc +181 -0
  26. data/docs/_reference/index.adoc +96 -0
  27. data/docs/_reference/troubleshooting.adoc +862 -0
  28. data/docs/_tutorials/complex-workflows.adoc +1022 -0
  29. data/docs/_tutorials/data-processing-pipeline.adoc +740 -0
  30. data/docs/_tutorials/first-application.adoc +384 -0
  31. data/docs/_tutorials/index.adoc +48 -0
  32. data/docs/_tutorials/long-running-services.adoc +931 -0
  33. data/docs/assets/images/favicon-16.png +0 -0
  34. data/docs/assets/images/favicon-32.png +0 -0
  35. data/docs/assets/images/favicon-48.png +0 -0
  36. data/docs/assets/images/favicon.ico +0 -0
  37. data/docs/assets/images/favicon.png +0 -0
  38. data/docs/assets/images/favicon.svg +45 -0
  39. data/docs/assets/images/fractor-icon.svg +49 -0
  40. data/docs/assets/images/fractor-logo.svg +61 -0
  41. data/docs/index.adoc +131 -0
  42. data/docs/lychee.toml +39 -0
  43. data/examples/api_aggregator/README.adoc +627 -0
  44. data/examples/api_aggregator/api_aggregator.rb +376 -0
  45. data/examples/auto_detection/README.adoc +407 -29
  46. data/examples/continuous_chat_common/message_protocol.rb +1 -1
  47. data/examples/error_reporting.rb +207 -0
  48. data/examples/file_processor/README.adoc +170 -0
  49. data/examples/file_processor/file_processor.rb +615 -0
  50. data/examples/file_processor/sample_files/invalid.csv +1 -0
  51. data/examples/file_processor/sample_files/orders.xml +24 -0
  52. data/examples/file_processor/sample_files/products.json +23 -0
  53. data/examples/file_processor/sample_files/users.csv +6 -0
  54. data/examples/hierarchical_hasher/README.adoc +629 -41
  55. data/examples/image_processor/README.adoc +610 -0
  56. data/examples/image_processor/image_processor.rb +349 -0
  57. data/examples/image_processor/processed_images/sample_10_processed.jpg.json +12 -0
  58. data/examples/image_processor/processed_images/sample_1_processed.jpg.json +12 -0
  59. data/examples/image_processor/processed_images/sample_2_processed.jpg.json +12 -0
  60. data/examples/image_processor/processed_images/sample_3_processed.jpg.json +12 -0
  61. data/examples/image_processor/processed_images/sample_4_processed.jpg.json +12 -0
  62. data/examples/image_processor/processed_images/sample_5_processed.jpg.json +12 -0
  63. data/examples/image_processor/processed_images/sample_6_processed.jpg.json +12 -0
  64. data/examples/image_processor/processed_images/sample_7_processed.jpg.json +12 -0
  65. data/examples/image_processor/processed_images/sample_8_processed.jpg.json +12 -0
  66. data/examples/image_processor/processed_images/sample_9_processed.jpg.json +12 -0
  67. data/examples/image_processor/test_images/sample_1.png +1 -0
  68. data/examples/image_processor/test_images/sample_10.png +1 -0
  69. data/examples/image_processor/test_images/sample_2.png +1 -0
  70. data/examples/image_processor/test_images/sample_3.png +1 -0
  71. data/examples/image_processor/test_images/sample_4.png +1 -0
  72. data/examples/image_processor/test_images/sample_5.png +1 -0
  73. data/examples/image_processor/test_images/sample_6.png +1 -0
  74. data/examples/image_processor/test_images/sample_7.png +1 -0
  75. data/examples/image_processor/test_images/sample_8.png +1 -0
  76. data/examples/image_processor/test_images/sample_9.png +1 -0
  77. data/examples/log_analyzer/README.adoc +662 -0
  78. data/examples/log_analyzer/log_analyzer.rb +579 -0
  79. data/examples/log_analyzer/sample_logs/apache.log +20 -0
  80. data/examples/log_analyzer/sample_logs/json.log +15 -0
  81. data/examples/log_analyzer/sample_logs/nginx.log +15 -0
  82. data/examples/log_analyzer/sample_logs/rails.log +29 -0
  83. data/examples/multi_work_type/README.adoc +576 -26
  84. data/examples/performance_monitoring.rb +120 -0
  85. data/examples/pipeline_processing/README.adoc +740 -26
  86. data/examples/pipeline_processing/pipeline_processing.rb +2 -2
  87. data/examples/priority_work_example.rb +155 -0
  88. data/examples/producer_subscriber/README.adoc +889 -46
  89. data/examples/scatter_gather/README.adoc +829 -27
  90. data/examples/simple/README.adoc +347 -0
  91. data/examples/specialized_workers/README.adoc +622 -26
  92. data/examples/specialized_workers/specialized_workers.rb +44 -8
  93. data/examples/stream_processor/README.adoc +206 -0
  94. data/examples/stream_processor/stream_processor.rb +284 -0
  95. data/examples/web_scraper/README.adoc +625 -0
  96. data/examples/web_scraper/web_scraper.rb +285 -0
  97. data/examples/workflow/README.adoc +406 -0
  98. data/examples/workflow/circuit_breaker/README.adoc +360 -0
  99. data/examples/workflow/circuit_breaker/circuit_breaker_workflow.rb +225 -0
  100. data/examples/workflow/conditional/README.adoc +483 -0
  101. data/examples/workflow/conditional/conditional_workflow.rb +215 -0
  102. data/examples/workflow/dead_letter_queue/README.adoc +374 -0
  103. data/examples/workflow/dead_letter_queue/dead_letter_queue_workflow.rb +217 -0
  104. data/examples/workflow/fan_out/README.adoc +381 -0
  105. data/examples/workflow/fan_out/fan_out_workflow.rb +202 -0
  106. data/examples/workflow/retry/README.adoc +248 -0
  107. data/examples/workflow/retry/retry_workflow.rb +195 -0
  108. data/examples/workflow/simple_linear/README.adoc +267 -0
  109. data/examples/workflow/simple_linear/simple_linear_workflow.rb +175 -0
  110. data/examples/workflow/simplified/README.adoc +329 -0
  111. data/examples/workflow/simplified/simplified_workflow.rb +222 -0
  112. data/exe/fractor +10 -0
  113. data/lib/fractor/cli.rb +288 -0
  114. data/lib/fractor/configuration.rb +307 -0
  115. data/lib/fractor/continuous_server.rb +60 -65
  116. data/lib/fractor/error_formatter.rb +72 -0
  117. data/lib/fractor/error_report_generator.rb +152 -0
  118. data/lib/fractor/error_reporter.rb +244 -0
  119. data/lib/fractor/error_statistics.rb +147 -0
  120. data/lib/fractor/execution_tracer.rb +162 -0
  121. data/lib/fractor/logger.rb +230 -0
  122. data/lib/fractor/main_loop_handler.rb +406 -0
  123. data/lib/fractor/main_loop_handler3.rb +135 -0
  124. data/lib/fractor/main_loop_handler4.rb +299 -0
  125. data/lib/fractor/performance_metrics_collector.rb +181 -0
  126. data/lib/fractor/performance_monitor.rb +215 -0
  127. data/lib/fractor/performance_report_generator.rb +202 -0
  128. data/lib/fractor/priority_work.rb +93 -0
  129. data/lib/fractor/priority_work_queue.rb +189 -0
  130. data/lib/fractor/result_aggregator.rb +32 -0
  131. data/lib/fractor/shutdown_handler.rb +168 -0
  132. data/lib/fractor/signal_handler.rb +80 -0
  133. data/lib/fractor/supervisor.rb +382 -269
  134. data/lib/fractor/supervisor_logger.rb +88 -0
  135. data/lib/fractor/version.rb +1 -1
  136. data/lib/fractor/work.rb +12 -0
  137. data/lib/fractor/work_distribution_manager.rb +151 -0
  138. data/lib/fractor/work_queue.rb +20 -0
  139. data/lib/fractor/work_result.rb +181 -9
  140. data/lib/fractor/worker.rb +73 -0
  141. data/lib/fractor/workflow/builder.rb +210 -0
  142. data/lib/fractor/workflow/chain_builder.rb +169 -0
  143. data/lib/fractor/workflow/circuit_breaker.rb +183 -0
  144. data/lib/fractor/workflow/circuit_breaker_orchestrator.rb +208 -0
  145. data/lib/fractor/workflow/circuit_breaker_registry.rb +112 -0
  146. data/lib/fractor/workflow/dead_letter_queue.rb +334 -0
  147. data/lib/fractor/workflow/execution_hooks.rb +39 -0
  148. data/lib/fractor/workflow/execution_strategy.rb +225 -0
  149. data/lib/fractor/workflow/execution_trace.rb +134 -0
  150. data/lib/fractor/workflow/helpers.rb +191 -0
  151. data/lib/fractor/workflow/job.rb +290 -0
  152. data/lib/fractor/workflow/job_dependency_validator.rb +120 -0
  153. data/lib/fractor/workflow/logger.rb +110 -0
  154. data/lib/fractor/workflow/pre_execution_context.rb +193 -0
  155. data/lib/fractor/workflow/retry_config.rb +156 -0
  156. data/lib/fractor/workflow/retry_orchestrator.rb +184 -0
  157. data/lib/fractor/workflow/retry_strategy.rb +93 -0
  158. data/lib/fractor/workflow/structured_logger.rb +30 -0
  159. data/lib/fractor/workflow/type_compatibility_validator.rb +222 -0
  160. data/lib/fractor/workflow/visualizer.rb +211 -0
  161. data/lib/fractor/workflow/workflow_context.rb +132 -0
  162. data/lib/fractor/workflow/workflow_executor.rb +669 -0
  163. data/lib/fractor/workflow/workflow_result.rb +55 -0
  164. data/lib/fractor/workflow/workflow_validator.rb +295 -0
  165. data/lib/fractor/workflow.rb +333 -0
  166. data/lib/fractor/wrapped_ractor.rb +66 -101
  167. data/lib/fractor/wrapped_ractor3.rb +161 -0
  168. data/lib/fractor/wrapped_ractor4.rb +242 -0
  169. data/lib/fractor.rb +92 -4
  170. metadata +179 -6
  171. data/tests/sample.rb.bak +0 -309
  172. data/tests/sample_working.rb.bak +0 -209
@@ -0,0 +1,1133 @@
1
+ ---
2
+ layout: default
3
+ title: Cookbook
4
+ nav_order: 4
5
+ ---
6
+
7
+ == Cookbook
8
+
9
+ === Overview
10
+
11
+ This cookbook provides ready-to-use patterns and recipes for common Fractor use cases. Each pattern includes complete code examples, when to use it, and best practices.
12
+
13
+ === Pattern Index
14
+
15
+ 1. <<batch-processing,Batch Processing>>
16
+ 2. <<file-processing,File Processing>>
17
+ 3. <<api-rate-limiting,API Rate Limiting>>
18
+ 4. <<producer-consumer,Producer-Consumer>>
19
+ 5. <<fan-out-fan-in,Fan-Out/Fan-In>>
20
+ 6. <<retry-with-backoff,Retry with Backoff>>
21
+ 7. <<circuit-breaker-pattern,Circuit Breaker>>
22
+ 8. <<dead-letter-queue,Dead Letter Queue>>
23
+ 9. <<priority-queues,Priority Queues>>
24
+ 10. <<streaming-data,Streaming Data Processing>>
25
+ 11. <<parallel-aggregation,Parallel Aggregation>>
26
+ 12. <<worker-pools,Multi-Type Worker Pools>>
27
+
28
+ ---
29
+
30
+ [[batch-processing]]
31
+ === Pattern 1: Batch Processing
32
+
33
+ Process large batches of items in parallel.
34
+
35
+ ==== When to Use
36
+
37
+ * Processing large datasets
38
+ * Batch ETL operations
39
+ * Bulk data transformations
40
+ * Report generation
41
+
42
+ ==== Example
43
+
44
+ [source,ruby]
45
+ ----
46
+ require 'fractor'
47
+
48
+ class BatchWork < Fractor::Work
49
+ def initialize(items)
50
+ super(items: items)
51
+ end
52
+ end
53
+
54
+ class BatchWorker < Fractor::Worker
55
+ def process(work)
56
+ results = work.input[:items].map { |item| transform(item) }
57
+ Fractor::WorkResult.new(result: results, work: work)
58
+ end
59
+
60
+ private
61
+
62
+ def transform(item)
63
+ # Your transformation logic
64
+ item.upcase
65
+ end
66
+ end
67
+
68
+ # Process in batches
69
+ supervisor = Fractor::Supervisor.new(
70
+ worker_pools: [{ worker_class: BatchWorker, num_workers: 4 }]
71
+ )
72
+
73
+ # Split large dataset into batches
74
+ data = (1..10000).to_a
75
+ batch_size = 100
76
+
77
+ batches = data.each_slice(batch_size).map { |batch| BatchWork.new(batch) }
78
+ supervisor.add_work_items(batches)
79
+ supervisor.run
80
+
81
+ # Collect results
82
+ results = supervisor.results.results.flat_map(&:result)
83
+ puts "Processed #{results.size} items"
84
+ ----
85
+
86
+ ==== Best Practices
87
+
88
+ * Choose batch size based on memory constraints (100-1000 items typical)
89
+ * Monitor memory usage for large batches
90
+ * Use appropriate worker count for your CPU cores
91
+ * Consider checkpointing for very large datasets
92
+
93
+ ---
94
+
95
+ [[file-processing]]
96
+ === Pattern 2: File Processing
97
+
98
+ Process multiple files in parallel.
99
+
100
+ ==== When to Use
101
+
102
+ * Log file analysis
103
+ * Image processing
104
+ * Document conversion
105
+ * File validation
106
+
107
+ ==== Example
108
+
109
+ [source,ruby]
110
+ ----
111
+ class FileWork < Fractor::Work
112
+ def initialize(filepath)
113
+ super(filepath: filepath)
114
+ end
115
+ end
116
+
117
+ class FileProcessor < Fractor::Worker
118
+ def process(work)
119
+ filepath = work.input[:filepath]
120
+
121
+ # Read and process file
122
+ content = File.read(filepath)
123
+ result = process_content(content)
124
+
125
+ Fractor::WorkResult.new(
126
+ result: {
127
+ filepath: filepath,
128
+ size: content.size,
129
+ processed: result
130
+ },
131
+ work: work
132
+ )
133
+ rescue => e
134
+ Fractor::WorkResult.new(
135
+ error: e,
136
+ error_code: :file_processing_failed,
137
+ error_context: { filepath: filepath },
138
+ work: work
139
+ )
140
+ end
141
+
142
+ private
143
+
144
+ def process_content(content)
145
+ # Your processing logic
146
+ content.lines.count
147
+ end
148
+ end
149
+
150
+ # Process all files in directory
151
+ files = Dir.glob('data/**/*.txt')
152
+ supervisor = Fractor::Supervisor.new(
153
+ worker_pools: [{ worker_class: FileProcessor, num_workers: 4 }]
154
+ )
155
+
156
+ work_items = files.map { |file| FileWork.new(file) }
157
+ supervisor.add_work_items(work_items)
158
+ supervisor.run
159
+
160
+ # Report results
161
+ supervisor.results.results.each do |result|
162
+ puts "#{result.result[:filepath]}: #{result.result[:processed]} lines"
163
+ end
164
+ ----
165
+
166
+ ==== Best Practices
167
+
168
+ * Handle missing files gracefully
169
+ * Use appropriate worker count for I/O-bound operations (higher than CPU cores)
170
+ * Consider file size when setting worker count
171
+ * Implement progress tracking for large file sets
172
+
173
+ ---
174
+
175
+ [[api-rate-limiting]]
176
+ === Pattern 3: API Rate Limiting
177
+
178
+ Make API calls while respecting rate limits.
179
+
180
+ ==== When to Use
181
+
182
+ * External API integration
183
+ * Web scraping
184
+ * Data aggregation from APIs
185
+ * Bulk data sync
186
+
187
+ ==== Example
188
+
189
+ [source,ruby]
190
+ ----
191
+ class ApiWork < Fractor::Work
192
+ def initialize(endpoint, params = {})
193
+ super(endpoint: endpoint, params: params)
194
+ end
195
+ end
196
+
197
+ class RateLimitedApiWorker < Fractor::Worker
198
+ # Class-level rate limiting
199
+ @last_call = Time.now
200
+ @calls_in_window = 0
201
+ @mutex = Mutex.new
202
+
203
+ MAX_CALLS_PER_MINUTE = 60
204
+
205
+ def process(work)
206
+ endpoint = work.input[:endpoint]
207
+ params = work.input[:params]
208
+
209
+ # Rate limit before making call
210
+ rate_limit
211
+
212
+ # Make API call
213
+ response = make_api_call(endpoint, params)
214
+
215
+ Fractor::WorkResult.new(
216
+ result: { endpoint: endpoint, data: response },
217
+ work: work
218
+ )
219
+ rescue => e
220
+ Fractor::WorkResult.new(
221
+ error: e,
222
+ error_code: :api_call_failed,
223
+ error_context: { endpoint: endpoint },
224
+ work: work
225
+ )
226
+ end
227
+
228
+ private
229
+
230
+ def rate_limit
231
+ self.class.class_variable_get(:@@mutex).synchronize do
232
+ now = Time.now
233
+ last_call = self.class.class_variable_get(:@@last_call)
234
+ calls = self.class.class_variable_get(:@@calls_in_window)
235
+
236
+ # Reset window if minute has passed
237
+ if now - last_call >= 60
238
+ self.class.class_variable_set(:@@calls_in_window, 0)
239
+ self.class.class_variable_set(:@@last_call, now)
240
+ end
241
+
242
+ # Wait if limit reached
243
+ if calls >= MAX_CALLS_PER_MINUTE
244
+ sleep_time = 60 - (now - last_call)
245
+ sleep(sleep_time) if sleep_time > 0
246
+ self.class.class_variable_set(:@@calls_in_window, 0)
247
+ self.class.class_variable_set(:@@last_call, Time.now)
248
+ end
249
+
250
+ self.class.class_variable_set(:@@calls_in_window, calls + 1)
251
+ end
252
+ end
253
+
254
+ def make_api_call(endpoint, params)
255
+ # Simulate API call
256
+ sleep(0.1)
257
+ { status: 'success', data: params }
258
+ end
259
+ end
260
+
261
+ # Make multiple API calls with rate limiting
262
+ endpoints = Array.new(100) { |i| "/api/endpoint/#{i}" }
263
+ supervisor = Fractor::Supervisor.new(
264
+ worker_pools: [{ worker_class: RateLimitedApiWorker, num_workers: 5 }]
265
+ )
266
+
267
+ work_items = endpoints.map { |ep| ApiWork.new(ep) }
268
+ supervisor.add_work_items(work_items)
269
+ supervisor.run
270
+ ----
271
+
272
+ ==== Best Practices
273
+
274
+ * Implement rate limiting at the worker level
275
+ * Use class variables with mutex for thread safety
276
+ * Add retry logic for rate limit errors (429 status)
277
+ * Monitor API usage and adjust worker count accordingly
278
+ * Consider exponential backoff for failed requests
279
+
280
+ ---
281
+
282
+ [[producer-consumer]]
283
+ === Pattern 4: Producer-Consumer
284
+
285
+ One worker produces items for other workers to consume.
286
+
287
+ ==== When to Use
288
+
289
+ * Hierarchical data processing
290
+ * Tree traversal
291
+ * Recursive operations
292
+ * Dynamic work generation
293
+
294
+ ==== Example
295
+
296
+ [source,ruby]
297
+ ----
298
+ class ProducerWork < Fractor::Work
299
+ def initialize(data, supervisor)
300
+ super(data: data, supervisor: supervisor)
301
+ end
302
+ end
303
+
304
+ class ProducerWorker < Fractor::Worker
305
+ def process(work)
306
+ data = work.input[:data]
307
+ supervisor = work.input[:supervisor]
308
+
309
+ # Generate new work items
310
+ children = generate_children(data)
311
+
312
+ # Add work for consumers
313
+ children.each do |child|
314
+ supervisor.add_work(ConsumerWork.new(child))
315
+ end
316
+
317
+ Fractor::WorkResult.new(
318
+ result: { produced: children.size },
319
+ work: work
320
+ )
321
+ end
322
+
323
+ private
324
+
325
+ def generate_children(data)
326
+ # Generate child items
327
+ (1..5).map { |i| "#{data}-child-#{i}" }
328
+ end
329
+ end
330
+
331
+ class ConsumerWork < Fractor::Work
332
+ def initialize(data)
333
+ super(data: data)
334
+ end
335
+ end
336
+
337
+ class ConsumerWorker < Fractor::Worker
338
+ def process(work)
339
+ data = work.input[:data]
340
+ result = consume(data)
341
+
342
+ Fractor::WorkResult.new(result: result, work: work)
343
+ end
344
+
345
+ private
346
+
347
+ def consume(data)
348
+ # Process the data
349
+ data.upcase
350
+ end
351
+ end
352
+
353
+ supervisor = Fractor::Supervisor.new(
354
+ worker_pools: [
355
+ { worker_class: ProducerWorker, num_workers: 2 },
356
+ { worker_class: ConsumerWorker, num_workers: 4 }
357
+ ]
358
+ )
359
+
360
+ # Start with initial producer work
361
+ supervisor.add_work(ProducerWork.new('root', supervisor))
362
+ supervisor.run
363
+ ----
364
+
365
+ ==== Best Practices
366
+
367
+ * Balance producer and consumer worker counts
368
+ * Prevent infinite loops in work generation
369
+ * Monitor queue depth to avoid memory issues
370
+ * Consider max depth for recursive operations
371
+
372
+ ---
373
+
374
+ [[fan-out-fan-in]]
375
+ === Pattern 5: Fan-Out/Fan-In
376
+
377
+ Distribute work to multiple workers, then aggregate results.
378
+
379
+ ==== When to Use
380
+
381
+ * Parallel data transformation
382
+ * Multi-source data aggregation
383
+ * Distributed computation
384
+ * Map-reduce operations
385
+
386
+ ==== Example
387
+
388
+ [source,ruby]
389
+ ----
390
+ class FanOutWork < Fractor::Work
391
+ def initialize(id, data)
392
+ super(id: id, data: data)
393
+ end
394
+ end
395
+
396
+ class FanOutWorker < Fractor::Worker
397
+ def process(work)
398
+ # Process partition
399
+ result = work.input[:data].map { |item| item * 2 }
400
+
401
+ Fractor::WorkResult.new(
402
+ result: { id: work.input[:id], data: result },
403
+ work: work
404
+ )
405
+ end
406
+ end
407
+
408
+ # Fan-out: Split data into partitions
409
+ data = (1..1000).to_a
410
+ partitions = data.each_slice(100).to_a
411
+
412
+ supervisor = Fractor::Supervisor.new(
413
+ worker_pools: [{ worker_class: FanOutWorker, num_workers: 10 }]
414
+ )
415
+
416
+ work_items = partitions.each_with_index.map do |partition, i|
417
+ FanOutWork.new(i, partition)
418
+ end
419
+
420
+ supervisor.add_work_items(work_items)
421
+ supervisor.run
422
+
423
+ # Fan-in: Aggregate results
424
+ all_results = supervisor.results.results
425
+ .sort_by { |r| r.result[:id] }
426
+ .flat_map { |r| r.result[:data] }
427
+
428
+ puts "Processed #{all_results.size} items"
429
+ ----
430
+
431
+ ==== Best Practices
432
+
433
+ * Choose partition size based on work complexity
434
+ * Ensure deterministic ordering if needed
435
+ * Use appropriate aggregation strategy
436
+ * Monitor memory for large result sets
437
+
438
+ ---
439
+
440
+ [[retry-with-backoff]]
441
+ === Pattern 6: Retry with Backoff
442
+
443
+ Automatically retry failed operations with increasing delays.
444
+
445
+ ==== When to Use
446
+
447
+ * External service calls
448
+ * Network operations
449
+ * Transient failures
450
+ * Resource contention
451
+
452
+ ==== Example
453
+
454
+ [source,ruby]
455
+ ----
456
+ class RetryableWork < Fractor::Work
457
+ def initialize(url, max_attempts: 3)
458
+ super(url: url, max_attempts: max_attempts, attempt: 0)
459
+ end
460
+ end
461
+
462
+ class RetryableWorker < Fractor::Worker
463
+ def process(work)
464
+ url = work.input[:url]
465
+
466
+ # Attempt operation
467
+ result = fetch_data(url)
468
+
469
+ Fractor::WorkResult.new(result: result, work: work)
470
+ rescue => e
471
+ attempt = work.input[:attempt] + 1
472
+ max_attempts = work.input[:max_attempts]
473
+
474
+ if attempt < max_attempts
475
+ # Calculate backoff delay (exponential)
476
+ delay = 2 ** attempt
477
+ puts "Attempt #{attempt} failed, retrying in #{delay}s..."
478
+ sleep(delay)
479
+
480
+ # Retry by creating new work
481
+ work.input[:attempt] = attempt
482
+ process(work)
483
+ else
484
+ Fractor::WorkResult.new(
485
+ error: e,
486
+ error_code: :max_retries_exceeded,
487
+ work: work
488
+ )
489
+ end
490
+ end
491
+
492
+ private
493
+
494
+ def fetch_data(url)
495
+ # Simulate fetch with occasional failures
496
+ raise "Connection timeout" if rand < 0.3
497
+ { url: url, data: 'success' }
498
+ end
499
+ end
500
+ ----
501
+
502
+ ==== Best Practices
503
+
504
+ * Use exponential backoff to avoid hammering failing services
505
+ * Set maximum retry count to prevent infinite loops
506
+ * Add jitter to prevent thundering herd
507
+ * Log retry attempts for debugging
508
+ * Consider different strategies for different error types
509
+
510
+ ---
511
+
512
+ [[circuit-breaker-pattern]]
513
+ === Pattern 7: Circuit Breaker
514
+
515
+ Prevent cascading failures by failing fast when a service is down.
516
+
517
+ ==== When to Use
518
+
519
+ * External service dependencies
520
+ * Microservice communication
521
+ * Database connections
522
+ * Third-party APIs
523
+
524
+ ==== Example
525
+
526
+ [source,ruby]
527
+ ----
528
+ class CircuitBreaker
529
+ attr_reader :failure_threshold, :timeout, :failures, :last_failure_time, :state
530
+
531
+ def initialize(failure_threshold: 5, timeout: 60)
532
+ @failure_threshold = failure_threshold
533
+ @timeout = timeout
534
+ @failures = 0
535
+ @last_failure_time = nil
536
+ @state = :closed # :closed, :open, :half_open
537
+ @mutex = Mutex.new
538
+ end
539
+
540
+ def call
541
+ @mutex.synchronize do
542
+ case @state
543
+ when :open
544
+ if Time.now - @last_failure_time >= @timeout
545
+ @state = :half_open
546
+ else
547
+ raise CircuitOpenError, "Circuit breaker is open"
548
+ end
549
+ end
550
+ end
551
+
552
+ begin
553
+ result = yield
554
+ on_success
555
+ result
556
+ rescue => e
557
+ on_failure
558
+ raise e
559
+ end
560
+ end
561
+
562
+ private
563
+
564
+ def on_success
565
+ @mutex.synchronize do
566
+ @failures = 0
567
+ @state = :closed
568
+ end
569
+ end
570
+
571
+ def on_failure
572
+ @mutex.synchronize do
573
+ @failures += 1
574
+ @last_failure_time = Time.now
575
+
576
+ if @failures >= @failure_threshold
577
+ @state = :open
578
+ end
579
+ end
580
+ end
581
+ end
582
+
583
+ class CircuitOpenError < StandardError; end
584
+
585
+ class ProtectedWorker < Fractor::Worker
586
+ @@circuit_breaker = CircuitBreaker.new(failure_threshold: 3, timeout: 30)
587
+
588
+ def process(work)
589
+ result = @@circuit_breaker.call do
590
+ call_external_service(work.input[:data])
591
+ end
592
+
593
+ Fractor::WorkResult.new(result: result, work: work)
594
+ rescue CircuitOpenError => e
595
+ Fractor::WorkResult.new(
596
+ error: e,
597
+ error_code: :circuit_open,
598
+ work: work
599
+ )
600
+ rescue => e
601
+ Fractor::WorkResult.new(
602
+ error: e,
603
+ error_code: :service_failed,
604
+ work: work
605
+ )
606
+ end
607
+
608
+ private
609
+
610
+ def call_external_service(data)
611
+ # Simulate service call
612
+ raise "Service unavailable" if rand < 0.4
613
+ { data: data, status: 'success' }
614
+ end
615
+ end
616
+ ----
617
+
618
+ ==== Best Practices
619
+
620
+ * Set appropriate threshold based on error rate
621
+ * Use timeout to allow recovery
622
+ * Implement half-open state for testing recovery
623
+ * Monitor circuit breaker state
624
+ * Provide fallback behavior when circuit is open
625
+
626
+ ---
627
+
628
+ [[dead-letter-queue]]
629
+ === Pattern 8: Dead Letter Queue
630
+
631
+ Capture permanently failed work for manual inspection and retry.
632
+
633
+ ==== When to Use
634
+
635
+ * Critical operations that cannot be lost
636
+ * Complex error scenarios requiring human intervention
637
+ * Compliance and audit requirements
638
+ * Debugging production issues
639
+
640
+ ==== Example
641
+
642
+ [source,ruby]
643
+ ----
644
+ class DeadLetterQueue
645
+ def initialize
646
+ @entries = []
647
+ @mutex = Mutex.new
648
+ end
649
+
650
+ def add(work, error, context = {})
651
+ @mutex.synchronize do
652
+ @entries << {
653
+ work: work,
654
+ error: error,
655
+ context: context,
656
+ timestamp: Time.now
657
+ }
658
+ end
659
+ end
660
+
661
+ def all
662
+ @mutex.synchronize { @entries.dup }
663
+ end
664
+
665
+ def size
666
+ @mutex.synchronize { @entries.size }
667
+ end
668
+
669
+ def retry_all(&block)
670
+ entries = all
671
+ entries.each do |entry|
672
+ begin
673
+ yield entry[:work]
674
+ remove(entry)
675
+ rescue => e
676
+ puts "Retry failed: #{e.message}"
677
+ end
678
+ end
679
+ end
680
+
681
+ private
682
+
683
+ def remove(entry)
684
+ @mutex.synchronize { @entries.delete(entry) }
685
+ end
686
+ end
687
+
688
+ class DLQWorker < Fractor::Worker
689
+ @@dlq = DeadLetterQueue.new
690
+
691
+ def self.dead_letter_queue
692
+ @@dlq
693
+ end
694
+
695
+ def process(work)
696
+ result = perform_operation(work.input[:data])
697
+ Fractor::WorkResult.new(result: result, work: work)
698
+ rescue => e
699
+ # Add to DLQ if not retriable
700
+ unless retriable?(e)
701
+ self.class.dead_letter_queue.add(work, e)
702
+ end
703
+
704
+ Fractor::WorkResult.new(
705
+ error: e,
706
+ error_code: error_code_for(e),
707
+ work: work
708
+ )
709
+ end
710
+
711
+ private
712
+
713
+ def perform_operation(data)
714
+ # Simulate operation
715
+ raise ArgumentError, "Invalid data" if data.nil?
716
+ { processed: data }
717
+ end
718
+
719
+ def retriable?(error)
720
+ error.is_a?(IOError) || error.is_a?(Timeout::Error)
721
+ end
722
+
723
+ def error_code_for(error)
724
+ case error
725
+ when ArgumentError then :validation_error
726
+ when IOError then :io_error
727
+ else :unknown_error
728
+ end
729
+ end
730
+ end
731
+
732
+ # After processing, check DLQ
733
+ dlq = DLQWorker.dead_letter_queue
734
+ puts "DLQ has #{dlq.size} failed items"
735
+
736
+ # Manual retry
737
+ dlq.retry_all do |work|
738
+ # Fix the issue and retry
739
+ puts "Retrying work: #{work.input}"
740
+ end
741
+ ----
742
+
743
+ ==== Best Practices
744
+
745
+ * Persist DLQ to disk or database for durability
746
+ * Set max size to prevent memory issues
747
+ * Implement DLQ monitoring and alerting
748
+ * Provide tools for DLQ inspection and retry
749
+ * Archive old DLQ entries
750
+
751
+ ---
752
+
753
+ [[priority-queues]]
754
+ === Pattern 9: Priority Queues
755
+
756
+ Process high-priority work before low-priority work.
757
+
758
+ ==== When to Use
759
+
760
+ * SLA-based processing
761
+ * VIP customer handling
762
+ * Time-sensitive operations
763
+ * Mixed workload types
764
+
765
+ ==== Example
766
+
767
+ [source,ruby]
768
+ ----
769
+ class PriorityWork < Fractor::Work
770
+ attr_reader :priority
771
+
772
+ def initialize(data, priority: 0)
773
+ @priority = priority
774
+ super(data: data, priority: priority)
775
+ end
776
+
777
+ def <=>(other)
778
+ # Higher priority first
779
+ other.priority <=> self.priority
780
+ end
781
+ end
782
+
783
+ class PriorityQueue
784
+ def initialize
785
+ @queue = []
786
+ @mutex = Mutex.new
787
+ end
788
+
789
+ def <<(work)
790
+ @mutex.synchronize do
791
+ @queue << work
792
+ @queue.sort!
793
+ end
794
+ end
795
+
796
+ def pop
797
+ @mutex.synchronize { @queue.shift }
798
+ end
799
+
800
+ def size
801
+ @mutex.synchronize { @queue.size }
802
+ end
803
+
804
+ def empty?
805
+ @mutex.synchronize { @queue.empty? }
806
+ end
807
+ end
808
+
809
+ # Usage
810
+ priority_queue = PriorityQueue.new
811
+
812
+ # Add work with different priorities
813
+ priority_queue << PriorityWork.new('low priority', priority: 1)
814
+ priority_queue << PriorityWork.new('high priority', priority: 10)
815
+ priority_queue << PriorityWork.new('medium priority', priority: 5)
816
+
817
+ # Work is processed in priority order: 10, 5, 1
818
+ ----
819
+
820
+ ==== Best Practices
821
+
822
+ * Define clear priority levels (e.g., 1-10)
823
+ * Prevent starvation of low-priority work
824
+ * Monitor queue distribution by priority
825
+ * Consider aging to increase priority over time
826
+ * Use separate queues for vastly different priorities
827
+
828
+ ---
829
+
830
+ [[streaming-data]]
831
+ === Pattern 10: Streaming Data Processing
832
+
833
+ Process continuous streams of data in real-time.
834
+
835
+ ==== When to Use
836
+
837
+ * Real-time analytics
838
+ * Event processing
839
+ * Log aggregation
840
+ * Sensor data processing
841
+
842
+ ==== Example
843
+
844
+ [source,ruby]
845
+ ----
846
+ class StreamProcessor
847
+ def initialize(worker_class, num_workers: 4)
848
+ @work_queue = Fractor::WorkQueue.new
849
+ @server = Fractor::ContinuousServer.new(
850
+ worker_pools: [{ worker_class: worker_class, num_workers: num_workers }],
851
+ work_queue: @work_queue
852
+ )
853
+
854
+ setup_handlers
855
+ end
856
+
857
+ def start
858
+ Thread.new { @server.run }
859
+ end
860
+
861
+ def process(event)
862
+ @work_queue << EventWork.new(event)
863
+ end
864
+
865
+ def stop
866
+ @server.stop
867
+ end
868
+
869
+ private
870
+
871
+ def setup_handlers
872
+ @server.on_result do |result|
873
+ # Handle processed event
874
+ publish_result(result)
875
+ end
876
+
877
+ @server.on_error do |error|
878
+ # Handle errors
879
+ log_error(error)
880
+ end
881
+ end
882
+
883
+ def publish_result(result)
884
+ # Publish to downstream systems
885
+ puts "Processed event: #{result.result}"
886
+ end
887
+
888
+ def log_error(error)
889
+ puts "Error: #{error.error}"
890
+ end
891
+ end
892
+
893
+ class EventWork < Fractor::Work
894
+ def initialize(event)
895
+ super(event: event, timestamp: Time.now)
896
+ end
897
+ end
898
+
899
+ class EventWorker < Fractor::Worker
900
+ def process(work)
901
+ event = work.input[:event]
902
+
903
+ # Process event
904
+ result = analyze_event(event)
905
+
906
+ Fractor::WorkResult.new(result: result, work: work)
907
+ end
908
+
909
+ private
910
+
911
+ def analyze_event(event)
912
+ # Event processing logic
913
+ {
914
+ type: event[:type],
915
+ count: event[:data]&.size || 0,
916
+ processed_at: Time.now
917
+ }
918
+ end
919
+ end
920
+
921
+ # Usage
922
+ processor = StreamProcessor.new(EventWorker, num_workers: 8)
923
+ processor.start
924
+
925
+ # Stream events
926
+ loop do
927
+ event = { type: 'click', data: [1, 2, 3] }
928
+ processor.process(event)
929
+ sleep(0.1)
930
+ end
931
+ ----
932
+
933
+ ==== Best Practices
934
+
935
+ * Use continuous mode for indefinite operation
936
+ * Implement backpressure to handle load spikes
937
+ * Monitor queue depth and processing latency
938
+ * Add checkpointing for exactly-once processing
939
+ * Consider time-based windows for aggregation
940
+
941
+ ---
942
+
943
+ [[parallel-aggregation]]
944
+ === Pattern 11: Parallel Aggregation
945
+
946
+ Aggregate results from parallel operations efficiently.
947
+
948
+ ==== When to Use
949
+
950
+ * Statistical analysis
951
+ * Report generation
952
+ * Data summarization
953
+ * Metrics collection
954
+
955
+ ==== Example
956
+
957
+ [source,ruby]
958
+ ----
959
+ class AggregationWork < Fractor::Work
960
+ def initialize(partition_id, data)
961
+ super(partition_id: partition_id, data: data)
962
+ end
963
+ end
964
+
965
+ class AggregationWorker < Fractor::Worker
966
+ def process(work)
967
+ data = work.input[:data]
968
+
969
+ # Compute local aggregates
970
+ local_sum = data.sum
971
+ local_count = data.size
972
+ local_min = data.min
973
+ local_max = data.max
974
+
975
+ Fractor::WorkResult.new(
976
+ result: {
977
+ partition: work.input[:partition_id],
978
+ sum: local_sum,
979
+ count: local_count,
980
+ min: local_min,
981
+ max: local_max
982
+ },
983
+ work: work
984
+ )
985
+ end
986
+ end
987
+
988
+ # Parallel aggregation
989
+ data = (1..10000).to_a
990
+ partitions = data.each_slice(1000).to_a
991
+
992
+ supervisor = Fractor::Supervisor.new(
993
+ worker_pools: [{ worker_class: AggregationWorker, num_workers: 8 }]
994
+ )
995
+
996
+ work_items = partitions.each_with_index.map do |partition, i|
997
+ AggregationWork.new(i, partition)
998
+ end
999
+
1000
+ supervisor.add_work_items(work_items)
1001
+ supervisor.run
1002
+
1003
+ # Final aggregation
1004
+ results = supervisor.results.results.map(&:result)
1005
+
1006
+ final_sum = results.sum { |r| r[:sum] }
1007
+ final_count = results.sum { |r| r[:count] }
1008
+ final_min = results.map { |r| r[:min] }.min
1009
+ final_max = results.map { |r| r[:max] }.max
1010
+ final_avg = final_sum.to_f / final_count
1011
+
1012
+ puts "Sum: #{final_sum}"
1013
+ puts "Count: #{final_count}"
1014
+ puts "Min: #{final_min}"
1015
+ puts "Max: #{final_max}"
1016
+ puts "Average: #{final_avg}"
1017
+ ----
1018
+
1019
+ ==== Best Practices
1020
+
1021
+ * Use commutative and associative operations when possible
1022
+ * Partition data evenly for balanced load
1023
+ * Keep intermediate results small
1024
+ * Consider hierarchical aggregation for very large datasets
1025
+
1026
+ ---
1027
+
1028
+ [[worker-pools]]
1029
+ === Pattern 12: Multi-Type Worker Pools
1030
+
1031
+ Run different worker types with optimal resource allocation.
1032
+
1033
+ ==== When to Use
1034
+
1035
+ * Mixed workload types
1036
+ * Different resource requirements
1037
+ * Priority-based processing
1038
+ * Specialized processing
1039
+
1040
+ ==== Example
1041
+
1042
+ [source,ruby]
1043
+ ----
1044
+ # CPU-intensive worker
1045
+ class CPUWorker < Fractor::Worker
1046
+ def process(work)
1047
+ # CPU-bound computation
1048
+ result = expensive_computation(work.input[:data])
1049
+ Fractor::WorkResult.new(result: result, work: work)
1050
+ end
1051
+
1052
+ private
1053
+
1054
+ def expensive_computation(data)
1055
+ # Simulate CPU work
1056
+ sleep(0.5)
1057
+ data.map { |x| x ** 2 }.sum
1058
+ end
1059
+ end
1060
+
1061
+ # I/O-intensive worker
1062
+ class IOWorker < Fractor::Worker
1063
+ def process(work)
1064
+ # I/O-bound operation
1065
+ result = fetch_from_network(work.input[:url])
1066
+ Fractor::WorkResult.new(result: result, work: work)
1067
+ end
1068
+
1069
+ private
1070
+
1071
+ def fetch_from_network(url)
1072
+ # Simulate I/O work
1073
+ sleep(1)
1074
+ { url: url, data: 'fetched' }
1075
+ end
1076
+ end
1077
+
1078
+ # Memory-intensive worker
1079
+ class MemoryWorker < Fractor::Worker
1080
+ def process(work)
1081
+ # Memory-bound operation
1082
+ result = process_large_dataset(work.input[:dataset])
1083
+ Fractor::WorkResult.new(result: result, work: work)
1084
+ end
1085
+
1086
+ private
1087
+
1088
+ def process_large_dataset(dataset)
1089
+ # Simulate memory work
1090
+ large_array = Array.new(1000000) { rand }
1091
+ large_array.sum
1092
+ end
1093
+ end
1094
+
1095
+ # Configure pools based on resource characteristics
1096
+ supervisor = Fractor::Supervisor.new(
1097
+ worker_pools: [
1098
+ { worker_class: CPUWorker, num_workers: 4 }, # CPU cores
1099
+ { worker_class: IOWorker, num_workers: 20 }, # High for I/O
1100
+ { worker_class: MemoryWorker, num_workers: 2 } # Limited for memory
1101
+ ]
1102
+ )
1103
+ ----
1104
+
1105
+ ==== Best Practices
1106
+
1107
+ * Tune worker counts based on resource type:
1108
+ ** CPU-bound: Number of cores
1109
+ ** I/O-bound: 2-4x number of cores
1110
+ ** Memory-bound: Based on available RAM
1111
+ * Monitor resource utilization
1112
+ * Use separate supervisors for vastly different workloads
1113
+ * Consider dynamic worker scaling for variable loads
1114
+
1115
+ ---
1116
+
1117
+ === Summary
1118
+
1119
+ These patterns provide a solid foundation for building robust Fractor applications. Combine patterns as needed for your specific use case, and always:
1120
+
1121
+ * Monitor performance and resource usage
1122
+ * Implement proper error handling
1123
+ * Add logging and observability
1124
+ * Test with production-like data
1125
+ * Document your pattern choices
1126
+
1127
+ === See Also
1128
+
1129
+ * link:../tutorials/data-processing-pipeline[Data Processing Pipeline Tutorial]
1130
+ * link:../tutorials/long-running-services[Long-Running Services Tutorial]
1131
+ * link:../tutorials/complex-workflows[Complex Workflows Tutorial]
1132
+ * link:../reference/api[API Reference]
1133
+ * link:../reference/examples[Examples]