fractor 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop_todo.yml +227 -102
  3. data/README.adoc +113 -1940
  4. data/docs/.lycheeignore +16 -0
  5. data/docs/Gemfile +24 -0
  6. data/docs/README.md +157 -0
  7. data/docs/_config.yml +151 -0
  8. data/docs/_features/error-handling.adoc +1192 -0
  9. data/docs/_features/index.adoc +80 -0
  10. data/docs/_features/monitoring.adoc +589 -0
  11. data/docs/_features/signal-handling.adoc +202 -0
  12. data/docs/_features/workflows.adoc +1235 -0
  13. data/docs/_guides/continuous-mode.adoc +736 -0
  14. data/docs/_guides/cookbook.adoc +1133 -0
  15. data/docs/_guides/index.adoc +55 -0
  16. data/docs/_guides/pipeline-mode.adoc +730 -0
  17. data/docs/_guides/troubleshooting.adoc +358 -0
  18. data/docs/_pages/architecture.adoc +1390 -0
  19. data/docs/_pages/core-concepts.adoc +1392 -0
  20. data/docs/_pages/design-principles.adoc +862 -0
  21. data/docs/_pages/getting-started.adoc +290 -0
  22. data/docs/_pages/installation.adoc +143 -0
  23. data/docs/_reference/api.adoc +1080 -0
  24. data/docs/_reference/error-reporting.adoc +670 -0
  25. data/docs/_reference/examples.adoc +181 -0
  26. data/docs/_reference/index.adoc +96 -0
  27. data/docs/_reference/troubleshooting.adoc +862 -0
  28. data/docs/_tutorials/complex-workflows.adoc +1022 -0
  29. data/docs/_tutorials/data-processing-pipeline.adoc +740 -0
  30. data/docs/_tutorials/first-application.adoc +384 -0
  31. data/docs/_tutorials/index.adoc +48 -0
  32. data/docs/_tutorials/long-running-services.adoc +931 -0
  33. data/docs/assets/images/favicon-16.png +0 -0
  34. data/docs/assets/images/favicon-32.png +0 -0
  35. data/docs/assets/images/favicon-48.png +0 -0
  36. data/docs/assets/images/favicon.ico +0 -0
  37. data/docs/assets/images/favicon.png +0 -0
  38. data/docs/assets/images/favicon.svg +45 -0
  39. data/docs/assets/images/fractor-icon.svg +49 -0
  40. data/docs/assets/images/fractor-logo.svg +61 -0
  41. data/docs/index.adoc +131 -0
  42. data/docs/lychee.toml +39 -0
  43. data/examples/api_aggregator/README.adoc +627 -0
  44. data/examples/api_aggregator/api_aggregator.rb +376 -0
  45. data/examples/auto_detection/README.adoc +407 -29
  46. data/examples/continuous_chat_common/message_protocol.rb +1 -1
  47. data/examples/error_reporting.rb +207 -0
  48. data/examples/file_processor/README.adoc +170 -0
  49. data/examples/file_processor/file_processor.rb +615 -0
  50. data/examples/file_processor/sample_files/invalid.csv +1 -0
  51. data/examples/file_processor/sample_files/orders.xml +24 -0
  52. data/examples/file_processor/sample_files/products.json +23 -0
  53. data/examples/file_processor/sample_files/users.csv +6 -0
  54. data/examples/hierarchical_hasher/README.adoc +629 -41
  55. data/examples/image_processor/README.adoc +610 -0
  56. data/examples/image_processor/image_processor.rb +349 -0
  57. data/examples/image_processor/processed_images/sample_10_processed.jpg.json +12 -0
  58. data/examples/image_processor/processed_images/sample_1_processed.jpg.json +12 -0
  59. data/examples/image_processor/processed_images/sample_2_processed.jpg.json +12 -0
  60. data/examples/image_processor/processed_images/sample_3_processed.jpg.json +12 -0
  61. data/examples/image_processor/processed_images/sample_4_processed.jpg.json +12 -0
  62. data/examples/image_processor/processed_images/sample_5_processed.jpg.json +12 -0
  63. data/examples/image_processor/processed_images/sample_6_processed.jpg.json +12 -0
  64. data/examples/image_processor/processed_images/sample_7_processed.jpg.json +12 -0
  65. data/examples/image_processor/processed_images/sample_8_processed.jpg.json +12 -0
  66. data/examples/image_processor/processed_images/sample_9_processed.jpg.json +12 -0
  67. data/examples/image_processor/test_images/sample_1.png +1 -0
  68. data/examples/image_processor/test_images/sample_10.png +1 -0
  69. data/examples/image_processor/test_images/sample_2.png +1 -0
  70. data/examples/image_processor/test_images/sample_3.png +1 -0
  71. data/examples/image_processor/test_images/sample_4.png +1 -0
  72. data/examples/image_processor/test_images/sample_5.png +1 -0
  73. data/examples/image_processor/test_images/sample_6.png +1 -0
  74. data/examples/image_processor/test_images/sample_7.png +1 -0
  75. data/examples/image_processor/test_images/sample_8.png +1 -0
  76. data/examples/image_processor/test_images/sample_9.png +1 -0
  77. data/examples/log_analyzer/README.adoc +662 -0
  78. data/examples/log_analyzer/log_analyzer.rb +579 -0
  79. data/examples/log_analyzer/sample_logs/apache.log +20 -0
  80. data/examples/log_analyzer/sample_logs/json.log +15 -0
  81. data/examples/log_analyzer/sample_logs/nginx.log +15 -0
  82. data/examples/log_analyzer/sample_logs/rails.log +29 -0
  83. data/examples/multi_work_type/README.adoc +576 -26
  84. data/examples/performance_monitoring.rb +120 -0
  85. data/examples/pipeline_processing/README.adoc +740 -26
  86. data/examples/pipeline_processing/pipeline_processing.rb +2 -2
  87. data/examples/priority_work_example.rb +155 -0
  88. data/examples/producer_subscriber/README.adoc +889 -46
  89. data/examples/scatter_gather/README.adoc +829 -27
  90. data/examples/simple/README.adoc +347 -0
  91. data/examples/specialized_workers/README.adoc +622 -26
  92. data/examples/specialized_workers/specialized_workers.rb +44 -8
  93. data/examples/stream_processor/README.adoc +206 -0
  94. data/examples/stream_processor/stream_processor.rb +284 -0
  95. data/examples/web_scraper/README.adoc +625 -0
  96. data/examples/web_scraper/web_scraper.rb +285 -0
  97. data/examples/workflow/README.adoc +406 -0
  98. data/examples/workflow/circuit_breaker/README.adoc +360 -0
  99. data/examples/workflow/circuit_breaker/circuit_breaker_workflow.rb +225 -0
  100. data/examples/workflow/conditional/README.adoc +483 -0
  101. data/examples/workflow/conditional/conditional_workflow.rb +215 -0
  102. data/examples/workflow/dead_letter_queue/README.adoc +374 -0
  103. data/examples/workflow/dead_letter_queue/dead_letter_queue_workflow.rb +217 -0
  104. data/examples/workflow/fan_out/README.adoc +381 -0
  105. data/examples/workflow/fan_out/fan_out_workflow.rb +202 -0
  106. data/examples/workflow/retry/README.adoc +248 -0
  107. data/examples/workflow/retry/retry_workflow.rb +195 -0
  108. data/examples/workflow/simple_linear/README.adoc +267 -0
  109. data/examples/workflow/simple_linear/simple_linear_workflow.rb +175 -0
  110. data/examples/workflow/simplified/README.adoc +329 -0
  111. data/examples/workflow/simplified/simplified_workflow.rb +222 -0
  112. data/exe/fractor +10 -0
  113. data/lib/fractor/cli.rb +288 -0
  114. data/lib/fractor/configuration.rb +307 -0
  115. data/lib/fractor/continuous_server.rb +60 -65
  116. data/lib/fractor/error_formatter.rb +72 -0
  117. data/lib/fractor/error_report_generator.rb +152 -0
  118. data/lib/fractor/error_reporter.rb +244 -0
  119. data/lib/fractor/error_statistics.rb +147 -0
  120. data/lib/fractor/execution_tracer.rb +162 -0
  121. data/lib/fractor/logger.rb +230 -0
  122. data/lib/fractor/main_loop_handler.rb +406 -0
  123. data/lib/fractor/main_loop_handler3.rb +135 -0
  124. data/lib/fractor/main_loop_handler4.rb +299 -0
  125. data/lib/fractor/performance_metrics_collector.rb +181 -0
  126. data/lib/fractor/performance_monitor.rb +215 -0
  127. data/lib/fractor/performance_report_generator.rb +202 -0
  128. data/lib/fractor/priority_work.rb +93 -0
  129. data/lib/fractor/priority_work_queue.rb +189 -0
  130. data/lib/fractor/result_aggregator.rb +32 -0
  131. data/lib/fractor/shutdown_handler.rb +168 -0
  132. data/lib/fractor/signal_handler.rb +80 -0
  133. data/lib/fractor/supervisor.rb +382 -269
  134. data/lib/fractor/supervisor_logger.rb +88 -0
  135. data/lib/fractor/version.rb +1 -1
  136. data/lib/fractor/work.rb +12 -0
  137. data/lib/fractor/work_distribution_manager.rb +151 -0
  138. data/lib/fractor/work_queue.rb +20 -0
  139. data/lib/fractor/work_result.rb +181 -9
  140. data/lib/fractor/worker.rb +73 -0
  141. data/lib/fractor/workflow/builder.rb +210 -0
  142. data/lib/fractor/workflow/chain_builder.rb +169 -0
  143. data/lib/fractor/workflow/circuit_breaker.rb +183 -0
  144. data/lib/fractor/workflow/circuit_breaker_orchestrator.rb +208 -0
  145. data/lib/fractor/workflow/circuit_breaker_registry.rb +112 -0
  146. data/lib/fractor/workflow/dead_letter_queue.rb +334 -0
  147. data/lib/fractor/workflow/execution_hooks.rb +39 -0
  148. data/lib/fractor/workflow/execution_strategy.rb +225 -0
  149. data/lib/fractor/workflow/execution_trace.rb +134 -0
  150. data/lib/fractor/workflow/helpers.rb +191 -0
  151. data/lib/fractor/workflow/job.rb +290 -0
  152. data/lib/fractor/workflow/job_dependency_validator.rb +120 -0
  153. data/lib/fractor/workflow/logger.rb +110 -0
  154. data/lib/fractor/workflow/pre_execution_context.rb +193 -0
  155. data/lib/fractor/workflow/retry_config.rb +156 -0
  156. data/lib/fractor/workflow/retry_orchestrator.rb +184 -0
  157. data/lib/fractor/workflow/retry_strategy.rb +93 -0
  158. data/lib/fractor/workflow/structured_logger.rb +30 -0
  159. data/lib/fractor/workflow/type_compatibility_validator.rb +222 -0
  160. data/lib/fractor/workflow/visualizer.rb +211 -0
  161. data/lib/fractor/workflow/workflow_context.rb +132 -0
  162. data/lib/fractor/workflow/workflow_executor.rb +669 -0
  163. data/lib/fractor/workflow/workflow_result.rb +55 -0
  164. data/lib/fractor/workflow/workflow_validator.rb +295 -0
  165. data/lib/fractor/workflow.rb +333 -0
  166. data/lib/fractor/wrapped_ractor.rb +66 -101
  167. data/lib/fractor/wrapped_ractor3.rb +161 -0
  168. data/lib/fractor/wrapped_ractor4.rb +242 -0
  169. data/lib/fractor.rb +92 -4
  170. metadata +179 -6
  171. data/tests/sample.rb.bak +0 -309
  172. data/tests/sample_working.rb.bak +0 -209
@@ -57,8 +57,8 @@ module PipelineProcessing
57
57
 
58
58
  # Update metadata with processing information
59
59
  updated_metadata = work.metadata.merge(
60
- "#{work.stage}_completed" => true,
61
- "#{work.stage}_time" => Time.now.to_s,
60
+ "#{work.stage}_completed".to_sym => true,
61
+ "#{work.stage}_time".to_sym => Time.now.to_s,
62
62
  )
63
63
 
64
64
  # Return the result with next stage information
@@ -0,0 +1,155 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "../lib/fractor"
5
+
6
+ # Example demonstrating priority-based work processing
7
+ # This shows how to use PriorityWork and PriorityWorkQueue for
8
+ # processing tasks based on their priority levels
9
+
10
+ # Define a worker that processes priority work
11
+ class PriorityWorker < Fractor::Worker
12
+ def process(work)
13
+ # Simulate work processing
14
+ sleep 0.1
15
+
16
+ result = "Processed #{work.input[:task]} " \
17
+ "(priority: #{work.priority}, age: #{work.age.round(2)}s)"
18
+
19
+ Fractor::WorkResult.new(result: result, work: work)
20
+ end
21
+ end
22
+
23
+ puts "=" * 60
24
+ puts "Priority Work Example"
25
+ puts "=" * 60
26
+ puts
27
+
28
+ # Example 1: Basic Priority Queue
29
+ puts "Example 1: Basic Priority Ordering"
30
+ puts "-" * 60
31
+
32
+ queue = Fractor::PriorityWorkQueue.new
33
+
34
+ # Add work items with different priorities
35
+ queue.push(Fractor::PriorityWork.new({ task: "Background report" }, priority: :background))
36
+ queue.push(Fractor::PriorityWork.new({ task: "Critical bug fix" }, priority: :critical))
37
+ queue.push(Fractor::PriorityWork.new({ task: "Normal feature" }, priority: :normal))
38
+ queue.push(Fractor::PriorityWork.new({ task: "High priority task" }, priority: :high))
39
+ queue.push(Fractor::PriorityWork.new({ task: "Low priority cleanup" }, priority: :low))
40
+
41
+ puts "Queue statistics:"
42
+ stats = queue.stats
43
+ puts " Total items: #{stats[:total]}"
44
+ puts " By priority: #{stats[:by_priority]}"
45
+ puts
46
+
47
+ puts "Processing in priority order:"
48
+ 5.times do
49
+ work = queue.pop_non_blocking
50
+ puts " #{work.input[:task]} (#{work.priority})"
51
+ end
52
+ puts
53
+
54
+ # Example 2: Priority Aging
55
+ puts "Example 2: Priority Aging (Preventing Starvation)"
56
+ puts "-" * 60
57
+
58
+ aged_queue = Fractor::PriorityWorkQueue.new(
59
+ aging_enabled: true,
60
+ aging_threshold: 2 # 2 seconds
61
+ )
62
+
63
+ # Add a low-priority item first
64
+ aged_queue.push(Fractor::PriorityWork.new(
65
+ { task: "Old low-priority task" },
66
+ priority: :low
67
+ ))
68
+
69
+ puts "Added low-priority task at #{Time.now.strftime('%H:%M:%S')}"
70
+ puts "Waiting 3 seconds to let it age..."
71
+ sleep 3
72
+
73
+ # Add high-priority items after the low-priority one has aged
74
+ aged_queue.push(Fractor::PriorityWork.new(
75
+ { task: "New high-priority task" },
76
+ priority: :high
77
+ ))
78
+
79
+ puts "Added high-priority task at #{Time.now.strftime('%H:%M:%S')}"
80
+ puts
81
+
82
+ puts "Processing order with aging enabled:"
83
+ 2.times do
84
+ work = aged_queue.pop_non_blocking
85
+ puts " #{work.input[:task]} " \
86
+ "(priority: #{work.priority}, age: #{work.age.round(1)}s)"
87
+ end
88
+ puts "Note: The aged low-priority task was processed first!"
89
+ puts
90
+
91
+ # Example 3: Using with Supervisor
92
+ puts "Example 3: Integration with Supervisor"
93
+ puts "-" * 60
94
+
95
+ priority_queue = Fractor::PriorityWorkQueue.new
96
+
97
+ # Add mixed priority work
98
+ [
99
+ { task: "Process payment", priority: :critical },
100
+ { task: "Send email", priority: :normal },
101
+ { task: "Generate report", priority: :low },
102
+ { task: "Update inventory", priority: :high },
103
+ { task: "Cleanup cache", priority: :background }
104
+ ].each do |item|
105
+ priority_queue.push(Fractor::PriorityWork.new(item, priority: item[:priority]))
106
+ end
107
+
108
+ # Create supervisor with priority queue
109
+ supervisor = Fractor::Supervisor.new(
110
+ work_queue: priority_queue,
111
+ worker_pools: [
112
+ { worker_class: PriorityWorker, num_workers: 2 }
113
+ ]
114
+ )
115
+
116
+ puts "Processing #{priority_queue.size} tasks with 2 workers..."
117
+ supervisor.start
118
+
119
+ # Wait for all work to complete
120
+ sleep 1 until priority_queue.empty?
121
+
122
+ supervisor.shutdown
123
+ results = supervisor.results
124
+
125
+ puts "\nResults (in completion order):"
126
+ results.each_with_index do |result, i|
127
+ puts " #{i + 1}. #{result.result}"
128
+ end
129
+ puts
130
+
131
+ # Example 4: Queue Statistics
132
+ puts "Example 4: Monitoring Queue Statistics"
133
+ puts "-" * 60
134
+
135
+ stats_queue = Fractor::PriorityWorkQueue.new
136
+
137
+ # Add various priority items
138
+ 10.times do |i|
139
+ priority = [:critical, :high, :normal, :low, :background].sample
140
+ stats_queue.push(Fractor::PriorityWork.new({ id: i }, priority: priority))
141
+ end
142
+
143
+ stats = stats_queue.stats
144
+ puts "Queue statistics:"
145
+ puts " Total items: #{stats[:total]}"
146
+ puts " Closed: #{stats[:closed]}"
147
+ puts " Items by priority:"
148
+ stats[:by_priority].each do |priority, count|
149
+ puts " #{priority}: #{count}"
150
+ end
151
+ puts
152
+
153
+ puts "=" * 60
154
+ puts "Priority Work Examples Complete"
155
+ puts "=" * 60