fractor 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +227 -102
- data/README.adoc +113 -1940
- data/docs/.lycheeignore +16 -0
- data/docs/Gemfile +24 -0
- data/docs/README.md +157 -0
- data/docs/_config.yml +151 -0
- data/docs/_features/error-handling.adoc +1192 -0
- data/docs/_features/index.adoc +80 -0
- data/docs/_features/monitoring.adoc +589 -0
- data/docs/_features/signal-handling.adoc +202 -0
- data/docs/_features/workflows.adoc +1235 -0
- data/docs/_guides/continuous-mode.adoc +736 -0
- data/docs/_guides/cookbook.adoc +1133 -0
- data/docs/_guides/index.adoc +55 -0
- data/docs/_guides/pipeline-mode.adoc +730 -0
- data/docs/_guides/troubleshooting.adoc +358 -0
- data/docs/_pages/architecture.adoc +1390 -0
- data/docs/_pages/core-concepts.adoc +1392 -0
- data/docs/_pages/design-principles.adoc +862 -0
- data/docs/_pages/getting-started.adoc +290 -0
- data/docs/_pages/installation.adoc +143 -0
- data/docs/_reference/api.adoc +1080 -0
- data/docs/_reference/error-reporting.adoc +670 -0
- data/docs/_reference/examples.adoc +181 -0
- data/docs/_reference/index.adoc +96 -0
- data/docs/_reference/troubleshooting.adoc +862 -0
- data/docs/_tutorials/complex-workflows.adoc +1022 -0
- data/docs/_tutorials/data-processing-pipeline.adoc +740 -0
- data/docs/_tutorials/first-application.adoc +384 -0
- data/docs/_tutorials/index.adoc +48 -0
- data/docs/_tutorials/long-running-services.adoc +931 -0
- data/docs/assets/images/favicon-16.png +0 -0
- data/docs/assets/images/favicon-32.png +0 -0
- data/docs/assets/images/favicon-48.png +0 -0
- data/docs/assets/images/favicon.ico +0 -0
- data/docs/assets/images/favicon.png +0 -0
- data/docs/assets/images/favicon.svg +45 -0
- data/docs/assets/images/fractor-icon.svg +49 -0
- data/docs/assets/images/fractor-logo.svg +61 -0
- data/docs/index.adoc +131 -0
- data/docs/lychee.toml +39 -0
- data/examples/api_aggregator/README.adoc +627 -0
- data/examples/api_aggregator/api_aggregator.rb +376 -0
- data/examples/auto_detection/README.adoc +407 -29
- data/examples/continuous_chat_common/message_protocol.rb +1 -1
- data/examples/error_reporting.rb +207 -0
- data/examples/file_processor/README.adoc +170 -0
- data/examples/file_processor/file_processor.rb +615 -0
- data/examples/file_processor/sample_files/invalid.csv +1 -0
- data/examples/file_processor/sample_files/orders.xml +24 -0
- data/examples/file_processor/sample_files/products.json +23 -0
- data/examples/file_processor/sample_files/users.csv +6 -0
- data/examples/hierarchical_hasher/README.adoc +629 -41
- data/examples/image_processor/README.adoc +610 -0
- data/examples/image_processor/image_processor.rb +349 -0
- data/examples/image_processor/processed_images/sample_10_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_1_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_2_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_3_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_4_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_5_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_6_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_7_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_8_processed.jpg.json +12 -0
- data/examples/image_processor/processed_images/sample_9_processed.jpg.json +12 -0
- data/examples/image_processor/test_images/sample_1.png +1 -0
- data/examples/image_processor/test_images/sample_10.png +1 -0
- data/examples/image_processor/test_images/sample_2.png +1 -0
- data/examples/image_processor/test_images/sample_3.png +1 -0
- data/examples/image_processor/test_images/sample_4.png +1 -0
- data/examples/image_processor/test_images/sample_5.png +1 -0
- data/examples/image_processor/test_images/sample_6.png +1 -0
- data/examples/image_processor/test_images/sample_7.png +1 -0
- data/examples/image_processor/test_images/sample_8.png +1 -0
- data/examples/image_processor/test_images/sample_9.png +1 -0
- data/examples/log_analyzer/README.adoc +662 -0
- data/examples/log_analyzer/log_analyzer.rb +579 -0
- data/examples/log_analyzer/sample_logs/apache.log +20 -0
- data/examples/log_analyzer/sample_logs/json.log +15 -0
- data/examples/log_analyzer/sample_logs/nginx.log +15 -0
- data/examples/log_analyzer/sample_logs/rails.log +29 -0
- data/examples/multi_work_type/README.adoc +576 -26
- data/examples/performance_monitoring.rb +120 -0
- data/examples/pipeline_processing/README.adoc +740 -26
- data/examples/pipeline_processing/pipeline_processing.rb +2 -2
- data/examples/priority_work_example.rb +155 -0
- data/examples/producer_subscriber/README.adoc +889 -46
- data/examples/scatter_gather/README.adoc +829 -27
- data/examples/simple/README.adoc +347 -0
- data/examples/specialized_workers/README.adoc +622 -26
- data/examples/specialized_workers/specialized_workers.rb +44 -8
- data/examples/stream_processor/README.adoc +206 -0
- data/examples/stream_processor/stream_processor.rb +284 -0
- data/examples/web_scraper/README.adoc +625 -0
- data/examples/web_scraper/web_scraper.rb +285 -0
- data/examples/workflow/README.adoc +406 -0
- data/examples/workflow/circuit_breaker/README.adoc +360 -0
- data/examples/workflow/circuit_breaker/circuit_breaker_workflow.rb +225 -0
- data/examples/workflow/conditional/README.adoc +483 -0
- data/examples/workflow/conditional/conditional_workflow.rb +215 -0
- data/examples/workflow/dead_letter_queue/README.adoc +374 -0
- data/examples/workflow/dead_letter_queue/dead_letter_queue_workflow.rb +217 -0
- data/examples/workflow/fan_out/README.adoc +381 -0
- data/examples/workflow/fan_out/fan_out_workflow.rb +202 -0
- data/examples/workflow/retry/README.adoc +248 -0
- data/examples/workflow/retry/retry_workflow.rb +195 -0
- data/examples/workflow/simple_linear/README.adoc +267 -0
- data/examples/workflow/simple_linear/simple_linear_workflow.rb +175 -0
- data/examples/workflow/simplified/README.adoc +329 -0
- data/examples/workflow/simplified/simplified_workflow.rb +222 -0
- data/exe/fractor +10 -0
- data/lib/fractor/cli.rb +288 -0
- data/lib/fractor/configuration.rb +307 -0
- data/lib/fractor/continuous_server.rb +60 -65
- data/lib/fractor/error_formatter.rb +72 -0
- data/lib/fractor/error_report_generator.rb +152 -0
- data/lib/fractor/error_reporter.rb +244 -0
- data/lib/fractor/error_statistics.rb +147 -0
- data/lib/fractor/execution_tracer.rb +162 -0
- data/lib/fractor/logger.rb +230 -0
- data/lib/fractor/main_loop_handler.rb +406 -0
- data/lib/fractor/main_loop_handler3.rb +135 -0
- data/lib/fractor/main_loop_handler4.rb +299 -0
- data/lib/fractor/performance_metrics_collector.rb +181 -0
- data/lib/fractor/performance_monitor.rb +215 -0
- data/lib/fractor/performance_report_generator.rb +202 -0
- data/lib/fractor/priority_work.rb +93 -0
- data/lib/fractor/priority_work_queue.rb +189 -0
- data/lib/fractor/result_aggregator.rb +32 -0
- data/lib/fractor/shutdown_handler.rb +168 -0
- data/lib/fractor/signal_handler.rb +80 -0
- data/lib/fractor/supervisor.rb +382 -269
- data/lib/fractor/supervisor_logger.rb +88 -0
- data/lib/fractor/version.rb +1 -1
- data/lib/fractor/work.rb +12 -0
- data/lib/fractor/work_distribution_manager.rb +151 -0
- data/lib/fractor/work_queue.rb +20 -0
- data/lib/fractor/work_result.rb +181 -9
- data/lib/fractor/worker.rb +73 -0
- data/lib/fractor/workflow/builder.rb +210 -0
- data/lib/fractor/workflow/chain_builder.rb +169 -0
- data/lib/fractor/workflow/circuit_breaker.rb +183 -0
- data/lib/fractor/workflow/circuit_breaker_orchestrator.rb +208 -0
- data/lib/fractor/workflow/circuit_breaker_registry.rb +112 -0
- data/lib/fractor/workflow/dead_letter_queue.rb +334 -0
- data/lib/fractor/workflow/execution_hooks.rb +39 -0
- data/lib/fractor/workflow/execution_strategy.rb +225 -0
- data/lib/fractor/workflow/execution_trace.rb +134 -0
- data/lib/fractor/workflow/helpers.rb +191 -0
- data/lib/fractor/workflow/job.rb +290 -0
- data/lib/fractor/workflow/job_dependency_validator.rb +120 -0
- data/lib/fractor/workflow/logger.rb +110 -0
- data/lib/fractor/workflow/pre_execution_context.rb +193 -0
- data/lib/fractor/workflow/retry_config.rb +156 -0
- data/lib/fractor/workflow/retry_orchestrator.rb +184 -0
- data/lib/fractor/workflow/retry_strategy.rb +93 -0
- data/lib/fractor/workflow/structured_logger.rb +30 -0
- data/lib/fractor/workflow/type_compatibility_validator.rb +222 -0
- data/lib/fractor/workflow/visualizer.rb +211 -0
- data/lib/fractor/workflow/workflow_context.rb +132 -0
- data/lib/fractor/workflow/workflow_executor.rb +669 -0
- data/lib/fractor/workflow/workflow_result.rb +55 -0
- data/lib/fractor/workflow/workflow_validator.rb +295 -0
- data/lib/fractor/workflow.rb +333 -0
- data/lib/fractor/wrapped_ractor.rb +66 -101
- data/lib/fractor/wrapped_ractor3.rb +161 -0
- data/lib/fractor/wrapped_ractor4.rb +242 -0
- data/lib/fractor.rb +92 -4
- metadata +179 -6
- data/tests/sample.rb.bak +0 -309
- data/tests/sample_working.rb.bak +0 -209
|
@@ -57,8 +57,8 @@ module PipelineProcessing
|
|
|
57
57
|
|
|
58
58
|
# Update metadata with processing information
|
|
59
59
|
updated_metadata = work.metadata.merge(
|
|
60
|
-
"#{work.stage}_completed" => true,
|
|
61
|
-
"#{work.stage}_time" => Time.now.to_s,
|
|
60
|
+
"#{work.stage}_completed".to_sym => true,
|
|
61
|
+
"#{work.stage}_time".to_sym => Time.now.to_s,
|
|
62
62
|
)
|
|
63
63
|
|
|
64
64
|
# Return the result with next stage information
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "../lib/fractor"
|
|
5
|
+
|
|
6
|
+
# Example demonstrating priority-based work processing
|
|
7
|
+
# This shows how to use PriorityWork and PriorityWorkQueue for
|
|
8
|
+
# processing tasks based on their priority levels
|
|
9
|
+
|
|
10
|
+
# Define a worker that processes priority work
|
|
11
|
+
class PriorityWorker < Fractor::Worker
|
|
12
|
+
def process(work)
|
|
13
|
+
# Simulate work processing
|
|
14
|
+
sleep 0.1
|
|
15
|
+
|
|
16
|
+
result = "Processed #{work.input[:task]} " \
|
|
17
|
+
"(priority: #{work.priority}, age: #{work.age.round(2)}s)"
|
|
18
|
+
|
|
19
|
+
Fractor::WorkResult.new(result: result, work: work)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
puts "=" * 60
|
|
24
|
+
puts "Priority Work Example"
|
|
25
|
+
puts "=" * 60
|
|
26
|
+
puts
|
|
27
|
+
|
|
28
|
+
# Example 1: Basic Priority Queue
|
|
29
|
+
puts "Example 1: Basic Priority Ordering"
|
|
30
|
+
puts "-" * 60
|
|
31
|
+
|
|
32
|
+
queue = Fractor::PriorityWorkQueue.new
|
|
33
|
+
|
|
34
|
+
# Add work items with different priorities
|
|
35
|
+
queue.push(Fractor::PriorityWork.new({ task: "Background report" }, priority: :background))
|
|
36
|
+
queue.push(Fractor::PriorityWork.new({ task: "Critical bug fix" }, priority: :critical))
|
|
37
|
+
queue.push(Fractor::PriorityWork.new({ task: "Normal feature" }, priority: :normal))
|
|
38
|
+
queue.push(Fractor::PriorityWork.new({ task: "High priority task" }, priority: :high))
|
|
39
|
+
queue.push(Fractor::PriorityWork.new({ task: "Low priority cleanup" }, priority: :low))
|
|
40
|
+
|
|
41
|
+
puts "Queue statistics:"
|
|
42
|
+
stats = queue.stats
|
|
43
|
+
puts " Total items: #{stats[:total]}"
|
|
44
|
+
puts " By priority: #{stats[:by_priority]}"
|
|
45
|
+
puts
|
|
46
|
+
|
|
47
|
+
puts "Processing in priority order:"
|
|
48
|
+
5.times do
|
|
49
|
+
work = queue.pop_non_blocking
|
|
50
|
+
puts " #{work.input[:task]} (#{work.priority})"
|
|
51
|
+
end
|
|
52
|
+
puts
|
|
53
|
+
|
|
54
|
+
# Example 2: Priority Aging
|
|
55
|
+
puts "Example 2: Priority Aging (Preventing Starvation)"
|
|
56
|
+
puts "-" * 60
|
|
57
|
+
|
|
58
|
+
aged_queue = Fractor::PriorityWorkQueue.new(
|
|
59
|
+
aging_enabled: true,
|
|
60
|
+
aging_threshold: 2 # 2 seconds
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Add a low-priority item first
|
|
64
|
+
aged_queue.push(Fractor::PriorityWork.new(
|
|
65
|
+
{ task: "Old low-priority task" },
|
|
66
|
+
priority: :low
|
|
67
|
+
))
|
|
68
|
+
|
|
69
|
+
puts "Added low-priority task at #{Time.now.strftime('%H:%M:%S')}"
|
|
70
|
+
puts "Waiting 3 seconds to let it age..."
|
|
71
|
+
sleep 3
|
|
72
|
+
|
|
73
|
+
# Add high-priority items after the low-priority one has aged
|
|
74
|
+
aged_queue.push(Fractor::PriorityWork.new(
|
|
75
|
+
{ task: "New high-priority task" },
|
|
76
|
+
priority: :high
|
|
77
|
+
))
|
|
78
|
+
|
|
79
|
+
puts "Added high-priority task at #{Time.now.strftime('%H:%M:%S')}"
|
|
80
|
+
puts
|
|
81
|
+
|
|
82
|
+
puts "Processing order with aging enabled:"
|
|
83
|
+
2.times do
|
|
84
|
+
work = aged_queue.pop_non_blocking
|
|
85
|
+
puts " #{work.input[:task]} " \
|
|
86
|
+
"(priority: #{work.priority}, age: #{work.age.round(1)}s)"
|
|
87
|
+
end
|
|
88
|
+
puts "Note: The aged low-priority task was processed first!"
|
|
89
|
+
puts
|
|
90
|
+
|
|
91
|
+
# Example 3: Using with Supervisor
|
|
92
|
+
puts "Example 3: Integration with Supervisor"
|
|
93
|
+
puts "-" * 60
|
|
94
|
+
|
|
95
|
+
priority_queue = Fractor::PriorityWorkQueue.new
|
|
96
|
+
|
|
97
|
+
# Add mixed priority work
|
|
98
|
+
[
|
|
99
|
+
{ task: "Process payment", priority: :critical },
|
|
100
|
+
{ task: "Send email", priority: :normal },
|
|
101
|
+
{ task: "Generate report", priority: :low },
|
|
102
|
+
{ task: "Update inventory", priority: :high },
|
|
103
|
+
{ task: "Cleanup cache", priority: :background }
|
|
104
|
+
].each do |item|
|
|
105
|
+
priority_queue.push(Fractor::PriorityWork.new(item, priority: item[:priority]))
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Create supervisor with priority queue
|
|
109
|
+
supervisor = Fractor::Supervisor.new(
|
|
110
|
+
work_queue: priority_queue,
|
|
111
|
+
worker_pools: [
|
|
112
|
+
{ worker_class: PriorityWorker, num_workers: 2 }
|
|
113
|
+
]
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
puts "Processing #{priority_queue.size} tasks with 2 workers..."
|
|
117
|
+
supervisor.start
|
|
118
|
+
|
|
119
|
+
# Wait for all work to complete
|
|
120
|
+
sleep 1 until priority_queue.empty?
|
|
121
|
+
|
|
122
|
+
supervisor.shutdown
|
|
123
|
+
results = supervisor.results
|
|
124
|
+
|
|
125
|
+
puts "\nResults (in completion order):"
|
|
126
|
+
results.each_with_index do |result, i|
|
|
127
|
+
puts " #{i + 1}. #{result.result}"
|
|
128
|
+
end
|
|
129
|
+
puts
|
|
130
|
+
|
|
131
|
+
# Example 4: Queue Statistics
|
|
132
|
+
puts "Example 4: Monitoring Queue Statistics"
|
|
133
|
+
puts "-" * 60
|
|
134
|
+
|
|
135
|
+
stats_queue = Fractor::PriorityWorkQueue.new
|
|
136
|
+
|
|
137
|
+
# Add various priority items
|
|
138
|
+
10.times do |i|
|
|
139
|
+
priority = [:critical, :high, :normal, :low, :background].sample
|
|
140
|
+
stats_queue.push(Fractor::PriorityWork.new({ id: i }, priority: priority))
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
stats = stats_queue.stats
|
|
144
|
+
puts "Queue statistics:"
|
|
145
|
+
puts " Total items: #{stats[:total]}"
|
|
146
|
+
puts " Closed: #{stats[:closed]}"
|
|
147
|
+
puts " Items by priority:"
|
|
148
|
+
stats[:by_priority].each do |priority, count|
|
|
149
|
+
puts " #{priority}: #{count}"
|
|
150
|
+
end
|
|
151
|
+
puts
|
|
152
|
+
|
|
153
|
+
puts "=" * 60
|
|
154
|
+
puts "Priority Work Examples Complete"
|
|
155
|
+
puts "=" * 60
|