fractor 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +28 -91
- data/docs/ARCHITECTURE.md +317 -0
- data/docs/PERFORMANCE_TUNING.md +355 -0
- data/docs/TROUBLESHOOTING.md +463 -0
- data/lib/fractor/callback_registry.rb +106 -0
- data/lib/fractor/config_schema.rb +170 -0
- data/lib/fractor/main_loop_handler.rb +4 -8
- data/lib/fractor/main_loop_handler3.rb +10 -12
- data/lib/fractor/main_loop_handler4.rb +48 -20
- data/lib/fractor/result_cache.rb +58 -10
- data/lib/fractor/shutdown_handler.rb +12 -6
- data/lib/fractor/supervisor.rb +100 -13
- data/lib/fractor/version.rb +1 -1
- data/lib/fractor/workflow/execution/dependency_resolver.rb +149 -0
- data/lib/fractor/workflow/execution/fallback_job_handler.rb +68 -0
- data/lib/fractor/workflow/execution/job_executor.rb +242 -0
- data/lib/fractor/workflow/execution/result_builder.rb +76 -0
- data/lib/fractor/workflow/execution/workflow_execution_logger.rb +241 -0
- data/lib/fractor/workflow/workflow_executor.rb +97 -476
- data/lib/fractor/wrapped_ractor.rb +2 -4
- data/lib/fractor.rb +11 -0
- metadata +12 -2
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../workflow_result"
|
|
4
|
+
|
|
5
|
+
module Fractor
|
|
6
|
+
class Workflow
|
|
7
|
+
# Builds the final workflow result from completed jobs and context.
|
|
8
|
+
# Responsible for finding the workflow output and creating the result object.
|
|
9
|
+
class ResultBuilder
|
|
10
|
+
# Initialize the result builder.
|
|
11
|
+
#
|
|
12
|
+
# @param workflow [Workflow] The workflow instance
|
|
13
|
+
# @param context [WorkflowContext] The execution context
|
|
14
|
+
# @param completed_jobs [Set<String>] Set of completed job names
|
|
15
|
+
# @param failed_jobs [Set<String>] Set of failed job names
|
|
16
|
+
# @param trace [ExecutionTrace, nil] Optional execution trace
|
|
17
|
+
def initialize(workflow, context, completed_jobs, failed_jobs, trace: nil)
|
|
18
|
+
@workflow = workflow
|
|
19
|
+
@context = context
|
|
20
|
+
@completed_jobs = completed_jobs
|
|
21
|
+
@failed_jobs = failed_jobs
|
|
22
|
+
@trace = trace
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Build the workflow result.
|
|
26
|
+
#
|
|
27
|
+
# @param start_time [Time] Workflow start time
|
|
28
|
+
# @param end_time [Time] Workflow end time
|
|
29
|
+
# @return [WorkflowResult] The workflow execution result
|
|
30
|
+
def build(start_time, end_time)
|
|
31
|
+
output = find_workflow_output
|
|
32
|
+
|
|
33
|
+
WorkflowResult.new(
|
|
34
|
+
workflow_name: @workflow.class.workflow_name,
|
|
35
|
+
output: output,
|
|
36
|
+
completed_jobs: @completed_jobs.to_a,
|
|
37
|
+
failed_jobs: @failed_jobs.to_a,
|
|
38
|
+
execution_time: end_time - start_time,
|
|
39
|
+
success: @failed_jobs.empty?,
|
|
40
|
+
trace: @trace,
|
|
41
|
+
correlation_id: @context.correlation_id,
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# Find the workflow output from completed jobs.
|
|
48
|
+
# Looks for jobs marked with outputs_to_workflow, then falls back to end jobs.
|
|
49
|
+
#
|
|
50
|
+
# @return [Object, nil] The workflow output, or nil if not found
|
|
51
|
+
def find_workflow_output
|
|
52
|
+
# Look for jobs that map to workflow output
|
|
53
|
+
@workflow.class.jobs.each do |name, job|
|
|
54
|
+
if job.outputs_to_workflow? && @completed_jobs.include?(name)
|
|
55
|
+
output = @context.job_output(name)
|
|
56
|
+
puts "Found workflow output from job '#{name}': #{output.class}" if ENV["FRACTOR_DEBUG"]
|
|
57
|
+
return output
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Fallback: return output from the first end job that completed
|
|
62
|
+
@workflow.class.end_job_names.each do |end_job_spec|
|
|
63
|
+
job_name = end_job_spec[:name]
|
|
64
|
+
if @completed_jobs.include?(job_name)
|
|
65
|
+
output = @context.job_output(job_name)
|
|
66
|
+
puts "Using end job '#{job_name}' output: #{output.class}" if ENV["FRACTOR_DEBUG"]
|
|
67
|
+
return output
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
puts "Warning: No workflow output found!" if ENV["FRACTOR_DEBUG"]
|
|
72
|
+
nil
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Fractor
|
|
4
|
+
class Workflow
|
|
5
|
+
# Handles all logging operations for workflow execution.
|
|
6
|
+
# Provides a clean separation of logging concerns from execution logic.
|
|
7
|
+
# NOTE: This is different from the WorkflowLogger in logger.rb which is a
|
|
8
|
+
# structured logging wrapper. This class extracts logging logic from the executor.
|
|
9
|
+
class WorkflowExecutionLogger
|
|
10
|
+
# Initialize the logger with a context logger.
|
|
11
|
+
#
|
|
12
|
+
# @param context_logger [Logger, nil] The logger from the workflow context
|
|
13
|
+
def initialize(context_logger)
|
|
14
|
+
@logger = context_logger
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Log workflow start.
|
|
18
|
+
#
|
|
19
|
+
# @param workflow_name [String] Name of the workflow
|
|
20
|
+
# @param correlation_id [String, nil] Correlation ID for tracking
|
|
21
|
+
def workflow_start(workflow_name, correlation_id)
|
|
22
|
+
return unless @logger
|
|
23
|
+
|
|
24
|
+
@logger.info(
|
|
25
|
+
"Workflow starting",
|
|
26
|
+
workflow: workflow_name,
|
|
27
|
+
correlation_id: correlation_id,
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Log workflow completion.
|
|
32
|
+
#
|
|
33
|
+
# @param workflow_name [String] Name of the workflow
|
|
34
|
+
# @param duration [Float] Execution duration in seconds
|
|
35
|
+
# @param jobs_completed [Integer] Number of jobs completed
|
|
36
|
+
# @param jobs_failed [Integer] Number of jobs failed
|
|
37
|
+
def workflow_complete(workflow_name, duration, jobs_completed:,
|
|
38
|
+
jobs_failed:)
|
|
39
|
+
return unless @logger
|
|
40
|
+
|
|
41
|
+
@logger.info(
|
|
42
|
+
"Workflow complete",
|
|
43
|
+
workflow: workflow_name,
|
|
44
|
+
duration_ms: (duration * 1000).round(2),
|
|
45
|
+
jobs_completed: jobs_completed,
|
|
46
|
+
jobs_failed: jobs_failed,
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Log job start.
|
|
51
|
+
#
|
|
52
|
+
# @param job_name [String] Name of the job
|
|
53
|
+
# @param worker_class [String] Class name of the worker
|
|
54
|
+
def job_start(job_name, worker_class)
|
|
55
|
+
return unless @logger
|
|
56
|
+
|
|
57
|
+
@logger.info(
|
|
58
|
+
"Job starting",
|
|
59
|
+
job: job_name,
|
|
60
|
+
worker: worker_class,
|
|
61
|
+
)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Log job completion.
|
|
65
|
+
#
|
|
66
|
+
# @param job_name [String] Name of the job
|
|
67
|
+
# @param duration [Float] Execution duration in seconds
|
|
68
|
+
def job_complete(job_name, duration)
|
|
69
|
+
return unless @logger
|
|
70
|
+
|
|
71
|
+
@logger.info(
|
|
72
|
+
"Job complete",
|
|
73
|
+
job: job_name,
|
|
74
|
+
duration_ms: (duration * 1000).round(2),
|
|
75
|
+
)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Log job error.
|
|
79
|
+
#
|
|
80
|
+
# @param job_name [String] Name of the job
|
|
81
|
+
# @param error [Exception] The error that occurred
|
|
82
|
+
# @param has_fallback [Boolean] Whether a fallback job is available
|
|
83
|
+
def job_error(job_name, error, has_fallback: false)
|
|
84
|
+
return unless @logger
|
|
85
|
+
|
|
86
|
+
# Log at WARN level if fallback is available (error is handled),
|
|
87
|
+
# otherwise log at ERROR level (error causes workflow failure)
|
|
88
|
+
log_method = has_fallback ? @logger.method(:warn) : @logger.method(:error)
|
|
89
|
+
|
|
90
|
+
log_method.call(
|
|
91
|
+
"Job '#{job_name}' encountered error: #{error}",
|
|
92
|
+
job: job_name,
|
|
93
|
+
error: error.class.name,
|
|
94
|
+
)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Log retry attempt.
|
|
98
|
+
#
|
|
99
|
+
# @param job_name [String] Name of the job
|
|
100
|
+
# @param attempt [Integer] Current attempt number
|
|
101
|
+
# @param max_attempts [Integer] Maximum number of attempts
|
|
102
|
+
# @param delay [Float] Delay before this retry in seconds
|
|
103
|
+
# @param last_error [Exception, nil] Last error that occurred
|
|
104
|
+
def retry_attempt(job_name, attempt, max_attempts, delay, last_error: nil)
|
|
105
|
+
return unless @logger
|
|
106
|
+
|
|
107
|
+
@logger.warn(
|
|
108
|
+
"Job retry attempt",
|
|
109
|
+
job: job_name,
|
|
110
|
+
attempt: attempt,
|
|
111
|
+
max_attempts: max_attempts,
|
|
112
|
+
delay_seconds: delay,
|
|
113
|
+
last_error: last_error&.message,
|
|
114
|
+
)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Log retry success.
|
|
118
|
+
#
|
|
119
|
+
# @param job_name [String] Name of the job
|
|
120
|
+
# @param attempt [Integer] Successful attempt number
|
|
121
|
+
# @param total_attempts [Integer] Total number of attempts made
|
|
122
|
+
# @param total_time [Float] Total time spent retrying in seconds
|
|
123
|
+
def retry_success(job_name, attempt, total_attempts, total_time)
|
|
124
|
+
return unless @logger
|
|
125
|
+
|
|
126
|
+
@logger.info(
|
|
127
|
+
"Job retry succeeded",
|
|
128
|
+
job: job_name,
|
|
129
|
+
successful_attempt: attempt,
|
|
130
|
+
total_attempts: total_attempts,
|
|
131
|
+
total_time: total_time,
|
|
132
|
+
)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Log retry exhausted.
|
|
136
|
+
#
|
|
137
|
+
# @param job_name [String] Name of the job
|
|
138
|
+
# @param attempts [Integer] Total number of attempts made
|
|
139
|
+
# @param total_time [Float] Total time spent retrying in seconds
|
|
140
|
+
# @param errors [Array<Exception>] All errors that occurred
|
|
141
|
+
def retry_exhausted(job_name, attempts, total_time, errors)
|
|
142
|
+
return unless @logger
|
|
143
|
+
|
|
144
|
+
@logger.error(
|
|
145
|
+
"Job retry attempts exhausted",
|
|
146
|
+
job: job_name,
|
|
147
|
+
total_attempts: attempts,
|
|
148
|
+
total_time: total_time,
|
|
149
|
+
errors: errors,
|
|
150
|
+
)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Log fallback job execution.
|
|
154
|
+
#
|
|
155
|
+
# @param job_name [String] Name of the original job
|
|
156
|
+
# @param fallback_job_name [String] Name of the fallback job
|
|
157
|
+
# @param original_error [Exception] The error that triggered fallback
|
|
158
|
+
def fallback_execution(job_name, fallback_job_name, original_error)
|
|
159
|
+
return unless @logger
|
|
160
|
+
|
|
161
|
+
@logger.warn(
|
|
162
|
+
"Executing fallback job",
|
|
163
|
+
job: job_name,
|
|
164
|
+
fallback_job: fallback_job_name,
|
|
165
|
+
original_error: original_error.message,
|
|
166
|
+
)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Log fallback job failure.
|
|
170
|
+
#
|
|
171
|
+
# @param job_name [String] Name of the original job
|
|
172
|
+
# @param fallback_job_name [String] Name of the fallback job
|
|
173
|
+
# @param error [Exception] The error that occurred in fallback
|
|
174
|
+
def fallback_failed(job_name, fallback_job_name, error)
|
|
175
|
+
return unless @logger
|
|
176
|
+
|
|
177
|
+
@logger.error(
|
|
178
|
+
"Fallback job failed",
|
|
179
|
+
job: job_name,
|
|
180
|
+
fallback_job: fallback_job_name,
|
|
181
|
+
error: error.message,
|
|
182
|
+
)
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Log circuit breaker state.
|
|
186
|
+
#
|
|
187
|
+
# @param job_name [String] Name of the job
|
|
188
|
+
# @param state [Symbol] Current circuit breaker state
|
|
189
|
+
# @param failure_count [Integer] Number of failures
|
|
190
|
+
# @param threshold [Integer] Failure threshold
|
|
191
|
+
def circuit_breaker_state(job_name, state, failure_count:, threshold:)
|
|
192
|
+
return unless @logger
|
|
193
|
+
return if state == :closed
|
|
194
|
+
|
|
195
|
+
@logger.warn(
|
|
196
|
+
"Circuit breaker state",
|
|
197
|
+
job: job_name,
|
|
198
|
+
state: state,
|
|
199
|
+
failure_count: failure_count,
|
|
200
|
+
threshold: threshold,
|
|
201
|
+
)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Log circuit breaker open.
|
|
205
|
+
#
|
|
206
|
+
# @param job_name [String] Name of the job
|
|
207
|
+
# @param failure_count [Integer] Number of failures
|
|
208
|
+
# @param threshold [Integer] Failure threshold
|
|
209
|
+
# @param last_failure [Time, nil] Time of last failure
|
|
210
|
+
def circuit_breaker_open(job_name, failure_count, threshold,
|
|
211
|
+
last_failure: nil)
|
|
212
|
+
return unless @logger
|
|
213
|
+
|
|
214
|
+
@logger.error(
|
|
215
|
+
"Circuit breaker open",
|
|
216
|
+
job: job_name,
|
|
217
|
+
failure_count: failure_count,
|
|
218
|
+
threshold: threshold,
|
|
219
|
+
last_failure: last_failure,
|
|
220
|
+
)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# Log work added to dead letter queue.
|
|
224
|
+
#
|
|
225
|
+
# @param job_name [String] Name of the job
|
|
226
|
+
# @param error [Exception] The error that occurred
|
|
227
|
+
# @param dlq_size [Integer] Current size of the dead letter queue
|
|
228
|
+
def added_to_dead_letter_queue(job_name, error, dlq_size)
|
|
229
|
+
return unless @logger
|
|
230
|
+
|
|
231
|
+
@logger.warn(
|
|
232
|
+
"Work added to Dead Letter Queue",
|
|
233
|
+
job: job_name,
|
|
234
|
+
error: error.class.name,
|
|
235
|
+
message: error.message,
|
|
236
|
+
dlq_size: dlq_size,
|
|
237
|
+
)
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|