job-workflow 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +91 -0
- data/CHANGELOG.md +23 -0
- data/LICENSE.txt +21 -0
- data/README.md +47 -0
- data/Rakefile +55 -0
- data/Steepfile +10 -0
- data/guides/API_REFERENCE.md +112 -0
- data/guides/BEST_PRACTICES.md +113 -0
- data/guides/CACHE_STORE_INTEGRATION.md +145 -0
- data/guides/CONDITIONAL_EXECUTION.md +66 -0
- data/guides/DEPENDENCY_WAIT.md +386 -0
- data/guides/DRY_RUN.md +390 -0
- data/guides/DSL_BASICS.md +216 -0
- data/guides/ERROR_HANDLING.md +187 -0
- data/guides/GETTING_STARTED.md +524 -0
- data/guides/INSTRUMENTATION.md +131 -0
- data/guides/LIFECYCLE_HOOKS.md +415 -0
- data/guides/NAMESPACES.md +75 -0
- data/guides/OPENTELEMETRY_INTEGRATION.md +86 -0
- data/guides/PARALLEL_PROCESSING.md +302 -0
- data/guides/PRODUCTION_DEPLOYMENT.md +110 -0
- data/guides/QUEUE_MANAGEMENT.md +141 -0
- data/guides/README.md +174 -0
- data/guides/SCHEDULED_JOBS.md +165 -0
- data/guides/STRUCTURED_LOGGING.md +268 -0
- data/guides/TASK_OUTPUTS.md +240 -0
- data/guides/TESTING_STRATEGY.md +56 -0
- data/guides/THROTTLING.md +198 -0
- data/guides/TROUBLESHOOTING.md +53 -0
- data/guides/WORKFLOW_COMPOSITION.md +675 -0
- data/guides/WORKFLOW_STATUS_QUERY.md +288 -0
- data/lib/job-workflow.rb +3 -0
- data/lib/job_workflow/argument_def.rb +16 -0
- data/lib/job_workflow/arguments.rb +40 -0
- data/lib/job_workflow/auto_scaling/adapter/aws_adapter.rb +66 -0
- data/lib/job_workflow/auto_scaling/adapter.rb +31 -0
- data/lib/job_workflow/auto_scaling/configuration.rb +85 -0
- data/lib/job_workflow/auto_scaling/executor.rb +43 -0
- data/lib/job_workflow/auto_scaling.rb +69 -0
- data/lib/job_workflow/cache_store_adapters.rb +46 -0
- data/lib/job_workflow/context.rb +352 -0
- data/lib/job_workflow/dry_run_config.rb +31 -0
- data/lib/job_workflow/dsl.rb +236 -0
- data/lib/job_workflow/error_hook.rb +24 -0
- data/lib/job_workflow/hook.rb +24 -0
- data/lib/job_workflow/hook_registry.rb +66 -0
- data/lib/job_workflow/instrumentation/log_subscriber.rb +194 -0
- data/lib/job_workflow/instrumentation/opentelemetry_subscriber.rb +221 -0
- data/lib/job_workflow/instrumentation.rb +257 -0
- data/lib/job_workflow/job_status.rb +92 -0
- data/lib/job_workflow/logger.rb +86 -0
- data/lib/job_workflow/namespace.rb +36 -0
- data/lib/job_workflow/output.rb +81 -0
- data/lib/job_workflow/output_def.rb +14 -0
- data/lib/job_workflow/queue.rb +74 -0
- data/lib/job_workflow/queue_adapter.rb +38 -0
- data/lib/job_workflow/queue_adapters/abstract.rb +87 -0
- data/lib/job_workflow/queue_adapters/null_adapter.rb +127 -0
- data/lib/job_workflow/queue_adapters/solid_queue_adapter.rb +224 -0
- data/lib/job_workflow/runner.rb +173 -0
- data/lib/job_workflow/schedule.rb +46 -0
- data/lib/job_workflow/semaphore.rb +71 -0
- data/lib/job_workflow/task.rb +83 -0
- data/lib/job_workflow/task_callable.rb +43 -0
- data/lib/job_workflow/task_context.rb +70 -0
- data/lib/job_workflow/task_dependency_wait.rb +66 -0
- data/lib/job_workflow/task_enqueue.rb +50 -0
- data/lib/job_workflow/task_graph.rb +43 -0
- data/lib/job_workflow/task_job_status.rb +70 -0
- data/lib/job_workflow/task_output.rb +51 -0
- data/lib/job_workflow/task_retry.rb +64 -0
- data/lib/job_workflow/task_throttle.rb +46 -0
- data/lib/job_workflow/version.rb +5 -0
- data/lib/job_workflow/workflow.rb +87 -0
- data/lib/job_workflow/workflow_status.rb +112 -0
- data/lib/job_workflow.rb +59 -0
- data/rbs_collection.lock.yaml +172 -0
- data/rbs_collection.yaml +14 -0
- data/sig/generated/job-workflow.rbs +2 -0
- data/sig/generated/job_workflow/argument_def.rbs +14 -0
- data/sig/generated/job_workflow/arguments.rbs +26 -0
- data/sig/generated/job_workflow/auto_scaling/adapter/aws_adapter.rbs +32 -0
- data/sig/generated/job_workflow/auto_scaling/adapter.rbs +22 -0
- data/sig/generated/job_workflow/auto_scaling/configuration.rbs +50 -0
- data/sig/generated/job_workflow/auto_scaling/executor.rbs +29 -0
- data/sig/generated/job_workflow/auto_scaling.rbs +47 -0
- data/sig/generated/job_workflow/cache_store_adapters.rbs +28 -0
- data/sig/generated/job_workflow/context.rbs +155 -0
- data/sig/generated/job_workflow/dry_run_config.rbs +16 -0
- data/sig/generated/job_workflow/dsl.rbs +117 -0
- data/sig/generated/job_workflow/error_hook.rbs +18 -0
- data/sig/generated/job_workflow/hook.rbs +18 -0
- data/sig/generated/job_workflow/hook_registry.rbs +47 -0
- data/sig/generated/job_workflow/instrumentation/log_subscriber.rbs +102 -0
- data/sig/generated/job_workflow/instrumentation/opentelemetry_subscriber.rbs +113 -0
- data/sig/generated/job_workflow/instrumentation.rbs +138 -0
- data/sig/generated/job_workflow/job_status.rbs +46 -0
- data/sig/generated/job_workflow/logger.rbs +56 -0
- data/sig/generated/job_workflow/namespace.rbs +24 -0
- data/sig/generated/job_workflow/output.rbs +39 -0
- data/sig/generated/job_workflow/output_def.rbs +12 -0
- data/sig/generated/job_workflow/queue.rbs +49 -0
- data/sig/generated/job_workflow/queue_adapter.rbs +18 -0
- data/sig/generated/job_workflow/queue_adapters/abstract.rbs +56 -0
- data/sig/generated/job_workflow/queue_adapters/null_adapter.rbs +73 -0
- data/sig/generated/job_workflow/queue_adapters/solid_queue_adapter.rbs +111 -0
- data/sig/generated/job_workflow/runner.rbs +66 -0
- data/sig/generated/job_workflow/schedule.rbs +34 -0
- data/sig/generated/job_workflow/semaphore.rbs +37 -0
- data/sig/generated/job_workflow/task.rbs +60 -0
- data/sig/generated/job_workflow/task_callable.rbs +30 -0
- data/sig/generated/job_workflow/task_context.rbs +52 -0
- data/sig/generated/job_workflow/task_dependency_wait.rbs +42 -0
- data/sig/generated/job_workflow/task_enqueue.rbs +27 -0
- data/sig/generated/job_workflow/task_graph.rbs +27 -0
- data/sig/generated/job_workflow/task_job_status.rbs +42 -0
- data/sig/generated/job_workflow/task_output.rbs +29 -0
- data/sig/generated/job_workflow/task_retry.rbs +30 -0
- data/sig/generated/job_workflow/task_throttle.rbs +20 -0
- data/sig/generated/job_workflow/version.rbs +5 -0
- data/sig/generated/job_workflow/workflow.rbs +48 -0
- data/sig/generated/job_workflow/workflow_status.rbs +55 -0
- data/sig/generated/job_workflow.rbs +8 -0
- data/sig-private/activejob.rbs +35 -0
- data/sig-private/activesupport.rbs +23 -0
- data/sig-private/aws.rbs +32 -0
- data/sig-private/opentelemetry.rbs +40 -0
- data/sig-private/solid_queue.rbs +108 -0
- data/tmp/.keep +0 -0
- metadata +190 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/instrumentation.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
# Instrumentation provides ActiveSupport::Notifications-based event instrumentation for JobWorkflow workflows and tasks.
|
|
5
|
+
#
|
|
6
|
+
# @example Subscribing to events
|
|
7
|
+
# ```ruby
|
|
8
|
+
# ActiveSupport::Notifications.subscribe("task.start.job_workflow") do |name, start, finish, id, payload|
|
|
9
|
+
# puts "Task #{payload[:task_name]} started"
|
|
10
|
+
# end
|
|
11
|
+
# ```
|
|
12
|
+
module Instrumentation
|
|
13
|
+
# rubocop:disable Metrics/ModuleLength
|
|
14
|
+
NAMESPACE: ::String
|
|
15
|
+
|
|
16
|
+
module Events
|
|
17
|
+
WORKFLOW: untyped
|
|
18
|
+
|
|
19
|
+
WORKFLOW_START: untyped
|
|
20
|
+
|
|
21
|
+
WORKFLOW_COMPLETE: untyped
|
|
22
|
+
|
|
23
|
+
TASK: untyped
|
|
24
|
+
|
|
25
|
+
TASK_START: untyped
|
|
26
|
+
|
|
27
|
+
TASK_COMPLETE: untyped
|
|
28
|
+
|
|
29
|
+
TASK_ERROR: untyped
|
|
30
|
+
|
|
31
|
+
TASK_SKIP: untyped
|
|
32
|
+
|
|
33
|
+
TASK_ENQUEUE: untyped
|
|
34
|
+
|
|
35
|
+
TASK_RETRY: untyped
|
|
36
|
+
|
|
37
|
+
THROTTLE_ACQUIRE: untyped
|
|
38
|
+
|
|
39
|
+
THROTTLE_ACQUIRE_START: untyped
|
|
40
|
+
|
|
41
|
+
THROTTLE_ACQUIRE_COMPLETE: untyped
|
|
42
|
+
|
|
43
|
+
THROTTLE_RELEASE: untyped
|
|
44
|
+
|
|
45
|
+
DEPENDENT_WAIT: untyped
|
|
46
|
+
|
|
47
|
+
DEPENDENT_WAIT_START: untyped
|
|
48
|
+
|
|
49
|
+
DEPENDENT_WAIT_COMPLETE: untyped
|
|
50
|
+
|
|
51
|
+
DEPENDENT_RESCHEDULE: untyped
|
|
52
|
+
|
|
53
|
+
QUEUE_PAUSE: untyped
|
|
54
|
+
|
|
55
|
+
QUEUE_RESUME: untyped
|
|
56
|
+
|
|
57
|
+
CUSTOM: untyped
|
|
58
|
+
|
|
59
|
+
DRY_RUN: untyped
|
|
60
|
+
|
|
61
|
+
DRY_RUN_SKIP: untyped
|
|
62
|
+
|
|
63
|
+
DRY_RUN_EXECUTE: untyped
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# : (DSL) { () -> untyped } -> untyped
|
|
67
|
+
def self.instrument_workflow: (DSL) { () -> untyped } -> untyped
|
|
68
|
+
|
|
69
|
+
# : (DSL, Task, Context) { () -> untyped } -> untyped
|
|
70
|
+
def self.instrument_task: (DSL, Task, Context) { () -> untyped } -> untyped
|
|
71
|
+
|
|
72
|
+
# : (DSL, Task, String) -> void
|
|
73
|
+
def self.notify_task_skip: (DSL, Task, String) -> void
|
|
74
|
+
|
|
75
|
+
# : (DSL, Task, Integer) -> void
|
|
76
|
+
def self.notify_task_enqueue: (DSL, Task, Integer) -> void
|
|
77
|
+
|
|
78
|
+
# : (Task, Context, String, Integer, Float, StandardError) -> void
|
|
79
|
+
def self.notify_task_retry: (Task, Context, String, Integer, Float, StandardError) -> void
|
|
80
|
+
|
|
81
|
+
# : (DSL, Task) { () -> untyped } -> untyped
|
|
82
|
+
def self.instrument_dependent_wait: (DSL, Task) { () -> untyped } -> untyped
|
|
83
|
+
|
|
84
|
+
# : (DSL, Task, Numeric, Integer) -> void
|
|
85
|
+
def self.notify_dependent_reschedule: (DSL, Task, Numeric, Integer) -> void
|
|
86
|
+
|
|
87
|
+
# : (Semaphore) { () -> untyped } -> untyped
|
|
88
|
+
def self.instrument_throttle: (Semaphore) { () -> untyped } -> untyped
|
|
89
|
+
|
|
90
|
+
# : (Semaphore) -> void
|
|
91
|
+
def self.notify_throttle_release: (Semaphore) -> void
|
|
92
|
+
|
|
93
|
+
# : (String) -> void
|
|
94
|
+
def self.notify_queue_pause: (String) -> void
|
|
95
|
+
|
|
96
|
+
# : (String) -> void
|
|
97
|
+
def self.notify_queue_resume: (String) -> void
|
|
98
|
+
|
|
99
|
+
# : (String, Hash[Symbol, untyped]) { () -> untyped } -> untyped
|
|
100
|
+
def self.instrument_custom: (String, Hash[Symbol, untyped]) { () -> untyped } -> untyped
|
|
101
|
+
|
|
102
|
+
# : (DSL, Context, Symbol?, Integer, bool) { () -> untyped } -> untyped
|
|
103
|
+
def self.instrument_dry_run: (DSL, Context, Symbol?, Integer, bool) { () -> untyped } -> untyped
|
|
104
|
+
|
|
105
|
+
# : (String, Hash[Symbol, untyped]) ?{ () -> untyped } -> untyped
|
|
106
|
+
private def self.instrument: (String, Hash[Symbol, untyped]) ?{ () -> untyped } -> untyped
|
|
107
|
+
|
|
108
|
+
# : (DSL) -> Hash[Symbol, untyped]
|
|
109
|
+
private def self.build_workflow_payload: (DSL) -> Hash[Symbol, untyped]
|
|
110
|
+
|
|
111
|
+
# : (DSL, Task, Context) -> Hash[Symbol, untyped]
|
|
112
|
+
private def self.build_task_payload: (DSL, Task, Context) -> Hash[Symbol, untyped]
|
|
113
|
+
|
|
114
|
+
# : (DSL, Task, String) -> Hash[Symbol, untyped]
|
|
115
|
+
private def self.build_task_skip_payload: (DSL, Task, String) -> Hash[Symbol, untyped]
|
|
116
|
+
|
|
117
|
+
# : (DSL, Task, Integer) -> Hash[Symbol, untyped]
|
|
118
|
+
private def self.build_task_enqueue_payload: (DSL, Task, Integer) -> Hash[Symbol, untyped]
|
|
119
|
+
|
|
120
|
+
# : (Task, Context, String, Integer, Float, StandardError) -> Hash[Symbol, untyped]
|
|
121
|
+
private def self.build_task_retry_payload: (Task, Context, String, Integer, Float, StandardError) -> Hash[Symbol, untyped]
|
|
122
|
+
|
|
123
|
+
# : (DSL, Task) -> Hash[Symbol, untyped]
|
|
124
|
+
private def self.build_dependent_payload: (DSL, Task) -> Hash[Symbol, untyped]
|
|
125
|
+
|
|
126
|
+
# : (DSL, Task, Numeric, Integer) -> Hash[Symbol, untyped]
|
|
127
|
+
private def self.build_dependent_reschedule_payload: (DSL, Task, Numeric, Integer) -> Hash[Symbol, untyped]
|
|
128
|
+
|
|
129
|
+
# : (Semaphore) -> Hash[Symbol, untyped]
|
|
130
|
+
private def self.build_throttle_payload: (Semaphore) -> Hash[Symbol, untyped]
|
|
131
|
+
|
|
132
|
+
# : (String) -> Hash[Symbol, untyped]
|
|
133
|
+
private def self.build_queue_payload: (String) -> Hash[Symbol, untyped]
|
|
134
|
+
|
|
135
|
+
# : (DSL, Context, Symbol?, Integer, bool) -> Hash[Symbol, untyped]
|
|
136
|
+
private def self.build_skip_in_dry_run_payload: (DSL, Context, Symbol?, Integer, bool) -> Hash[Symbol, untyped]
|
|
137
|
+
end
|
|
138
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/job_status.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
class JobStatus
|
|
5
|
+
# : (Array[Hash[untyped, untyped]]) -> JobStatus
|
|
6
|
+
def self.from_hash_array: (Array[Hash[untyped, untyped]]) -> JobStatus
|
|
7
|
+
|
|
8
|
+
# : (Hash[String, untyped]) -> JobStatus
|
|
9
|
+
def self.deserialize: (Hash[String, untyped]) -> JobStatus
|
|
10
|
+
|
|
11
|
+
# : (?task_job_statuses: Array[TaskJobStatus]) -> void
|
|
12
|
+
def initialize: (?task_job_statuses: Array[TaskJobStatus]) -> void
|
|
13
|
+
|
|
14
|
+
# : (task_name: Symbol) -> Array[TaskJobStatus]
|
|
15
|
+
def fetch_all: (task_name: Symbol) -> Array[TaskJobStatus]
|
|
16
|
+
|
|
17
|
+
# : (task_name: Symbol, index: Integer) -> TaskJobStatus?
|
|
18
|
+
def fetch: (task_name: Symbol, index: Integer) -> TaskJobStatus?
|
|
19
|
+
|
|
20
|
+
# : (task_name: Symbol) -> Array[String]
|
|
21
|
+
def finished_job_ids: (task_name: Symbol) -> Array[String]
|
|
22
|
+
|
|
23
|
+
# : () -> Array[TaskJobStatus]
|
|
24
|
+
def flat_task_job_statuses: () -> Array[TaskJobStatus]
|
|
25
|
+
|
|
26
|
+
# @note
|
|
27
|
+
# - If the array is empty, the task is not enqueued and is considered completed.
|
|
28
|
+
# - If we add a task existence check in the future, we'll check here.
|
|
29
|
+
#
|
|
30
|
+
# : (Symbol) -> bool
|
|
31
|
+
def needs_waiting?: (Symbol) -> bool
|
|
32
|
+
|
|
33
|
+
# : (TaskJobStatus) -> void
|
|
34
|
+
def update_task_job_status: (TaskJobStatus) -> void
|
|
35
|
+
|
|
36
|
+
# : (task_name: Symbol, jobs: Array[DSL]) -> void
|
|
37
|
+
def update_task_job_statuses_from_jobs: (task_name: Symbol, jobs: Array[DSL]) -> void
|
|
38
|
+
|
|
39
|
+
# : (Symbol) -> void
|
|
40
|
+
def update_task_job_statuses_from_db: (Symbol) -> void
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
attr_accessor task_job_statuses: Hash[Symbol, Array[TaskJobStatus]]
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/logger.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
# Logger provides structured JSON logging for JobWorkflow workflows.
|
|
5
|
+
#
|
|
6
|
+
# @example Basic usage
|
|
7
|
+
# ```ruby
|
|
8
|
+
# JobWorkflow.logger = ActiveSupport::Logger.new($stdout)
|
|
9
|
+
# JobWorkflow.logger.formatter = JobWorkflow::Logger::JsonFormatter.new
|
|
10
|
+
# ```
|
|
11
|
+
#
|
|
12
|
+
# @example With custom log tags
|
|
13
|
+
# ```ruby
|
|
14
|
+
# JobWorkflow.logger.formatter = JobWorkflow::Logger::JsonFormatter.new(log_tags: [:request_id])
|
|
15
|
+
# ```
|
|
16
|
+
module Logger
|
|
17
|
+
# : (ActiveSupport::Logger) -> void
|
|
18
|
+
attr_writer logger: untyped
|
|
19
|
+
|
|
20
|
+
# : () -> ActiveSupport::Logger
|
|
21
|
+
def logger: () -> ActiveSupport::Logger
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
# : () -> ActiveSupport::Logger
|
|
26
|
+
def build_default_logger: () -> ActiveSupport::Logger
|
|
27
|
+
|
|
28
|
+
# JSON formatter for structured logging output.
|
|
29
|
+
# @rbs inherits ::Logger::Formatter
|
|
30
|
+
class JsonFormatter < ::Logger::Formatter
|
|
31
|
+
include ActiveSupport::TaggedLogging::Formatter
|
|
32
|
+
|
|
33
|
+
# : (?log_tags: Array[Symbol]) -> void
|
|
34
|
+
def initialize: (?log_tags: Array[Symbol]) -> void
|
|
35
|
+
|
|
36
|
+
# : (String, Time, String, String | Hash[untyped, untyped]) -> String
|
|
37
|
+
def call: (String, Time, String, String | Hash[untyped, untyped]) -> String
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
attr_reader log_tags: Array[Symbol]
|
|
42
|
+
|
|
43
|
+
# : (String, Time, String) -> Hash[Symbol, untyped]
|
|
44
|
+
def build_base_hash: (String, Time, String) -> Hash[Symbol, untyped]
|
|
45
|
+
|
|
46
|
+
# : () -> Hash[Symbol, untyped]
|
|
47
|
+
def build_tags_hash: () -> Hash[Symbol, untyped]
|
|
48
|
+
|
|
49
|
+
# : (String | Hash[untyped, untyped]) -> Hash[Symbol, untyped]
|
|
50
|
+
def build_msg_hash: (String | Hash[untyped, untyped]) -> Hash[Symbol, untyped]
|
|
51
|
+
|
|
52
|
+
# : (String) -> Hash[Symbol, untyped]
|
|
53
|
+
def parse_json_or_message: (String) -> Hash[Symbol, untyped]
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/namespace.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
class Namespace
|
|
5
|
+
attr_reader name: Symbol
|
|
6
|
+
|
|
7
|
+
attr_reader parent: Namespace?
|
|
8
|
+
|
|
9
|
+
# : () -> Namespace
|
|
10
|
+
def self.default: () -> Namespace
|
|
11
|
+
|
|
12
|
+
# : (name: Symbol, ?parent: Namespace?) -> void
|
|
13
|
+
def initialize: (name: Symbol, ?parent: Namespace?) -> void
|
|
14
|
+
|
|
15
|
+
# : () -> bool
|
|
16
|
+
def default?: () -> bool
|
|
17
|
+
|
|
18
|
+
# : (Namespace) -> Namespace
|
|
19
|
+
def update_parent: (Namespace) -> Namespace
|
|
20
|
+
|
|
21
|
+
# : () -> Symbol
|
|
22
|
+
def full_name: () -> Symbol
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/output.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
class Output
|
|
5
|
+
# : (Array[Hash[untyped, untyped]]) -> Output
|
|
6
|
+
def self.from_hash_array: (Array[Hash[untyped, untyped]]) -> Output
|
|
7
|
+
|
|
8
|
+
# : (Hash[String, untyped]) -> Output
|
|
9
|
+
def self.deserialize: (Hash[String, untyped]) -> Output
|
|
10
|
+
|
|
11
|
+
# : (?task_outputs: Array[TaskOutput]) -> void
|
|
12
|
+
def initialize: (?task_outputs: Array[TaskOutput]) -> void
|
|
13
|
+
|
|
14
|
+
# : (task_name: Symbol?) -> Array[TaskOutput]
|
|
15
|
+
def fetch_all: (task_name: Symbol?) -> Array[TaskOutput]
|
|
16
|
+
|
|
17
|
+
# : (task_name: Symbol?, each_index: Integer) -> TaskOutput?
|
|
18
|
+
def fetch: (task_name: Symbol?, each_index: Integer) -> TaskOutput?
|
|
19
|
+
|
|
20
|
+
# : (Symbol | String) -> Array[TaskOutput?]
|
|
21
|
+
def []: (Symbol | String) -> Array[TaskOutput?]
|
|
22
|
+
|
|
23
|
+
# : (TaskOutput) -> void
|
|
24
|
+
def add_task_output: (TaskOutput) -> void
|
|
25
|
+
|
|
26
|
+
# : (Array[String], Workflow) -> void
|
|
27
|
+
def update_task_outputs_from_db: (Array[String], Workflow) -> void
|
|
28
|
+
|
|
29
|
+
# : (Array[SolidQueue::Job], Workflow) -> void
|
|
30
|
+
def update_task_outputs_from_jobs: (Array[SolidQueue::Job], Workflow) -> void
|
|
31
|
+
|
|
32
|
+
# : () -> Array[TaskOutput]
|
|
33
|
+
def flat_task_outputs: () -> Array[TaskOutput]
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
attr_accessor task_outputs: Hash[Symbol, Array[TaskOutput]]
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/output_def.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
class OutputDef
|
|
5
|
+
attr_reader name: Symbol
|
|
6
|
+
|
|
7
|
+
attr_reader type: String
|
|
8
|
+
|
|
9
|
+
# : (name: Symbol, type: String) -> void
|
|
10
|
+
def initialize: (name: Symbol, type: String) -> void
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/queue.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
# Queue provides a unified interface for queue operations across different queue adapters.
|
|
5
|
+
#
|
|
6
|
+
# @example Pausing and resuming a queue
|
|
7
|
+
# ```ruby
|
|
8
|
+
# JobWorkflow::Queue.pause(:import_workflow)
|
|
9
|
+
# JobWorkflow::Queue.paused?(:import_workflow) # => true
|
|
10
|
+
# JobWorkflow::Queue.resume(:import_workflow)
|
|
11
|
+
# JobWorkflow::Queue.paused?(:import_workflow) # => false
|
|
12
|
+
# ```
|
|
13
|
+
#
|
|
14
|
+
# @example Getting queue metrics
|
|
15
|
+
# ```ruby
|
|
16
|
+
# JobWorkflow::Queue.latency(:import_workflow) # => 120 (seconds)
|
|
17
|
+
# JobWorkflow::Queue.size(:import_workflow) # => 42 (pending jobs)
|
|
18
|
+
# ```
|
|
19
|
+
#
|
|
20
|
+
# @example Listing workflows associated with a queue
|
|
21
|
+
# ```ruby
|
|
22
|
+
# JobWorkflow::Queue.workflows(:import_workflow) # => [ImportJob, DataSyncJob]
|
|
23
|
+
# ```
|
|
24
|
+
class Queue
|
|
25
|
+
# : (String | Symbol) -> bool
|
|
26
|
+
def self.pause: (String | Symbol) -> bool
|
|
27
|
+
|
|
28
|
+
# : (String | Symbol) -> bool
|
|
29
|
+
def self.resume: (String | Symbol) -> bool
|
|
30
|
+
|
|
31
|
+
# : (String | Symbol) -> bool
|
|
32
|
+
def self.paused?: (String | Symbol) -> bool
|
|
33
|
+
|
|
34
|
+
# : () -> Array[String]
|
|
35
|
+
def self.paused_queues: () -> Array[String]
|
|
36
|
+
|
|
37
|
+
# : (String | Symbol) -> Integer?
|
|
38
|
+
def self.latency: (String | Symbol) -> Integer?
|
|
39
|
+
|
|
40
|
+
# : (String | Symbol) -> Integer
|
|
41
|
+
def self.size: (String | Symbol) -> Integer
|
|
42
|
+
|
|
43
|
+
# : (String | Symbol) -> bool
|
|
44
|
+
def self.clear: (String | Symbol) -> bool
|
|
45
|
+
|
|
46
|
+
# : (String | Symbol) -> Array[singleton(DSL)]
|
|
47
|
+
def self.workflows: (String | Symbol) -> Array[singleton(DSL)]
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/queue_adapter.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
module QueueAdapter
|
|
5
|
+
def self._current: () -> QueueAdapters::Abstract
|
|
6
|
+
|
|
7
|
+
def self._current=: (QueueAdapters::Abstract?) -> void
|
|
8
|
+
|
|
9
|
+
# : () -> QueueAdapters::Abstract
|
|
10
|
+
def self.current: () -> QueueAdapters::Abstract
|
|
11
|
+
|
|
12
|
+
# : () -> void
|
|
13
|
+
def self.reset!: () -> void
|
|
14
|
+
|
|
15
|
+
# : () -> QueueAdapters::Abstract
|
|
16
|
+
private def self.detect_adapter: () -> QueueAdapters::Abstract
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/queue_adapters/abstract.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
module QueueAdapters
|
|
5
|
+
# rubocop:disable Naming/PredicateMethod
|
|
6
|
+
class Abstract
|
|
7
|
+
# : () -> void
|
|
8
|
+
def initialize_adapter!: () -> void
|
|
9
|
+
|
|
10
|
+
# : () -> bool
|
|
11
|
+
def semaphore_available?: () -> bool
|
|
12
|
+
|
|
13
|
+
# : (Semaphore) -> bool
|
|
14
|
+
def semaphore_wait: (Semaphore) -> bool
|
|
15
|
+
|
|
16
|
+
# : (Semaphore) -> bool
|
|
17
|
+
def semaphore_signal: (Semaphore) -> bool
|
|
18
|
+
|
|
19
|
+
# : (Array[String]) -> Hash[String, untyped]
|
|
20
|
+
def fetch_job_statuses: (Array[String]) -> Hash[String, untyped]
|
|
21
|
+
|
|
22
|
+
# : (untyped) -> Symbol
|
|
23
|
+
def job_status: (untyped) -> Symbol
|
|
24
|
+
|
|
25
|
+
# : () -> bool
|
|
26
|
+
def supports_concurrency_limits?: () -> bool
|
|
27
|
+
|
|
28
|
+
# : (String) -> bool
|
|
29
|
+
def pause_queue: (String) -> bool
|
|
30
|
+
|
|
31
|
+
# : (String) -> bool
|
|
32
|
+
def resume_queue: (String) -> bool
|
|
33
|
+
|
|
34
|
+
# : (String) -> bool
|
|
35
|
+
def queue_paused?: (String) -> bool
|
|
36
|
+
|
|
37
|
+
# : () -> Array[String]
|
|
38
|
+
def paused_queues: () -> Array[String]
|
|
39
|
+
|
|
40
|
+
# : (String) -> Integer?
|
|
41
|
+
def queue_latency: (String) -> Integer?
|
|
42
|
+
|
|
43
|
+
# : (String) -> Integer
|
|
44
|
+
def queue_size: (String) -> Integer
|
|
45
|
+
|
|
46
|
+
# : (String) -> bool
|
|
47
|
+
def clear_queue: (String) -> bool
|
|
48
|
+
|
|
49
|
+
# : (String) -> Hash[String, untyped]?
|
|
50
|
+
def find_job: (String) -> Hash[String, untyped]?
|
|
51
|
+
|
|
52
|
+
# : (DSL, Numeric) -> bool
|
|
53
|
+
def reschedule_job: (DSL, Numeric) -> bool
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/queue_adapters/null_adapter.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
module QueueAdapters
|
|
5
|
+
# rubocop:disable Naming/PredicateMethod
|
|
6
|
+
class NullAdapter < Abstract
|
|
7
|
+
# : () -> void
|
|
8
|
+
def initialize_adapter!: () -> void
|
|
9
|
+
|
|
10
|
+
def initialize: () -> untyped
|
|
11
|
+
|
|
12
|
+
# : () -> bool
|
|
13
|
+
def semaphore_available?: () -> bool
|
|
14
|
+
|
|
15
|
+
# : (Semaphore) -> bool
|
|
16
|
+
def semaphore_wait: (Semaphore) -> bool
|
|
17
|
+
|
|
18
|
+
# : (Semaphore) -> bool
|
|
19
|
+
def semaphore_signal: (Semaphore) -> bool
|
|
20
|
+
|
|
21
|
+
# : (Array[String]) -> Hash[String, untyped]
|
|
22
|
+
def fetch_job_statuses: (Array[String]) -> Hash[String, untyped]
|
|
23
|
+
|
|
24
|
+
# : (untyped) -> Symbol
|
|
25
|
+
def job_status: (untyped) -> Symbol
|
|
26
|
+
|
|
27
|
+
# : () -> bool
|
|
28
|
+
def supports_concurrency_limits?: () -> bool
|
|
29
|
+
|
|
30
|
+
# : (String) -> bool
|
|
31
|
+
def pause_queue: (String) -> bool
|
|
32
|
+
|
|
33
|
+
# : (String) -> bool
|
|
34
|
+
def resume_queue: (String) -> bool
|
|
35
|
+
|
|
36
|
+
# : (String) -> bool
|
|
37
|
+
def queue_paused?: (String) -> bool
|
|
38
|
+
|
|
39
|
+
# : () -> Array[String]
|
|
40
|
+
def paused_queues: () -> Array[String]
|
|
41
|
+
|
|
42
|
+
# : (String) -> Integer?
|
|
43
|
+
def queue_latency: (String) -> Integer?
|
|
44
|
+
|
|
45
|
+
# : (String) -> Integer
|
|
46
|
+
def queue_size: (String) -> Integer
|
|
47
|
+
|
|
48
|
+
# : (String) -> bool
|
|
49
|
+
def clear_queue: (String) -> bool
|
|
50
|
+
|
|
51
|
+
# : (String) -> Hash[String, untyped]?
|
|
52
|
+
def find_job: (String) -> Hash[String, untyped]?
|
|
53
|
+
|
|
54
|
+
# : (DSL, Numeric) -> bool
|
|
55
|
+
def reschedule_job: (DSL, Numeric) -> bool
|
|
56
|
+
|
|
57
|
+
# @note Test helpers
|
|
58
|
+
#
|
|
59
|
+
# : (String, untyped) -> void
|
|
60
|
+
def enqueue_test_job: (String, untyped) -> void
|
|
61
|
+
|
|
62
|
+
# @note Test helpers - stores job data for find_job
|
|
63
|
+
#
|
|
64
|
+
# : (String, Hash[String, untyped]) -> void
|
|
65
|
+
def store_job: (String, Hash[String, untyped]) -> void
|
|
66
|
+
|
|
67
|
+
# @note Test helpers
|
|
68
|
+
#
|
|
69
|
+
# : () -> void
|
|
70
|
+
def reset!: () -> void
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/queue_adapters/solid_queue_adapter.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
module QueueAdapters
|
|
5
|
+
# rubocop:disable Naming/PredicateMethod
|
|
6
|
+
class SolidQueueAdapter < Abstract
|
|
7
|
+
# @note
|
|
8
|
+
# - Registry scope: @semaphore_registry is process-scoped (shared across fibers/threads
|
|
9
|
+
# in the same process) and lives for the lifetime of the worker process. It is not
|
|
10
|
+
# serialized to persistent storage; semaphores are transient per worker instance.
|
|
11
|
+
# - Cleanup: The adapter relies on SolidQueue::Worker lifecycle hooks to clean up
|
|
12
|
+
# active semaphores when the worker stops. If a worker crashes, semaphores will
|
|
13
|
+
# leak until the underlying database records expire or are manually cleaned.
|
|
14
|
+
#
|
|
15
|
+
# : () -> void
|
|
16
|
+
def initialize: () -> void
|
|
17
|
+
|
|
18
|
+
# : () -> void
|
|
19
|
+
def initialize_adapter!: () -> void
|
|
20
|
+
|
|
21
|
+
# : () -> bool
|
|
22
|
+
def semaphore_available?: () -> bool
|
|
23
|
+
|
|
24
|
+
# @note
|
|
25
|
+
# - Thread safety: @semaphore_registry is a non-thread-safe Hash. In multi-threaded workers,
|
|
26
|
+
# concurrent calls to semaphore_wait or semaphore_signal may cause race conditions.
|
|
27
|
+
# Mitigation: SolidQueue workers typically run in single-threaded Fiber mode; verify
|
|
28
|
+
# worker configuration does not enable raw multithreading.
|
|
29
|
+
# - Double-wait behavior: If semaphore_wait is called twice for the same Semaphore
|
|
30
|
+
# (e.g., due to retry or requeue), the second call returns false and does not
|
|
31
|
+
# re-register the hook. This is a fail-fast contract: the semaphore is already
|
|
32
|
+
# being waited and will signal the registered hook.
|
|
33
|
+
#
|
|
34
|
+
# : (Semaphore) -> bool
|
|
35
|
+
def semaphore_wait: (Semaphore) -> bool
|
|
36
|
+
|
|
37
|
+
# @note
|
|
38
|
+
# - Lifecycle management: The adapter is responsible for removing the hook from
|
|
39
|
+
# SolidQueue::Worker.lifecycle_hooks[:stop] before calling signal. The hook must
|
|
40
|
+
# be deleted from the registry and the global lifecycle_hooks to prevent redundant
|
|
41
|
+
# signal calls after the semaphore has already been signaled.
|
|
42
|
+
# - Hook deletion order: The hook is deleted before calling signal to ensure the
|
|
43
|
+
# hook lambda is no longer invoked even if the signal triggers a worker stop.
|
|
44
|
+
#
|
|
45
|
+
# : (Semaphore) -> bool
|
|
46
|
+
def semaphore_signal: (Semaphore) -> bool
|
|
47
|
+
|
|
48
|
+
# : (Array[String]) -> Hash[String, untyped]
|
|
49
|
+
def fetch_job_statuses: (Array[String]) -> Hash[String, untyped]
|
|
50
|
+
|
|
51
|
+
# : (untyped) -> Symbol
|
|
52
|
+
def job_status: (untyped) -> Symbol
|
|
53
|
+
|
|
54
|
+
# : () -> bool
|
|
55
|
+
def supports_concurrency_limits?: () -> bool
|
|
56
|
+
|
|
57
|
+
# : (String) -> bool
|
|
58
|
+
def pause_queue: (String) -> bool
|
|
59
|
+
|
|
60
|
+
# : (String) -> bool
|
|
61
|
+
def resume_queue: (String) -> bool
|
|
62
|
+
|
|
63
|
+
# : (String) -> bool
|
|
64
|
+
def queue_paused?: (String) -> bool
|
|
65
|
+
|
|
66
|
+
# : () -> Array[String]
|
|
67
|
+
def paused_queues: () -> Array[String]
|
|
68
|
+
|
|
69
|
+
# : (String) -> Integer?
|
|
70
|
+
def queue_latency: (String) -> Integer?
|
|
71
|
+
|
|
72
|
+
# : (String) -> Integer
|
|
73
|
+
def queue_size: (String) -> Integer
|
|
74
|
+
|
|
75
|
+
# : (String) -> bool
|
|
76
|
+
def clear_queue: (String) -> bool
|
|
77
|
+
|
|
78
|
+
# @note
|
|
79
|
+
# - SolidQueue stores the full ActiveJob serialization in job.arguments
|
|
80
|
+
# - We need to extract the actual arguments array for consistency
|
|
81
|
+
#
|
|
82
|
+
# : (String) -> Hash[String, untyped]?
|
|
83
|
+
def find_job: (String) -> Hash[String, untyped]?
|
|
84
|
+
|
|
85
|
+
# : (DSL, Numeric) -> bool
|
|
86
|
+
def reschedule_job: (DSL, Numeric) -> bool
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
attr_reader semaphore_registry: Hash[Object, ^(SolidQueue::Worker) -> void]
|
|
91
|
+
|
|
92
|
+
# : (SolidQueue::Job, DSL, Numeric) -> bool
|
|
93
|
+
def reschedule_solid_queue_job: (SolidQueue::Job, DSL, Numeric) -> bool
|
|
94
|
+
|
|
95
|
+
# @rbs module-self SolidQueue::ClaimedExecution
|
|
96
|
+
module ClaimedExecutionPatch : SolidQueue::ClaimedExecution
|
|
97
|
+
private
|
|
98
|
+
|
|
99
|
+
# : () -> SolidQueue::ClaimedExecution
|
|
100
|
+
def finished: () -> SolidQueue::ClaimedExecution
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
module SchedulingPatch
|
|
104
|
+
private
|
|
105
|
+
|
|
106
|
+
# : () -> Hash[Symbol, Hash[Symbol, untyped]]
|
|
107
|
+
def recurring_tasks_config: () -> Hash[Symbol, Hash[Symbol, untyped]]
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Generated from lib/job_workflow/runner.rb with RBS::Inline
|
|
2
|
+
|
|
3
|
+
module JobWorkflow
|
|
4
|
+
class Runner
|
|
5
|
+
# rubocop:disable Metrics/ClassLength
|
|
6
|
+
attr_reader context: Context
|
|
7
|
+
|
|
8
|
+
# : (context: Context) -> void
|
|
9
|
+
def initialize: (context: Context) -> void
|
|
10
|
+
|
|
11
|
+
# : () -> void
|
|
12
|
+
def run: () -> void
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
attr_reader job: DSL
|
|
17
|
+
|
|
18
|
+
# : () -> Workflow
|
|
19
|
+
def workflow: () -> Workflow
|
|
20
|
+
|
|
21
|
+
# : () -> Array[Task]
|
|
22
|
+
def tasks: () -> Array[Task]
|
|
23
|
+
|
|
24
|
+
# : () -> HookRegistry
|
|
25
|
+
def hooks: () -> HookRegistry
|
|
26
|
+
|
|
27
|
+
# : () -> void
|
|
28
|
+
def run_workflow: () -> void
|
|
29
|
+
|
|
30
|
+
# : (Task) -> bool
|
|
31
|
+
def skip_task?: (Task) -> bool
|
|
32
|
+
|
|
33
|
+
# : (Task) -> void
|
|
34
|
+
def run_task: (Task) -> void
|
|
35
|
+
|
|
36
|
+
# : (Task, Context) -> void
|
|
37
|
+
def run_each_task: (Task, Context) -> void
|
|
38
|
+
|
|
39
|
+
# : (Task, Context) { () -> void } -> void
|
|
40
|
+
def run_hooks: (Task, Context) { () -> void } -> void
|
|
41
|
+
|
|
42
|
+
# : (Task, Context, Array[Hook]) { () -> void } -> void
|
|
43
|
+
def run_around_hooks: (Task, Context, Array[Hook]) { () -> void } -> void
|
|
44
|
+
|
|
45
|
+
# : (Task, Context, StandardError) -> void
|
|
46
|
+
def run_error_hooks: (Task, Context, StandardError) -> void
|
|
47
|
+
|
|
48
|
+
# : (Task) -> void
|
|
49
|
+
def enqueue_task: (Task) -> void
|
|
50
|
+
|
|
51
|
+
# : (Task, ActiveJob::Continuation::Step) -> void
|
|
52
|
+
def wait_for_dependent_tasks: (Task, ActiveJob::Continuation::Step) -> void
|
|
53
|
+
|
|
54
|
+
# : (Task, Task, ActiveJob::Continuation::Step) -> void
|
|
55
|
+
def poll_until_complete_or_reschedule: (Task, Task, ActiveJob::Continuation::Step) -> void
|
|
56
|
+
|
|
57
|
+
# : (Task, TaskDependencyWait, Hash[Symbol, untyped]) -> void
|
|
58
|
+
def reschedule_if_needed: (Task, TaskDependencyWait, Hash[Symbol, untyped]) -> void
|
|
59
|
+
|
|
60
|
+
# : (ctx: Context, task: Task, each_index: Integer, data: untyped) -> void
|
|
61
|
+
def add_task_output: (ctx: Context, task: Task, each_index: Integer, data: untyped) -> void
|
|
62
|
+
|
|
63
|
+
# : (Task) -> void
|
|
64
|
+
def update_task_outputs: (Task) -> void
|
|
65
|
+
end
|
|
66
|
+
end
|