temporalio 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.yardopts +2 -0
- data/Cargo.lock +659 -370
- data/Cargo.toml +2 -2
- data/Gemfile +3 -3
- data/README.md +589 -47
- data/Rakefile +10 -296
- data/ext/Cargo.toml +1 -0
- data/lib/temporalio/activity/complete_async_error.rb +1 -1
- data/lib/temporalio/activity/context.rb +5 -2
- data/lib/temporalio/activity/definition.rb +163 -65
- data/lib/temporalio/activity/info.rb +22 -21
- data/lib/temporalio/activity.rb +2 -59
- data/lib/temporalio/api/activity/v1/message.rb +25 -0
- data/lib/temporalio/api/cloud/account/v1/message.rb +28 -0
- data/lib/temporalio/api/cloud/cloudservice/v1/request_response.rb +34 -1
- data/lib/temporalio/api/cloud/cloudservice/v1/service.rb +1 -1
- data/lib/temporalio/api/cloud/identity/v1/message.rb +6 -1
- data/lib/temporalio/api/cloud/namespace/v1/message.rb +8 -1
- data/lib/temporalio/api/cloud/nexus/v1/message.rb +31 -0
- data/lib/temporalio/api/cloud/operation/v1/message.rb +2 -1
- data/lib/temporalio/api/cloud/region/v1/message.rb +2 -1
- data/lib/temporalio/api/cloud/resource/v1/message.rb +23 -0
- data/lib/temporalio/api/cloud/sink/v1/message.rb +24 -0
- data/lib/temporalio/api/cloud/usage/v1/message.rb +31 -0
- data/lib/temporalio/api/common/v1/message.rb +7 -1
- data/lib/temporalio/api/enums/v1/event_type.rb +1 -1
- data/lib/temporalio/api/enums/v1/failed_cause.rb +1 -1
- data/lib/temporalio/api/enums/v1/reset.rb +1 -1
- data/lib/temporalio/api/history/v1/message.rb +1 -1
- data/lib/temporalio/api/nexus/v1/message.rb +2 -2
- data/lib/temporalio/api/operatorservice/v1/service.rb +1 -1
- data/lib/temporalio/api/payload_visitor.rb +1513 -0
- data/lib/temporalio/api/schedule/v1/message.rb +2 -1
- data/lib/temporalio/api/testservice/v1/request_response.rb +31 -0
- data/lib/temporalio/api/testservice/v1/service.rb +23 -0
- data/lib/temporalio/api/workflow/v1/message.rb +1 -1
- data/lib/temporalio/api/workflowservice/v1/request_response.rb +17 -2
- data/lib/temporalio/api/workflowservice/v1/service.rb +1 -1
- data/lib/temporalio/api.rb +1 -0
- data/lib/temporalio/cancellation.rb +34 -14
- data/lib/temporalio/client/async_activity_handle.rb +12 -37
- data/lib/temporalio/client/connection/cloud_service.rb +309 -231
- data/lib/temporalio/client/connection/operator_service.rb +36 -84
- data/lib/temporalio/client/connection/service.rb +6 -5
- data/lib/temporalio/client/connection/test_service.rb +111 -0
- data/lib/temporalio/client/connection/workflow_service.rb +264 -441
- data/lib/temporalio/client/connection.rb +90 -44
- data/lib/temporalio/client/interceptor.rb +160 -60
- data/lib/temporalio/client/schedule.rb +967 -0
- data/lib/temporalio/client/schedule_handle.rb +126 -0
- data/lib/temporalio/client/workflow_execution.rb +7 -10
- data/lib/temporalio/client/workflow_handle.rb +38 -95
- data/lib/temporalio/client/workflow_update_handle.rb +3 -5
- data/lib/temporalio/client.rb +122 -42
- data/lib/temporalio/common_enums.rb +17 -0
- data/lib/temporalio/converters/data_converter.rb +4 -7
- data/lib/temporalio/converters/failure_converter.rb +5 -3
- data/lib/temporalio/converters/payload_converter/composite.rb +4 -0
- data/lib/temporalio/converters/payload_converter.rb +6 -8
- data/lib/temporalio/converters/raw_value.rb +20 -0
- data/lib/temporalio/error/failure.rb +1 -1
- data/lib/temporalio/error.rb +10 -2
- data/lib/temporalio/internal/bridge/api/core_interface.rb +5 -1
- data/lib/temporalio/internal/bridge/api/nexus/nexus.rb +33 -0
- data/lib/temporalio/internal/bridge/api/workflow_activation/workflow_activation.rb +5 -1
- data/lib/temporalio/internal/bridge/api/workflow_commands/workflow_commands.rb +4 -1
- data/lib/temporalio/internal/bridge/client.rb +11 -6
- data/lib/temporalio/internal/bridge/testing.rb +20 -0
- data/lib/temporalio/internal/bridge/worker.rb +2 -0
- data/lib/temporalio/internal/bridge.rb +1 -1
- data/lib/temporalio/internal/client/implementation.rb +245 -70
- data/lib/temporalio/internal/metric.rb +122 -0
- data/lib/temporalio/internal/proto_utils.rb +86 -7
- data/lib/temporalio/internal/worker/activity_worker.rb +52 -24
- data/lib/temporalio/internal/worker/multi_runner.rb +51 -7
- data/lib/temporalio/internal/worker/workflow_instance/child_workflow_handle.rb +54 -0
- data/lib/temporalio/internal/worker/workflow_instance/context.rb +329 -0
- data/lib/temporalio/internal/worker/workflow_instance/details.rb +44 -0
- data/lib/temporalio/internal/worker/workflow_instance/external_workflow_handle.rb +32 -0
- data/lib/temporalio/internal/worker/workflow_instance/externally_immutable_hash.rb +22 -0
- data/lib/temporalio/internal/worker/workflow_instance/handler_execution.rb +25 -0
- data/lib/temporalio/internal/worker/workflow_instance/handler_hash.rb +41 -0
- data/lib/temporalio/internal/worker/workflow_instance/illegal_call_tracer.rb +97 -0
- data/lib/temporalio/internal/worker/workflow_instance/inbound_implementation.rb +62 -0
- data/lib/temporalio/internal/worker/workflow_instance/outbound_implementation.rb +415 -0
- data/lib/temporalio/internal/worker/workflow_instance/replay_safe_logger.rb +37 -0
- data/lib/temporalio/internal/worker/workflow_instance/replay_safe_metric.rb +40 -0
- data/lib/temporalio/internal/worker/workflow_instance/scheduler.rb +163 -0
- data/lib/temporalio/internal/worker/workflow_instance.rb +730 -0
- data/lib/temporalio/internal/worker/workflow_worker.rb +196 -0
- data/lib/temporalio/metric.rb +109 -0
- data/lib/temporalio/retry_policy.rb +37 -14
- data/lib/temporalio/runtime.rb +118 -75
- data/lib/temporalio/search_attributes.rb +80 -37
- data/lib/temporalio/testing/activity_environment.rb +2 -2
- data/lib/temporalio/testing/workflow_environment.rb +251 -5
- data/lib/temporalio/version.rb +1 -1
- data/lib/temporalio/worker/activity_executor/thread_pool.rb +9 -217
- data/lib/temporalio/worker/activity_executor.rb +3 -3
- data/lib/temporalio/worker/interceptor.rb +340 -66
- data/lib/temporalio/worker/thread_pool.rb +237 -0
- data/lib/temporalio/worker/workflow_executor/thread_pool.rb +230 -0
- data/lib/temporalio/worker/workflow_executor.rb +26 -0
- data/lib/temporalio/worker.rb +201 -30
- data/lib/temporalio/workflow/activity_cancellation_type.rb +20 -0
- data/lib/temporalio/workflow/child_workflow_cancellation_type.rb +21 -0
- data/lib/temporalio/workflow/child_workflow_handle.rb +43 -0
- data/lib/temporalio/workflow/definition.rb +566 -0
- data/lib/temporalio/workflow/external_workflow_handle.rb +41 -0
- data/lib/temporalio/workflow/future.rb +151 -0
- data/lib/temporalio/workflow/handler_unfinished_policy.rb +13 -0
- data/lib/temporalio/workflow/info.rb +82 -0
- data/lib/temporalio/workflow/parent_close_policy.rb +19 -0
- data/lib/temporalio/workflow/update_info.rb +20 -0
- data/lib/temporalio/workflow.rb +523 -0
- data/lib/temporalio.rb +4 -0
- data/temporalio.gemspec +2 -2
- metadata +50 -8
@@ -0,0 +1,230 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'etc'
|
4
|
+
require 'temporalio/internal/bridge/api'
|
5
|
+
require 'temporalio/internal/proto_utils'
|
6
|
+
require 'temporalio/internal/worker/workflow_instance'
|
7
|
+
require 'temporalio/scoped_logger'
|
8
|
+
require 'temporalio/worker/thread_pool'
|
9
|
+
require 'temporalio/worker/workflow_executor'
|
10
|
+
require 'temporalio/workflow'
|
11
|
+
require 'temporalio/workflow/definition'
|
12
|
+
require 'timeout'
|
13
|
+
|
14
|
+
module Temporalio
|
15
|
+
class Worker
|
16
|
+
class WorkflowExecutor
|
17
|
+
# Thread pool implementation of {WorkflowExecutor}.
|
18
|
+
#
|
19
|
+
# Users should use {default} unless they have specific needs to change the thread pool or max threads.
|
20
|
+
class ThreadPool < WorkflowExecutor
|
21
|
+
# @return [ThreadPool] Default executor that lazily constructs an instance with default values.
|
22
|
+
def self.default
|
23
|
+
@default ||= ThreadPool.new
|
24
|
+
end
|
25
|
+
|
26
|
+
# Create a thread pool executor. Most users may prefer {default}.
|
27
|
+
#
|
28
|
+
# @param max_threads [Integer] Maximum number of threads to use concurrently.
|
29
|
+
# @param thread_pool [Worker::ThreadPool] Thread pool to use.
|
30
|
+
def initialize(max_threads: [4, Etc.nprocessors].max, thread_pool: Temporalio::Worker::ThreadPool.default) # rubocop:disable Lint/MissingSuper
|
31
|
+
@max_threads = max_threads
|
32
|
+
@thread_pool = thread_pool
|
33
|
+
@workers_mutex = Mutex.new
|
34
|
+
@workers = []
|
35
|
+
@workers_by_worker_state_and_run_id = {}
|
36
|
+
end
|
37
|
+
|
38
|
+
# @!visibility private
|
39
|
+
def _validate_worker(worker, worker_state)
|
40
|
+
# Do nothing
|
41
|
+
end
|
42
|
+
|
43
|
+
# @!visibility private
|
44
|
+
def _activate(activation, worker_state, &)
|
45
|
+
# Get applicable worker
|
46
|
+
worker = @workers_mutex.synchronize do
|
47
|
+
run_key = [worker_state, activation.run_id]
|
48
|
+
@workers_by_worker_state_and_run_id.fetch(run_key) do
|
49
|
+
# If not found, get a new one either by creating if not enough or find the one with the fewest.
|
50
|
+
new_worker = if @workers.size < @max_threads
|
51
|
+
created_worker = Worker.new(self)
|
52
|
+
@workers << Worker.new(self)
|
53
|
+
created_worker
|
54
|
+
else
|
55
|
+
@workers.min_by(&:workflow_count)
|
56
|
+
end
|
57
|
+
@workers_by_worker_state_and_run_id[run_key] = new_worker
|
58
|
+
new_worker.workflow_count += 1
|
59
|
+
new_worker
|
60
|
+
end
|
61
|
+
end
|
62
|
+
raise "No worker for run ID #{activation.run_id}" unless worker
|
63
|
+
|
64
|
+
# Enqueue activation
|
65
|
+
worker.enqueue_activation(activation, worker_state, &)
|
66
|
+
end
|
67
|
+
|
68
|
+
# @!visibility private
|
69
|
+
def _thread_pool
|
70
|
+
@thread_pool
|
71
|
+
end
|
72
|
+
|
73
|
+
# @!visibility private
|
74
|
+
def _remove_workflow(worker_state, run_id)
|
75
|
+
@workers_mutex.synchronize do
|
76
|
+
worker = @workers_by_worker_state_and_run_id.delete([worker_state, run_id])
|
77
|
+
if worker
|
78
|
+
worker.workflow_count -= 1
|
79
|
+
# Remove worker from array if done. The array should be small enough that the delete being O(N) is not
|
80
|
+
# worth using a set or a map.
|
81
|
+
if worker.workflow_count.zero?
|
82
|
+
@workers.delete(worker)
|
83
|
+
worker.shutdown
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# @!visibility private
|
90
|
+
class Worker
|
91
|
+
LOG_ACTIVATIONS = false
|
92
|
+
|
93
|
+
attr_accessor :workflow_count
|
94
|
+
|
95
|
+
def initialize(executor)
|
96
|
+
@executor = executor
|
97
|
+
@workflow_count = 0
|
98
|
+
@queue = Queue.new
|
99
|
+
executor._thread_pool.execute { run }
|
100
|
+
end
|
101
|
+
|
102
|
+
# @!visibility private
|
103
|
+
def enqueue_activation(activation, worker_state, &completion_block)
|
104
|
+
@queue << [:activate, activation, worker_state, completion_block]
|
105
|
+
end
|
106
|
+
|
107
|
+
# @!visibility private
|
108
|
+
def shutdown
|
109
|
+
@queue << [:shutdown]
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
def run
|
115
|
+
loop do
|
116
|
+
work = @queue.pop
|
117
|
+
if work.is_a?(Exception)
|
118
|
+
Warning.warn("Failed activation: #{work}")
|
119
|
+
elsif work.is_a?(Array)
|
120
|
+
case work.first
|
121
|
+
when :shutdown
|
122
|
+
return
|
123
|
+
when :activate
|
124
|
+
activate(work[1], work[2], &work[3])
|
125
|
+
end
|
126
|
+
end
|
127
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
128
|
+
Warning.warn("Unexpected failure during run: #{e.full_message}")
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def activate(activation, worker_state, &)
|
133
|
+
worker_state.logger.debug("Received workflow activation: #{activation}") if LOG_ACTIVATIONS
|
134
|
+
|
135
|
+
# Check whether it has eviction
|
136
|
+
cache_remove_job = activation.jobs.find { |j| !j.remove_from_cache.nil? }&.remove_from_cache
|
137
|
+
|
138
|
+
# If it's eviction only, just evict inline and do nothing else
|
139
|
+
if cache_remove_job && activation.jobs.size == 1
|
140
|
+
evict(worker_state, activation.run_id)
|
141
|
+
worker_state.logger.debug('Sending empty workflow completion') if LOG_ACTIVATIONS
|
142
|
+
yield Internal::Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new(
|
143
|
+
run_id: activation.run_id,
|
144
|
+
successful: Internal::Bridge::Api::WorkflowCompletion::Success.new
|
145
|
+
)
|
146
|
+
return
|
147
|
+
end
|
148
|
+
|
149
|
+
completion = Timeout.timeout(
|
150
|
+
worker_state.deadlock_timeout,
|
151
|
+
DeadlockError,
|
152
|
+
# TODO(cretz): Document that this affects all running workflows on this worker
|
153
|
+
# and maybe test to see how that is mitigated
|
154
|
+
"[TMPRL1101] Potential deadlock detected: workflow didn't yield " \
|
155
|
+
"within #{worker_state.deadlock_timeout} second(s)."
|
156
|
+
) do
|
157
|
+
# Get or create workflow
|
158
|
+
instance = worker_state.get_or_create_running_workflow(activation.run_id) do
|
159
|
+
create_instance(activation, worker_state)
|
160
|
+
end
|
161
|
+
|
162
|
+
# Activate. We expect most errors in here to have been captured inside.
|
163
|
+
instance.activate(activation)
|
164
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
165
|
+
worker_state.logger.error("Failed activation on workflow run ID: #{activation.run_id}")
|
166
|
+
worker_state.logger.error(e)
|
167
|
+
Internal::Worker::WorkflowInstance.new_completion_with_failure(
|
168
|
+
run_id: activation.run_id,
|
169
|
+
error: e,
|
170
|
+
failure_converter: worker_state.data_converter.failure_converter,
|
171
|
+
payload_converter: worker_state.data_converter.payload_converter
|
172
|
+
)
|
173
|
+
end
|
174
|
+
|
175
|
+
# Go ahead and evict if there is an eviction job
|
176
|
+
evict(worker_state, activation.run_id) if cache_remove_job
|
177
|
+
|
178
|
+
# Complete the activation
|
179
|
+
worker_state.logger.debug("Sending workflow completion: #{completion}") if LOG_ACTIVATIONS
|
180
|
+
yield completion
|
181
|
+
end
|
182
|
+
|
183
|
+
def create_instance(initial_activation, worker_state)
|
184
|
+
# Extract start job
|
185
|
+
init_job = initial_activation.jobs.find { |j| !j.initialize_workflow.nil? }&.initialize_workflow
|
186
|
+
raise 'Missing initialize job in initial activation' unless init_job
|
187
|
+
|
188
|
+
# Obtain definition
|
189
|
+
definition = worker_state.workflow_definitions[init_job.workflow_type] ||
|
190
|
+
worker_state.workflow_definitions[nil]
|
191
|
+
unless definition
|
192
|
+
raise Error::ApplicationError.new(
|
193
|
+
"Workflow type #{init_job.workflow_type} is not registered on this worker, available workflows: " +
|
194
|
+
worker_state.workflow_definitions.keys.compact.sort.join(', '),
|
195
|
+
type: 'NotFoundError'
|
196
|
+
)
|
197
|
+
end
|
198
|
+
|
199
|
+
Internal::Worker::WorkflowInstance.new(
|
200
|
+
Internal::Worker::WorkflowInstance::Details.new(
|
201
|
+
namespace: worker_state.namespace,
|
202
|
+
task_queue: worker_state.task_queue,
|
203
|
+
definition:,
|
204
|
+
initial_activation:,
|
205
|
+
logger: worker_state.logger,
|
206
|
+
metric_meter: worker_state.metric_meter,
|
207
|
+
payload_converter: worker_state.data_converter.payload_converter,
|
208
|
+
failure_converter: worker_state.data_converter.failure_converter,
|
209
|
+
interceptors: worker_state.workflow_interceptors,
|
210
|
+
disable_eager_activity_execution: worker_state.disable_eager_activity_execution,
|
211
|
+
illegal_calls: worker_state.illegal_calls,
|
212
|
+
workflow_failure_exception_types: worker_state.workflow_failure_exception_types
|
213
|
+
)
|
214
|
+
)
|
215
|
+
end
|
216
|
+
|
217
|
+
def evict(worker_state, run_id)
|
218
|
+
worker_state.evict_running_workflow(run_id)
|
219
|
+
@executor._remove_workflow(worker_state, run_id)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
|
223
|
+
private_constant :Worker
|
224
|
+
|
225
|
+
# Error raised when a processing a workflow task takes more than the expected amount of time.
|
226
|
+
class DeadlockError < Exception; end # rubocop:disable Lint/InheritException
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'temporalio/worker/workflow_executor/thread_pool'
|
4
|
+
|
5
|
+
module Temporalio
|
6
|
+
class Worker
|
7
|
+
# Workflow executor that executes workflow tasks. Unlike {ActivityExecutor}, this class is not meant for user
|
8
|
+
# implementation. The only implementation that is currently accepted is {WorkflowExecutor::ThreadPool}.
|
9
|
+
class WorkflowExecutor
|
10
|
+
# @!visibility private
|
11
|
+
def initialize
|
12
|
+
raise 'Cannot create custom executors'
|
13
|
+
end
|
14
|
+
|
15
|
+
# @!visibility private
|
16
|
+
def _validate_worker(worker, worker_state)
|
17
|
+
raise NotImplementedError
|
18
|
+
end
|
19
|
+
|
20
|
+
# @!visibility private
|
21
|
+
def _activate(activation, worker_state, &)
|
22
|
+
raise NotImplementedError
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/temporalio/worker.rb
CHANGED
@@ -8,9 +8,13 @@ require 'temporalio/internal/bridge'
|
|
8
8
|
require 'temporalio/internal/bridge/worker'
|
9
9
|
require 'temporalio/internal/worker/activity_worker'
|
10
10
|
require 'temporalio/internal/worker/multi_runner'
|
11
|
+
require 'temporalio/internal/worker/workflow_instance'
|
12
|
+
require 'temporalio/internal/worker/workflow_worker'
|
11
13
|
require 'temporalio/worker/activity_executor'
|
12
14
|
require 'temporalio/worker/interceptor'
|
15
|
+
require 'temporalio/worker/thread_pool'
|
13
16
|
require 'temporalio/worker/tuner'
|
17
|
+
require 'temporalio/worker/workflow_executor'
|
14
18
|
|
15
19
|
module Temporalio
|
16
20
|
# Worker for processing activities and workflows on a task queue.
|
@@ -19,13 +23,14 @@ module Temporalio
|
|
19
23
|
# {run_all} is used for a collection of workers. These can wait until a block is complete or a {Cancellation} is
|
20
24
|
# canceled.
|
21
25
|
class Worker
|
22
|
-
|
23
|
-
Options = Struct.new(
|
26
|
+
Options = Data.define(
|
24
27
|
:client,
|
25
28
|
:task_queue,
|
26
29
|
:activities,
|
27
|
-
:
|
30
|
+
:workflows,
|
28
31
|
:tuner,
|
32
|
+
:activity_executors,
|
33
|
+
:workflow_executor,
|
29
34
|
:interceptors,
|
30
35
|
:build_id,
|
31
36
|
:identity,
|
@@ -42,9 +47,16 @@ module Temporalio
|
|
42
47
|
:max_task_queue_activities_per_second,
|
43
48
|
:graceful_shutdown_period,
|
44
49
|
:use_worker_versioning,
|
45
|
-
|
50
|
+
:disable_eager_activity_execution,
|
51
|
+
:illegal_workflow_calls,
|
52
|
+
:workflow_failure_exception_types,
|
53
|
+
:workflow_payload_codec_thread_pool,
|
54
|
+
:debug_mode
|
46
55
|
)
|
47
56
|
|
57
|
+
# Options as returned from {options} for `**to_h` splat use in {initialize}. See {initialize} for details.
|
58
|
+
class Options; end # rubocop:disable Lint/EmptyClass
|
59
|
+
|
48
60
|
# @return [String] Memoized default build ID. This default value is built as a checksum of all of the loaded Ruby
|
49
61
|
# source files in `$LOADED_FEATURES`. Users may prefer to set the build ID to a better representation of the
|
50
62
|
# source.
|
@@ -108,7 +120,7 @@ module Temporalio
|
|
108
120
|
runner.apply_thread_or_fiber_block(&block)
|
109
121
|
|
110
122
|
# Reuse first worker logger
|
111
|
-
logger = workers.first&.options&.logger or raise #
|
123
|
+
logger = workers.first&.options&.logger or raise # Never nil
|
112
124
|
|
113
125
|
# On cancel, initiate shutdown
|
114
126
|
cancellation.add_cancel_callback do
|
@@ -121,16 +133,34 @@ module Temporalio
|
|
121
133
|
block_result = nil
|
122
134
|
loop do
|
123
135
|
event = runner.next_event
|
136
|
+
# TODO(cretz): Consider improving performance instead of this case statement
|
124
137
|
case event
|
125
138
|
when Internal::Worker::MultiRunner::Event::PollSuccess
|
126
139
|
# Successful poll
|
127
|
-
event.worker._on_poll_bytes(event.worker_type, event.bytes)
|
140
|
+
event.worker._on_poll_bytes(runner, event.worker_type, event.bytes)
|
128
141
|
when Internal::Worker::MultiRunner::Event::PollFailure
|
129
142
|
# Poll failure, this causes shutdown of all workers
|
130
|
-
logger.error('Poll failure (beginning worker shutdown if not
|
143
|
+
logger.error('Poll failure (beginning worker shutdown if not already occurring)')
|
131
144
|
logger.error(event.error)
|
132
145
|
first_error ||= event.error
|
133
146
|
runner.initiate_shutdown
|
147
|
+
when Internal::Worker::MultiRunner::Event::WorkflowActivationDecoded
|
148
|
+
# Came back from a codec as decoded
|
149
|
+
event.workflow_worker.handle_activation(runner:, activation: event.activation, decoded: true)
|
150
|
+
when Internal::Worker::MultiRunner::Event::WorkflowActivationComplete
|
151
|
+
# An activation is complete
|
152
|
+
event.workflow_worker.handle_activation_complete(
|
153
|
+
runner:,
|
154
|
+
activation_completion: event.activation_completion,
|
155
|
+
encoded: event.encoded,
|
156
|
+
completion_complete_queue: event.completion_complete_queue
|
157
|
+
)
|
158
|
+
when Internal::Worker::MultiRunner::Event::WorkflowActivationCompletionComplete
|
159
|
+
# Completion complete, only need to log error if it occurs here
|
160
|
+
if event.error
|
161
|
+
logger.error("Activation completion failed to record on run ID #{event.run_id}")
|
162
|
+
logger.error(event.error)
|
163
|
+
end
|
134
164
|
when Internal::Worker::MultiRunner::Event::PollerShutDown
|
135
165
|
# Individual poller shut down. Nothing to do here until we support
|
136
166
|
# worker status or something.
|
@@ -186,6 +216,9 @@ module Temporalio
|
|
186
216
|
end
|
187
217
|
end
|
188
218
|
|
219
|
+
# Notify each worker we're done with it
|
220
|
+
workers.each(&:_on_shutdown_complete)
|
221
|
+
|
189
222
|
# If there was an shutdown-causing error, we raise that
|
190
223
|
if !first_error.nil?
|
191
224
|
raise first_error
|
@@ -194,6 +227,53 @@ module Temporalio
|
|
194
227
|
end
|
195
228
|
end
|
196
229
|
|
230
|
+
# @return [Hash<String, [:all, Array<Symbol>]>] Default, immutable set illegal calls used for the
|
231
|
+
# `illegal_workflow_calls` worker option. See the documentation of that option for more details.
|
232
|
+
def self.default_illegal_workflow_calls
|
233
|
+
@default_illegal_workflow_calls ||= begin
|
234
|
+
hash = {
|
235
|
+
'BasicSocket' => :all,
|
236
|
+
'Date' => %i[initialize today],
|
237
|
+
'DateTime' => %i[initialize now],
|
238
|
+
'Dir' => :all,
|
239
|
+
'Fiber' => [:set_scheduler],
|
240
|
+
'File' => :all,
|
241
|
+
'FileTest' => :all,
|
242
|
+
'FileUtils' => :all,
|
243
|
+
'Find' => :all,
|
244
|
+
'GC' => :all,
|
245
|
+
'IO' => [
|
246
|
+
:read
|
247
|
+
# Intentionally leaving out write so puts will work. We don't want to add heavy logic replacing stdout or
|
248
|
+
# trying to derive whether it's file vs stdout write.
|
249
|
+
#:write
|
250
|
+
],
|
251
|
+
'Kernel' => %i[abort at_exit autoload autoload? eval exec exit fork gets load open rand readline readlines
|
252
|
+
spawn srand system test trap],
|
253
|
+
'Net::HTTP' => :all,
|
254
|
+
'Pathname' => :all,
|
255
|
+
# TODO(cretz): Investigate why clock_gettime called from Timeout thread affects this code at all. Stack trace
|
256
|
+
# test executing activities inside a timeout will fail if clock_gettime is blocked.
|
257
|
+
'Process' => %i[abort argv0 daemon detach exec exit exit! fork kill setpriority setproctitle setrlimit setsid
|
258
|
+
spawn times wait wait2 waitall warmup],
|
259
|
+
# TODO(cretz): Allow Ractor.current since exception formatting in error_highlight references it
|
260
|
+
# 'Ractor' => :all,
|
261
|
+
'Random::Base' => [:initialize],
|
262
|
+
'Resolv' => :all,
|
263
|
+
'SecureRandom' => :all,
|
264
|
+
'Signal' => :all,
|
265
|
+
'Socket' => :all,
|
266
|
+
'Tempfile' => :all,
|
267
|
+
'Thread' => %i[abort_on_exception= exit fork handle_interrupt ignore_deadlock= kill new pass
|
268
|
+
pending_interrupt? report_on_exception= start stop initialize join name= priority= raise run
|
269
|
+
terminate thread_variable_set wakeup],
|
270
|
+
'Time' => %i[initialize now]
|
271
|
+
} #: Hash[String, :all | Array[Symbol]]
|
272
|
+
hash.each_value(&:freeze)
|
273
|
+
hash.freeze
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
197
277
|
# @return [Options] Frozen options for this client which has the same attributes as {initialize}.
|
198
278
|
attr_reader :options
|
199
279
|
|
@@ -201,20 +281,26 @@ module Temporalio
|
|
201
281
|
#
|
202
282
|
# @param client [Client] Client for this worker.
|
203
283
|
# @param task_queue [String] Task queue for this worker.
|
204
|
-
# @param activities [Array<Activity, Class<Activity>, Activity::Definition>]
|
205
|
-
#
|
284
|
+
# @param activities [Array<Activity::Definition, Class<Activity::Definition>, Activity::Definition::Info>]
|
285
|
+
# Activities for this worker.
|
286
|
+
# @param workflows [Array<Class<Workflow::Definition>>] Workflows for this worker.
|
206
287
|
# @param tuner [Tuner] Tuner that controls the amount of concurrent activities/workflows that run at a time.
|
207
|
-
# @param
|
208
|
-
#
|
288
|
+
# @param activity_executors [Hash<Symbol, Worker::ActivityExecutor>] Executors that activities can run within.
|
289
|
+
# @param workflow_executor [WorkflowExecutor] Workflow executor that workflow tasks run within. This must be a
|
290
|
+
# {WorkflowExecutor::ThreadPool} currently.
|
291
|
+
# @param interceptors [Array<Interceptor::Activity, Interceptor::Workflow>] Interceptors specific to this worker.
|
292
|
+
# Note, interceptors set on the client that include the {Interceptor::Activity} or {Interceptor::Workflow} module
|
293
|
+
# are automatically included here, so no need to specify them again.
|
209
294
|
# @param build_id [String] Unique identifier for the current runtime. This is best set as a unique value
|
210
295
|
# representing all code and should change only when code does. This can be something like a git commit hash. If
|
211
296
|
# unset, default is hash of known Ruby code.
|
212
297
|
# @param identity [String, nil] Override the identity for this worker. If unset, client identity is used.
|
298
|
+
# @param logger [Logger] Logger to override client logger with. Default is the client logger.
|
213
299
|
# @param max_cached_workflows [Integer] Number of workflows held in cache for use by sticky task queue. If set to 0,
|
214
300
|
# workflow caching and sticky queuing are disabled.
|
215
301
|
# @param max_concurrent_workflow_task_polls [Integer] Maximum number of concurrent poll workflow task requests we
|
216
302
|
# will perform at a time on this worker's task queue.
|
217
|
-
# @param nonsticky_to_sticky_poll_ratio [Float] `max_concurrent_workflow_task_polls
|
303
|
+
# @param nonsticky_to_sticky_poll_ratio [Float] `max_concurrent_workflow_task_polls` * this number = the number of
|
218
304
|
# max pollers that will be allowed for the nonsticky queue when sticky tasks are enabled. If both defaults are
|
219
305
|
# used, the sticky queue will allow 4 max pollers while the nonsticky queue will allow one. The minimum for either
|
220
306
|
# poller is 1, so if `max_concurrent_workflow_task_polls` is 1 and sticky queues are enabled, there will be 2
|
@@ -239,12 +325,35 @@ module Temporalio
|
|
239
325
|
# @param use_worker_versioning [Boolean] If true, the `build_id` argument must be specified, and this worker opts
|
240
326
|
# into the worker versioning feature. This ensures it only receives workflow tasks for workflows which it claims
|
241
327
|
# to be compatible with. For more information, see https://docs.temporal.io/workers#worker-versioning.
|
328
|
+
# @param disable_eager_activity_execution [Boolean] If true, disables eager activity execution. Eager activity
|
329
|
+
# execution is an optimization on some servers that sends activities back to the same worker as the calling
|
330
|
+
# workflow if they can run there. This should be set to true for `max_task_queue_activities_per_second` to work
|
331
|
+
# and in a future version of this API may be implied as such (i.e. this setting will be ignored if that setting is
|
332
|
+
# set).
|
333
|
+
# @param illegal_workflow_calls [Hash<String, [:all, Array<Symbol>]>] Set of illegal workflow calls that are
|
334
|
+
# considered unsafe/non-deterministic and will raise if seen. The key of the hash is the fully qualified string
|
335
|
+
# class name (no leading `::`). The value is either `:all` which means any use of the class, or an array of
|
336
|
+
# symbols for methods on the class that cannot be used. The methods refer to either instance or class methods,
|
337
|
+
# there is no way to differentiate at this time.
|
338
|
+
# @param workflow_failure_exception_types [Array<Class<Exception>>] Workflow failure exception types. This is the
|
339
|
+
# set of exception types that, if a workflow-thrown exception extends, will cause the workflow/update to fail
|
340
|
+
# instead of suspending the workflow via task failure. These are applied in addition to the
|
341
|
+
# `workflow_failure_exception_type` on the workflow definition class itself. If {::Exception} is set, it
|
342
|
+
# effectively will fail a workflow/update in all user exception cases.
|
343
|
+
# @param workflow_payload_codec_thread_pool [ThreadPool, nil] Thread pool to run payload codec encode/decode within.
|
344
|
+
# This is required if a payload codec exists and the worker is not fiber based. Codecs can potentially block
|
345
|
+
# execution which is why they need to be run in the background.
|
346
|
+
# @param debug_mode [Boolean] If true, deadlock detection is disabled. Deadlock detection will fail workflow tasks
|
347
|
+
# if they block the thread for too long. This defaults to true if the `TEMPORAL_DEBUG` environment variable is
|
348
|
+
# `true` or `1`.
|
242
349
|
def initialize(
|
243
350
|
client:,
|
244
351
|
task_queue:,
|
245
352
|
activities: [],
|
246
|
-
|
353
|
+
workflows: [],
|
247
354
|
tuner: Tuner.create_fixed,
|
355
|
+
activity_executors: ActivityExecutor.defaults,
|
356
|
+
workflow_executor: WorkflowExecutor::ThreadPool.default,
|
248
357
|
interceptors: [],
|
249
358
|
build_id: Worker.default_build_id,
|
250
359
|
identity: nil,
|
@@ -260,17 +369,23 @@ module Temporalio
|
|
260
369
|
max_activities_per_second: nil,
|
261
370
|
max_task_queue_activities_per_second: nil,
|
262
371
|
graceful_shutdown_period: 0,
|
263
|
-
use_worker_versioning: false
|
372
|
+
use_worker_versioning: false,
|
373
|
+
disable_eager_activity_execution: false,
|
374
|
+
illegal_workflow_calls: Worker.default_illegal_workflow_calls,
|
375
|
+
workflow_failure_exception_types: [],
|
376
|
+
workflow_payload_codec_thread_pool: nil,
|
377
|
+
debug_mode: %w[true 1].include?(ENV['TEMPORAL_DEBUG'].to_s.downcase)
|
264
378
|
)
|
265
|
-
|
266
|
-
raise ArgumentError, 'Must have at least one activity' if activities.empty?
|
379
|
+
raise ArgumentError, 'Must have at least one activity or workflow' if activities.empty? && workflows.empty?
|
267
380
|
|
268
381
|
@options = Options.new(
|
269
382
|
client:,
|
270
383
|
task_queue:,
|
271
384
|
activities:,
|
272
|
-
|
385
|
+
workflows:,
|
273
386
|
tuner:,
|
387
|
+
activity_executors:,
|
388
|
+
workflow_executor:,
|
274
389
|
interceptors:,
|
275
390
|
build_id:,
|
276
391
|
identity:,
|
@@ -286,15 +401,36 @@ module Temporalio
|
|
286
401
|
max_activities_per_second:,
|
287
402
|
max_task_queue_activities_per_second:,
|
288
403
|
graceful_shutdown_period:,
|
289
|
-
use_worker_versioning
|
404
|
+
use_worker_versioning:,
|
405
|
+
disable_eager_activity_execution:,
|
406
|
+
illegal_workflow_calls:,
|
407
|
+
workflow_failure_exception_types:,
|
408
|
+
workflow_payload_codec_thread_pool:,
|
409
|
+
debug_mode:
|
290
410
|
).freeze
|
291
411
|
|
412
|
+
# Preload workflow definitions and some workflow settings for the bridge
|
413
|
+
workflow_definitions = Internal::Worker::WorkflowWorker.workflow_definitions(workflows)
|
414
|
+
nondeterminism_as_workflow_fail = workflow_failure_exception_types.any? do |t|
|
415
|
+
t.is_a?(Class) && t >= Workflow::NondeterminismError
|
416
|
+
end
|
417
|
+
nondeterminism_as_workflow_fail_for_types = workflow_definitions.values.map do |defn|
|
418
|
+
next unless defn.failure_exception_types.any? { |t| t.is_a?(Class) && t >= Workflow::NondeterminismError }
|
419
|
+
|
420
|
+
# If they tried to do this on a dynamic workflow and haven't already set worker-level option, warn
|
421
|
+
unless defn.name || nondeterminism_as_workflow_fail
|
422
|
+
warn('Note, dynamic workflows cannot trap non-determinism errors, so worker-level ' \
|
423
|
+
'workflow_failure_exception_types should be set to capture that if that is the intention')
|
424
|
+
end
|
425
|
+
defn.name
|
426
|
+
end.compact
|
427
|
+
|
292
428
|
# Create the bridge worker
|
293
429
|
@bridge_worker = Internal::Bridge::Worker.new(
|
294
430
|
client.connection._core_client,
|
295
431
|
Internal::Bridge::Worker::Options.new(
|
296
432
|
activity: !activities.empty?,
|
297
|
-
workflow:
|
433
|
+
workflow: !workflows.empty?,
|
298
434
|
namespace: client.namespace,
|
299
435
|
task_queue:,
|
300
436
|
tuner: Internal::Bridge::Worker::TunerOptions.new(
|
@@ -308,26 +444,42 @@ module Temporalio
|
|
308
444
|
max_concurrent_workflow_task_polls:,
|
309
445
|
nonsticky_to_sticky_poll_ratio:,
|
310
446
|
max_concurrent_activity_task_polls:,
|
311
|
-
|
447
|
+
# For shutdown to work properly, we must disable remote activities
|
448
|
+
# ourselves if there are no activities
|
449
|
+
no_remote_activities: no_remote_activities || activities.empty?,
|
312
450
|
sticky_queue_schedule_to_start_timeout:,
|
313
451
|
max_heartbeat_throttle_interval:,
|
314
452
|
default_heartbeat_throttle_interval:,
|
315
453
|
max_worker_activities_per_second: max_activities_per_second,
|
316
454
|
max_task_queue_activities_per_second:,
|
317
455
|
graceful_shutdown_period:,
|
318
|
-
use_worker_versioning
|
456
|
+
use_worker_versioning:,
|
457
|
+
nondeterminism_as_workflow_fail:,
|
458
|
+
nondeterminism_as_workflow_fail_for_types:
|
319
459
|
)
|
320
460
|
)
|
321
461
|
|
322
462
|
# Collect interceptors from client and params
|
323
|
-
@
|
463
|
+
@activity_interceptors = (client.options.interceptors + interceptors).select do |i|
|
464
|
+
i.is_a?(Interceptor::Activity)
|
465
|
+
end
|
466
|
+
@workflow_interceptors = (client.options.interceptors + interceptors).select do |i|
|
467
|
+
i.is_a?(Interceptor::Workflow)
|
468
|
+
end
|
324
469
|
|
325
470
|
# Cancellation for the whole worker
|
326
471
|
@worker_shutdown_cancellation = Cancellation.new
|
327
472
|
|
328
473
|
# Create workers
|
329
|
-
|
330
|
-
|
474
|
+
unless activities.empty?
|
475
|
+
@activity_worker = Internal::Worker::ActivityWorker.new(worker: self,
|
476
|
+
bridge_worker: @bridge_worker)
|
477
|
+
end
|
478
|
+
unless workflows.empty?
|
479
|
+
@workflow_worker = Internal::Worker::WorkflowWorker.new(worker: self,
|
480
|
+
bridge_worker: @bridge_worker,
|
481
|
+
workflow_definitions:)
|
482
|
+
end
|
331
483
|
|
332
484
|
# Validate worker
|
333
485
|
@bridge_worker.validate
|
@@ -387,16 +539,35 @@ module Temporalio
|
|
387
539
|
end
|
388
540
|
|
389
541
|
# @!visibility private
|
390
|
-
def
|
391
|
-
@
|
542
|
+
def _activity_interceptors
|
543
|
+
@activity_interceptors
|
392
544
|
end
|
393
545
|
|
394
546
|
# @!visibility private
|
395
|
-
def
|
396
|
-
|
397
|
-
|
547
|
+
def _workflow_interceptors
|
548
|
+
@workflow_interceptors
|
549
|
+
end
|
550
|
+
|
551
|
+
# @!visibility private
|
552
|
+
def _on_poll_bytes(runner, worker_type, bytes)
|
553
|
+
case worker_type
|
554
|
+
when :activity
|
555
|
+
@activity_worker.handle_task(Internal::Bridge::Api::ActivityTask::ActivityTask.decode(bytes))
|
556
|
+
when :workflow
|
557
|
+
@workflow_worker.handle_activation(
|
558
|
+
runner:,
|
559
|
+
activation: Internal::Bridge::Api::WorkflowActivation::WorkflowActivation.decode(bytes),
|
560
|
+
decoded: false
|
561
|
+
)
|
562
|
+
else
|
563
|
+
raise "Unrecognized worker type #{worker_type}"
|
564
|
+
end
|
565
|
+
end
|
398
566
|
|
399
|
-
|
567
|
+
# @!visibility private
|
568
|
+
def _on_shutdown_complete
|
569
|
+
@workflow_worker&.on_shutdown_complete
|
570
|
+
@workflow_worker = nil
|
400
571
|
end
|
401
572
|
|
402
573
|
private
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'temporalio/internal/bridge/api'
|
4
|
+
|
5
|
+
module Temporalio
|
6
|
+
module Workflow
|
7
|
+
# Cancellation types for activities.
|
8
|
+
module ActivityCancellationType
|
9
|
+
# Initiate a cancellation request and immediately report cancellation to the workflow.
|
10
|
+
TRY_CANCEL = Internal::Bridge::Api::WorkflowCommands::ActivityCancellationType::TRY_CANCEL
|
11
|
+
# Wait for activity cancellation completion. Note that activity must heartbeat to receive a cancellation
|
12
|
+
# notification. This can block the cancellation for a long time if activity doesn't heartbeat or chooses to ignore
|
13
|
+
# the cancellation request.
|
14
|
+
WAIT_CANCELLATION_COMPLETED =
|
15
|
+
Internal::Bridge::Api::WorkflowCommands::ActivityCancellationType::WAIT_CANCELLATION_COMPLETED
|
16
|
+
# Do not request cancellation of the activity and immediately report cancellation to the workflow.
|
17
|
+
ABANDON = Internal::Bridge::Api::WorkflowCommands::ActivityCancellationType::ABANDON
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|