temporal-ruby 0.0.1.pre.pre1 → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +2 -0
  3. data/README.md +23 -3
  4. data/lib/gen/temporal/api/command/v1/message_pb.rb +1 -1
  5. data/lib/gen/temporal/api/enums/v1/common_pb.rb +7 -0
  6. data/lib/gen/temporal/api/errordetails/v1/message_pb.rb +5 -6
  7. data/lib/gen/temporal/api/version/v1/message_pb.rb +19 -8
  8. data/lib/gen/temporal/api/workflowservice/v1/request_response_pb.rb +19 -8
  9. data/lib/gen/temporal/api/workflowservice/v1/service_services_pb.rb +0 -3
  10. data/lib/temporal.rb +104 -0
  11. data/lib/temporal/activity/context.rb +5 -1
  12. data/lib/temporal/activity/poller.rb +26 -9
  13. data/lib/temporal/activity/task_processor.rb +33 -20
  14. data/lib/temporal/client/converter/base.rb +35 -0
  15. data/lib/temporal/client/converter/composite.rb +49 -0
  16. data/lib/temporal/client/converter/payload/bytes.rb +30 -0
  17. data/lib/temporal/client/converter/payload/json.rb +28 -0
  18. data/lib/temporal/client/converter/payload/nil.rb +27 -0
  19. data/lib/temporal/client/grpc_client.rb +102 -27
  20. data/lib/temporal/client/retryer.rb +49 -0
  21. data/lib/temporal/client/serializer.rb +2 -0
  22. data/lib/temporal/client/serializer/cancel_timer.rb +2 -2
  23. data/lib/temporal/client/serializer/complete_workflow.rb +6 -4
  24. data/lib/temporal/client/serializer/continue_as_new.rb +37 -0
  25. data/lib/temporal/client/serializer/fail_workflow.rb +2 -2
  26. data/lib/temporal/client/serializer/failure.rb +4 -2
  27. data/lib/temporal/client/serializer/record_marker.rb +6 -4
  28. data/lib/temporal/client/serializer/request_activity_cancellation.rb +2 -2
  29. data/lib/temporal/client/serializer/retry_policy.rb +24 -0
  30. data/lib/temporal/client/serializer/schedule_activity.rb +8 -20
  31. data/lib/temporal/client/serializer/start_child_workflow.rb +9 -20
  32. data/lib/temporal/client/serializer/start_timer.rb +2 -2
  33. data/lib/temporal/concerns/payloads.rb +51 -0
  34. data/lib/temporal/configuration.rb +31 -4
  35. data/lib/temporal/error_handler.rb +11 -0
  36. data/lib/temporal/errors.rb +24 -0
  37. data/lib/temporal/execution_options.rb +9 -1
  38. data/lib/temporal/json.rb +3 -1
  39. data/lib/temporal/logger.rb +17 -0
  40. data/lib/temporal/metadata.rb +11 -3
  41. data/lib/temporal/metadata/activity.rb +15 -2
  42. data/lib/temporal/metadata/workflow.rb +8 -0
  43. data/lib/temporal/metadata/workflow_task.rb +11 -0
  44. data/lib/temporal/retry_policy.rb +6 -9
  45. data/lib/temporal/saga/concern.rb +1 -1
  46. data/lib/temporal/testing.rb +1 -0
  47. data/lib/temporal/testing/future_registry.rb +1 -1
  48. data/lib/temporal/testing/local_activity_context.rb +1 -1
  49. data/lib/temporal/testing/local_workflow_context.rb +38 -14
  50. data/lib/temporal/testing/scheduled_workflows.rb +75 -0
  51. data/lib/temporal/testing/temporal_override.rb +35 -7
  52. data/lib/temporal/testing/workflow_override.rb +6 -1
  53. data/lib/temporal/version.rb +1 -1
  54. data/lib/temporal/worker.rb +28 -10
  55. data/lib/temporal/workflow.rb +8 -2
  56. data/lib/temporal/workflow/command.rb +3 -0
  57. data/lib/temporal/workflow/context.rb +40 -5
  58. data/lib/temporal/workflow/errors.rb +39 -0
  59. data/lib/temporal/workflow/executor.rb +1 -1
  60. data/lib/temporal/workflow/future.rb +18 -6
  61. data/lib/temporal/workflow/history/event.rb +1 -3
  62. data/lib/temporal/workflow/history/event_target.rb +4 -0
  63. data/lib/temporal/workflow/history/window.rb +1 -1
  64. data/lib/temporal/workflow/poller.rb +41 -13
  65. data/lib/temporal/workflow/replay_aware_logger.rb +4 -4
  66. data/lib/temporal/workflow/state_manager.rb +33 -52
  67. data/lib/temporal/workflow/task_processor.rb +41 -11
  68. metadata +21 -9
  69. data/lib/temporal/client/serializer/payload.rb +0 -25
@@ -69,6 +69,10 @@ module Temporal
69
69
  def hash
70
70
  [id, type].hash
71
71
  end
72
+
73
+ def to_s
74
+ "#{type} (#{id})"
75
+ end
72
76
  end
73
77
  end
74
78
  end
@@ -28,7 +28,7 @@ module Temporal
28
28
  @local_time = nil
29
29
  when 'WORKFLOW_TASK_COMPLETED'
30
30
  @replay = true
31
- when 'WORKFLOW_TASK_SCHEDULED', 'WORKFLOW_TASK_FAILED'
31
+ when 'WORKFLOW_TASK_SCHEDULED'
32
32
  # no-op
33
33
  else
34
34
  events << event
@@ -1,16 +1,23 @@
1
1
  require 'temporal/client'
2
+ require 'temporal/thread_pool'
2
3
  require 'temporal/middleware/chain'
3
4
  require 'temporal/workflow/task_processor'
5
+ require 'temporal/error_handler'
4
6
 
5
7
  module Temporal
6
8
  class Workflow
7
9
  class Poller
8
- def initialize(namespace, task_queue, workflow_lookup, middleware = [])
10
+ DEFAULT_OPTIONS = {
11
+ thread_pool_size: 10
12
+ }.freeze
13
+
14
+ def initialize(namespace, task_queue, workflow_lookup, middleware = [], options = {})
9
15
  @namespace = namespace
10
16
  @task_queue = task_queue
11
17
  @workflow_lookup = workflow_lookup
12
18
  @middleware = middleware
13
19
  @shutting_down = false
20
+ @options = DEFAULT_OPTIONS.merge(options)
14
21
  end
15
22
 
16
23
  def start
@@ -18,50 +25,71 @@ module Temporal
18
25
  @thread = Thread.new(&method(:poll_loop))
19
26
  end
20
27
 
21
- def stop
28
+ def stop_polling
22
29
  @shutting_down = true
23
- Thread.new { Temporal.logger.info('Shutting down a workflow poller') }.join
30
+ Temporal.logger.info('Shutting down a workflow poller')
31
+ end
32
+
33
+ def cancel_pending_requests
34
+ client.cancel_polling_request
24
35
  end
25
36
 
26
37
  def wait
27
- @thread.join
38
+ thread.join
39
+ thread_pool.shutdown
28
40
  end
29
41
 
30
42
  private
31
43
 
32
- attr_reader :namespace, :task_queue, :client, :workflow_lookup, :middleware
44
+ attr_reader :namespace, :task_queue, :workflow_lookup, :middleware, :options, :thread
33
45
 
34
46
  def client
35
47
  @client ||= Temporal::Client.generate
36
48
  end
37
49
 
38
- def middleware_chain
39
- @middleware_chain ||= Middleware::Chain.new(middleware)
40
- end
41
-
42
50
  def shutting_down?
43
51
  @shutting_down
44
52
  end
45
53
 
46
54
  def poll_loop
47
- while !shutting_down? do
48
- Temporal.logger.debug("Polling worklow task queue (#{namespace} / #{task_queue})")
55
+ last_poll_time = Time.now
56
+ metrics_tags = { namespace: namespace, task_queue: task_queue }.freeze
57
+
58
+ loop do
59
+ thread_pool.wait_for_available_threads
60
+
61
+ return if shutting_down?
62
+
63
+ time_diff_ms = ((Time.now - last_poll_time) * 1000).round
64
+ Temporal.metrics.timing('workflow_poller.time_since_last_poll', time_diff_ms, metrics_tags)
65
+ Temporal.logger.debug("Polling Worklow task queue", { namespace: namespace, task_queue: task_queue })
49
66
 
50
67
  task = poll_for_task
51
- process(task) if task&.workflow_type
68
+ last_poll_time = Time.now
69
+ next unless task&.workflow_type
70
+
71
+ thread_pool.schedule { process(task) }
52
72
  end
53
73
  end
54
74
 
55
75
  def poll_for_task
56
76
  client.poll_workflow_task_queue(namespace: namespace, task_queue: task_queue)
57
77
  rescue StandardError => error
58
- Temporal.logger.error("Unable to poll workflow task queue: #{error.inspect}")
78
+ Temporal.logger.error("Unable to poll Workflow task queue", { namespace: namespace, task_queue: task_queue, error: error.inspect })
79
+ Temporal::ErrorHandler.handle(error)
80
+
59
81
  nil
60
82
  end
61
83
 
62
84
  def process(task)
85
+ middleware_chain = Middleware::Chain.new(middleware)
86
+
63
87
  TaskProcessor.new(task, namespace, workflow_lookup, client, middleware_chain).process
64
88
  end
89
+
90
+ def thread_pool
91
+ @thread_pool ||= ThreadPool.new(options[:thread_pool_size])
92
+ end
65
93
  end
66
94
  end
67
95
  end
@@ -11,17 +11,17 @@ module Temporal
11
11
  end
12
12
 
13
13
  SEVERITIES.each do |severity|
14
- define_method severity do |message|
14
+ define_method severity do |message, data = {}|
15
15
  return if replay?
16
16
 
17
- main_logger.public_send(severity, message)
17
+ main_logger.public_send(severity, message, data)
18
18
  end
19
19
  end
20
20
 
21
- def log(severity, message)
21
+ def log(severity, message, data = {})
22
22
  return if replay?
23
23
 
24
- main_logger.log(severity, message)
24
+ main_logger.log(severity, message, data)
25
25
  end
26
26
 
27
27
  private
@@ -1,13 +1,17 @@
1
- require 'temporal/json'
1
+ require 'set'
2
2
  require 'temporal/errors'
3
3
  require 'temporal/workflow/command'
4
4
  require 'temporal/workflow/command_state_machine'
5
5
  require 'temporal/workflow/history/event_target'
6
6
  require 'temporal/metadata'
7
+ require 'temporal/concerns/payloads'
8
+ require 'temporal/workflow/errors'
7
9
 
8
10
  module Temporal
9
11
  class Workflow
10
12
  class StateManager
13
+ include Concerns::Payloads
14
+
11
15
  SIDE_EFFECT_MARKER = 'SIDE_EFFECT'.freeze
12
16
  RELEASE_MARKER = 'RELEASE'.freeze
13
17
 
@@ -101,7 +105,7 @@ module Temporal
101
105
  dispatch(
102
106
  History::EventTarget.workflow,
103
107
  'started',
104
- parse_payload(event.attributes.input),
108
+ from_payloads(event.attributes.input),
105
109
  Metadata.generate(Metadata::WORKFLOW_TYPE, event.attributes)
106
110
  )
107
111
 
@@ -131,26 +135,26 @@ module Temporal
131
135
 
132
136
  when 'ACTIVITY_TASK_SCHEDULED'
133
137
  state_machine.schedule
134
- discard_command(event.originating_event_id)
138
+ discard_command(target)
135
139
 
136
140
  when 'ACTIVITY_TASK_STARTED'
137
141
  state_machine.start
138
142
 
139
143
  when 'ACTIVITY_TASK_COMPLETED'
140
144
  state_machine.complete
141
- dispatch(target, 'completed', parse_payload(event.attributes.result))
145
+ dispatch(target, 'completed', from_result_payloads(event.attributes.result))
142
146
 
143
147
  when 'ACTIVITY_TASK_FAILED'
144
148
  state_machine.fail
145
- dispatch(target, 'failed', parse_failure(event.attributes.failure, ActivityException))
149
+ dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure, ActivityException))
146
150
 
147
151
  when 'ACTIVITY_TASK_TIMED_OUT'
148
152
  state_machine.time_out
149
- dispatch(target, 'failed', parse_failure(event.attributes.failure))
153
+ dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
150
154
 
151
155
  when 'ACTIVITY_TASK_CANCEL_REQUESTED'
152
156
  state_machine.requested
153
- discard_command(event.originating_event_id)
157
+ discard_command(target)
154
158
 
155
159
  when 'REQUEST_CANCEL_ACTIVITY_TASK_FAILED'
156
160
  state_machine.fail
@@ -158,11 +162,11 @@ module Temporal
158
162
 
159
163
  when 'ACTIVITY_TASK_CANCELED'
160
164
  state_machine.cancel
161
- dispatch(target, 'failed', parse_failure(event.attributes.failure))
165
+ dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
162
166
 
163
167
  when 'TIMER_STARTED'
164
168
  state_machine.start
165
- discard_command(event.originating_event_id)
169
+ discard_command(target)
166
170
 
167
171
  when 'TIMER_FIRED'
168
172
  state_machine.complete
@@ -193,10 +197,10 @@ module Temporal
193
197
 
194
198
  when 'MARKER_RECORDED'
195
199
  state_machine.complete
196
- handle_marker(event.id, event.attributes.marker_name, parse_payload(event.attributes.details['data']))
200
+ handle_marker(event.id, event.attributes.marker_name, from_details_payloads(event.attributes.details['data']))
197
201
 
198
202
  when 'WORKFLOW_EXECUTION_SIGNALED'
199
- dispatch(target, 'signaled', event.attributes.signal_name, parse_payload(event.attributes.input))
203
+ dispatch(target, 'signaled', event.attributes.signal_name, from_signal_payloads(event.attributes.input))
200
204
 
201
205
  when 'WORKFLOW_EXECUTION_TERMINATED'
202
206
  # todo
@@ -206,30 +210,30 @@ module Temporal
206
210
 
207
211
  when 'START_CHILD_WORKFLOW_EXECUTION_INITIATED'
208
212
  state_machine.schedule
209
- discard_command(event.originating_event_id)
213
+ discard_command(target)
210
214
 
211
215
  when 'START_CHILD_WORKFLOW_EXECUTION_FAILED'
212
216
  state_machine.fail
213
- dispatch(target, 'failed', 'StandardError', parse_payload(event.attributes.cause))
217
+ dispatch(target, 'failed', 'StandardError', from_payloads(event.attributes.cause))
214
218
 
215
219
  when 'CHILD_WORKFLOW_EXECUTION_STARTED'
216
220
  state_machine.start
217
221
 
218
222
  when 'CHILD_WORKFLOW_EXECUTION_COMPLETED'
219
223
  state_machine.complete
220
- dispatch(target, 'completed', parse_payload(event.attributes.result))
224
+ dispatch(target, 'completed', from_result_payloads(event.attributes.result))
221
225
 
222
226
  when 'CHILD_WORKFLOW_EXECUTION_FAILED'
223
227
  state_machine.fail
224
- dispatch(target, 'failed', parse_failure(event.attributes.failure))
228
+ dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
225
229
 
226
230
  when 'CHILD_WORKFLOW_EXECUTION_CANCELED'
227
231
  state_machine.cancel
228
- dispatch(target, 'failed', parse_failure(event.attributes.failure))
232
+ dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
229
233
 
230
234
  when 'CHILD_WORKFLOW_EXECUTION_TIMED_OUT'
231
235
  state_machine.time_out
232
- dispatch(target, 'failed', parse_failure(event.attributes.failure))
236
+ dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
233
237
 
234
238
  when 'CHILD_WORKFLOW_EXECUTION_TERMINATED'
235
239
  # todo
@@ -277,8 +281,18 @@ module Temporal
277
281
  dispatcher.dispatch(target, name, attributes)
278
282
  end
279
283
 
280
- def discard_command(command_id)
281
- commands.delete_if { |(id, _)| id == command_id }
284
+ def discard_command(target)
285
+ # Pop the first command from the list, it is expected to match
286
+ existing_command_id, existing_command = commands.shift
287
+
288
+ if !existing_command_id
289
+ raise NonDeterministicWorkflowError, "A command #{target} was not scheduled upon replay"
290
+ end
291
+
292
+ existing_target = event_target_from(existing_command_id, existing_command)
293
+ if target != existing_target
294
+ raise NonDeterministicWorkflowError, "Unexpected command #{existing_target} (expected #{target})"
295
+ end
282
296
  end
283
297
 
284
298
  def handle_marker(id, type, details)
@@ -304,39 +318,6 @@ module Temporal
304
318
  end
305
319
  end
306
320
 
307
- def parse_payload(payload)
308
- return if payload.nil? || payload.payloads.empty?
309
-
310
- binary = payload.payloads.first.data
311
- JSON.deserialize(binary)
312
- end
313
-
314
- def parse_failure(failure, default_exception_class = StandardError)
315
- case failure.failure_info
316
- when :application_failure_info
317
- exception_class = safe_constantize(failure.application_failure_info.type)
318
- exception_class ||= default_exception_class
319
- details = parse_payload(failure.application_failure_info.details)
320
- backtrace = failure.stack_trace.split("\n")
321
-
322
- exception_class.new(details).tap do |exception|
323
- exception.set_backtrace(backtrace) if !backtrace.empty?
324
- end
325
- when :timeout_failure_info
326
- TimeoutError.new("Timeout type: #{failure.timeout_failure_info.timeout_type.to_s}")
327
- when :canceled_failure_info
328
- # TODO: Distinguish between different entity cancellations
329
- StandardError.new(parse_payload(failure.canceled_failure_info.details))
330
- else
331
- StandardError.new(failure.message)
332
- end
333
- end
334
-
335
- def safe_constantize(const)
336
- Object.const_get(const) if Object.const_defined?(const)
337
- rescue NameError
338
- nil
339
- end
340
321
  end
341
322
  end
342
323
  end
@@ -1,14 +1,18 @@
1
1
  require 'temporal/workflow/executor'
2
2
  require 'temporal/workflow/history'
3
3
  require 'temporal/metadata'
4
+ require 'temporal/error_handler'
4
5
  require 'temporal/errors'
5
6
 
6
7
  module Temporal
7
8
  class Workflow
8
9
  class TaskProcessor
10
+ MAX_FAILED_ATTEMPTS = 1
11
+
9
12
  def initialize(task, namespace, workflow_lookup, client, middleware_chain)
10
13
  @task = task
11
14
  @namespace = namespace
15
+ @metadata = Metadata.generate(Metadata::WORKFLOW_TASK_TYPE, task, namespace)
12
16
  @task_token = task.task_token
13
17
  @workflow_name = task.workflow_type.name
14
18
  @workflow_class = workflow_lookup.find(workflow_name)
@@ -19,37 +23,35 @@ module Temporal
19
23
  def process
20
24
  start_time = Time.now
21
25
 
22
- Temporal.logger.info("Processing a workflow task for #{workflow_name}")
26
+ Temporal.logger.debug("Processing Workflow task", metadata.to_h)
23
27
  Temporal.metrics.timing('workflow_task.queue_time', queue_time_ms, workflow: workflow_name)
24
28
 
25
29
  if !workflow_class
26
30
  raise Temporal::WorkflowNotRegistered, 'Workflow is not registered with this worker'
27
31
  end
28
32
 
29
- history = Workflow::History.new(task.history.events)
33
+ history = fetch_full_history
30
34
  # TODO: For sticky workflows we need to cache the Executor instance
31
35
  executor = Workflow::Executor.new(workflow_class, history)
32
- metadata = Metadata.generate(Metadata::WORKFLOW_TASK_TYPE, task, namespace)
33
36
 
34
37
  commands = middleware_chain.invoke(metadata) do
35
38
  executor.run
36
39
  end
37
40
 
38
41
  complete_task(commands)
39
- rescue Temporal::ClientError => error
40
- fail_task(error)
41
42
  rescue StandardError => error
42
- Temporal.logger.error("Workflow task for #{workflow_name} failed with: #{error.inspect}")
43
- Temporal.logger.debug(error.backtrace.join("\n"))
43
+ Temporal::ErrorHandler.handle(error, metadata: metadata)
44
+
45
+ fail_task(error)
44
46
  ensure
45
47
  time_diff_ms = ((Time.now - start_time) * 1000).round
46
48
  Temporal.metrics.timing('workflow_task.latency', time_diff_ms, workflow: workflow_name)
47
- Temporal.logger.debug("Workflow task processed in #{time_diff_ms}ms")
49
+ Temporal.logger.debug("Workflow task processed", metadata.to_h.merge(execution_time: time_diff_ms))
48
50
  end
49
51
 
50
52
  private
51
53
 
52
- attr_reader :task, :namespace, :task_token, :workflow_name, :workflow_class, :client, :middleware_chain
54
+ attr_reader :task, :namespace, :task_token, :workflow_name, :workflow_class, :client, :middleware_chain, :metadata
53
55
 
54
56
  def queue_time_ms
55
57
  scheduled = task.scheduled_time.to_f
@@ -57,21 +59,49 @@ module Temporal
57
59
  ((started - scheduled) * 1_000).round
58
60
  end
59
61
 
62
+ def fetch_full_history
63
+ events = task.history.events.to_a
64
+ next_page_token = task.next_page_token
65
+
66
+ while !next_page_token.empty? do
67
+ response = client.get_workflow_execution_history(
68
+ namespace: namespace,
69
+ workflow_id: task.workflow_execution.workflow_id,
70
+ run_id: task.workflow_execution.run_id,
71
+ next_page_token: next_page_token
72
+ )
73
+
74
+ events += response.history.events.to_a
75
+ next_page_token = response.next_page_token
76
+ end
77
+
78
+ Workflow::History.new(events)
79
+ end
80
+
60
81
  def complete_task(commands)
61
- Temporal.logger.info("Workflow task for #{workflow_name} completed")
82
+ Temporal.logger.info("Workflow task completed", metadata.to_h)
62
83
 
63
84
  client.respond_workflow_task_completed(task_token: task_token, commands: commands)
64
85
  end
65
86
 
66
87
  def fail_task(error)
67
- Temporal.logger.error("Workflow task for #{workflow_name} failed with: #{error.inspect}")
88
+ Temporal.logger.error("Workflow task failed", metadata.to_h.merge(error: error.inspect))
68
89
  Temporal.logger.debug(error.backtrace.join("\n"))
69
90
 
91
+ # Only fail the workflow task on the first attempt. Subsequent failures of the same workflow task
92
+ # should timeout. This is to avoid spinning on the failed workflow task as the service doesn't
93
+ # yet exponentially backoff on retries.
94
+ return if task.attempt > MAX_FAILED_ATTEMPTS
95
+
70
96
  client.respond_workflow_task_failed(
71
97
  task_token: task_token,
72
98
  cause: Temporal::Api::Enums::V1::WorkflowTaskFailedCause::WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND,
73
99
  exception: error
74
100
  )
101
+ rescue StandardError => error
102
+ Temporal.logger.error("Unable to fail Workflow task", metadata.to_h.merge(error: error.inspect))
103
+
104
+ Temporal::ErrorHandler.handle(error, metadata: metadata)
75
105
  end
76
106
  end
77
107
  end