temporalio 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (317) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +3 -0
  3. data/LICENSE +20 -0
  4. data/README.md +130 -0
  5. data/bridge/Cargo.lock +2865 -0
  6. data/bridge/Cargo.toml +26 -0
  7. data/bridge/sdk-core/ARCHITECTURE.md +76 -0
  8. data/bridge/sdk-core/Cargo.lock +2606 -0
  9. data/bridge/sdk-core/Cargo.toml +2 -0
  10. data/bridge/sdk-core/LICENSE.txt +23 -0
  11. data/bridge/sdk-core/README.md +107 -0
  12. data/bridge/sdk-core/arch_docs/diagrams/README.md +10 -0
  13. data/bridge/sdk-core/arch_docs/diagrams/sticky_queues.puml +40 -0
  14. data/bridge/sdk-core/arch_docs/diagrams/workflow_internals.svg +1 -0
  15. data/bridge/sdk-core/arch_docs/sticky_queues.md +51 -0
  16. data/bridge/sdk-core/bridge-ffi/Cargo.toml +24 -0
  17. data/bridge/sdk-core/bridge-ffi/LICENSE.txt +23 -0
  18. data/bridge/sdk-core/bridge-ffi/build.rs +25 -0
  19. data/bridge/sdk-core/bridge-ffi/include/sdk-core-bridge.h +249 -0
  20. data/bridge/sdk-core/bridge-ffi/src/lib.rs +825 -0
  21. data/bridge/sdk-core/bridge-ffi/src/wrappers.rs +211 -0
  22. data/bridge/sdk-core/client/Cargo.toml +40 -0
  23. data/bridge/sdk-core/client/LICENSE.txt +23 -0
  24. data/bridge/sdk-core/client/src/lib.rs +1294 -0
  25. data/bridge/sdk-core/client/src/metrics.rs +165 -0
  26. data/bridge/sdk-core/client/src/raw.rs +931 -0
  27. data/bridge/sdk-core/client/src/retry.rs +674 -0
  28. data/bridge/sdk-core/client/src/workflow_handle/mod.rs +185 -0
  29. data/bridge/sdk-core/core/Cargo.toml +116 -0
  30. data/bridge/sdk-core/core/LICENSE.txt +23 -0
  31. data/bridge/sdk-core/core/benches/workflow_replay.rs +73 -0
  32. data/bridge/sdk-core/core/src/abstractions.rs +166 -0
  33. data/bridge/sdk-core/core/src/core_tests/activity_tasks.rs +911 -0
  34. data/bridge/sdk-core/core/src/core_tests/child_workflows.rs +221 -0
  35. data/bridge/sdk-core/core/src/core_tests/determinism.rs +107 -0
  36. data/bridge/sdk-core/core/src/core_tests/local_activities.rs +515 -0
  37. data/bridge/sdk-core/core/src/core_tests/mod.rs +100 -0
  38. data/bridge/sdk-core/core/src/core_tests/queries.rs +736 -0
  39. data/bridge/sdk-core/core/src/core_tests/replay_flag.rs +65 -0
  40. data/bridge/sdk-core/core/src/core_tests/workers.rs +259 -0
  41. data/bridge/sdk-core/core/src/core_tests/workflow_cancels.rs +124 -0
  42. data/bridge/sdk-core/core/src/core_tests/workflow_tasks.rs +2070 -0
  43. data/bridge/sdk-core/core/src/ephemeral_server/mod.rs +515 -0
  44. data/bridge/sdk-core/core/src/lib.rs +175 -0
  45. data/bridge/sdk-core/core/src/log_export.rs +62 -0
  46. data/bridge/sdk-core/core/src/pollers/mod.rs +54 -0
  47. data/bridge/sdk-core/core/src/pollers/poll_buffer.rs +297 -0
  48. data/bridge/sdk-core/core/src/protosext/mod.rs +428 -0
  49. data/bridge/sdk-core/core/src/replay/mod.rs +71 -0
  50. data/bridge/sdk-core/core/src/retry_logic.rs +202 -0
  51. data/bridge/sdk-core/core/src/telemetry/metrics.rs +383 -0
  52. data/bridge/sdk-core/core/src/telemetry/mod.rs +412 -0
  53. data/bridge/sdk-core/core/src/telemetry/prometheus_server.rs +77 -0
  54. data/bridge/sdk-core/core/src/test_help/mod.rs +875 -0
  55. data/bridge/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +580 -0
  56. data/bridge/sdk-core/core/src/worker/activities/local_activities.rs +1042 -0
  57. data/bridge/sdk-core/core/src/worker/activities.rs +464 -0
  58. data/bridge/sdk-core/core/src/worker/client/mocks.rs +87 -0
  59. data/bridge/sdk-core/core/src/worker/client.rs +347 -0
  60. data/bridge/sdk-core/core/src/worker/mod.rs +566 -0
  61. data/bridge/sdk-core/core/src/worker/workflow/bridge.rs +37 -0
  62. data/bridge/sdk-core/core/src/worker/workflow/driven_workflow.rs +110 -0
  63. data/bridge/sdk-core/core/src/worker/workflow/history_update.rs +458 -0
  64. data/bridge/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs +911 -0
  65. data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_external_state_machine.rs +298 -0
  66. data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_workflow_state_machine.rs +171 -0
  67. data/bridge/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +860 -0
  68. data/bridge/sdk-core/core/src/worker/workflow/machines/complete_workflow_state_machine.rs +140 -0
  69. data/bridge/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs +161 -0
  70. data/bridge/sdk-core/core/src/worker/workflow/machines/fail_workflow_state_machine.rs +133 -0
  71. data/bridge/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs +1448 -0
  72. data/bridge/sdk-core/core/src/worker/workflow/machines/mod.rs +342 -0
  73. data/bridge/sdk-core/core/src/worker/workflow/machines/mutable_side_effect_state_machine.rs +127 -0
  74. data/bridge/sdk-core/core/src/worker/workflow/machines/patch_state_machine.rs +712 -0
  75. data/bridge/sdk-core/core/src/worker/workflow/machines/side_effect_state_machine.rs +71 -0
  76. data/bridge/sdk-core/core/src/worker/workflow/machines/signal_external_state_machine.rs +443 -0
  77. data/bridge/sdk-core/core/src/worker/workflow/machines/timer_state_machine.rs +439 -0
  78. data/bridge/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs +169 -0
  79. data/bridge/sdk-core/core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs +246 -0
  80. data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines/local_acts.rs +96 -0
  81. data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +1184 -0
  82. data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_task_state_machine.rs +277 -0
  83. data/bridge/sdk-core/core/src/worker/workflow/managed_run/managed_wf_test.rs +198 -0
  84. data/bridge/sdk-core/core/src/worker/workflow/managed_run.rs +647 -0
  85. data/bridge/sdk-core/core/src/worker/workflow/mod.rs +1143 -0
  86. data/bridge/sdk-core/core/src/worker/workflow/run_cache.rs +145 -0
  87. data/bridge/sdk-core/core/src/worker/workflow/wft_poller.rs +88 -0
  88. data/bridge/sdk-core/core/src/worker/workflow/workflow_stream.rs +940 -0
  89. data/bridge/sdk-core/core-api/Cargo.toml +31 -0
  90. data/bridge/sdk-core/core-api/LICENSE.txt +23 -0
  91. data/bridge/sdk-core/core-api/src/errors.rs +95 -0
  92. data/bridge/sdk-core/core-api/src/lib.rs +151 -0
  93. data/bridge/sdk-core/core-api/src/worker.rs +135 -0
  94. data/bridge/sdk-core/etc/deps.svg +187 -0
  95. data/bridge/sdk-core/etc/dynamic-config.yaml +2 -0
  96. data/bridge/sdk-core/etc/otel-collector-config.yaml +36 -0
  97. data/bridge/sdk-core/etc/prometheus.yaml +6 -0
  98. data/bridge/sdk-core/fsm/Cargo.toml +18 -0
  99. data/bridge/sdk-core/fsm/LICENSE.txt +23 -0
  100. data/bridge/sdk-core/fsm/README.md +3 -0
  101. data/bridge/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +27 -0
  102. data/bridge/sdk-core/fsm/rustfsm_procmacro/LICENSE.txt +23 -0
  103. data/bridge/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +647 -0
  104. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/progress.rs +8 -0
  105. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.rs +18 -0
  106. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.stderr +12 -0
  107. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dynamic_dest_pass.rs +41 -0
  108. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.rs +14 -0
  109. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.stderr +11 -0
  110. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_arg_pass.rs +32 -0
  111. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_pass.rs +31 -0
  112. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/medium_complex_pass.rs +46 -0
  113. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.rs +29 -0
  114. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +12 -0
  115. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/simple_pass.rs +32 -0
  116. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.rs +18 -0
  117. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.stderr +5 -0
  118. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.rs +11 -0
  119. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.stderr +5 -0
  120. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.rs +11 -0
  121. data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.stderr +5 -0
  122. data/bridge/sdk-core/fsm/rustfsm_trait/Cargo.toml +14 -0
  123. data/bridge/sdk-core/fsm/rustfsm_trait/LICENSE.txt +23 -0
  124. data/bridge/sdk-core/fsm/rustfsm_trait/src/lib.rs +249 -0
  125. data/bridge/sdk-core/fsm/src/lib.rs +2 -0
  126. data/bridge/sdk-core/histories/fail_wf_task.bin +0 -0
  127. data/bridge/sdk-core/histories/timer_workflow_history.bin +0 -0
  128. data/bridge/sdk-core/integ-with-otel.sh +7 -0
  129. data/bridge/sdk-core/protos/api_upstream/README.md +9 -0
  130. data/bridge/sdk-core/protos/api_upstream/api-linter.yaml +40 -0
  131. data/bridge/sdk-core/protos/api_upstream/buf.yaml +12 -0
  132. data/bridge/sdk-core/protos/api_upstream/dependencies/gogoproto/gogo.proto +141 -0
  133. data/bridge/sdk-core/protos/api_upstream/temporal/api/batch/v1/message.proto +86 -0
  134. data/bridge/sdk-core/protos/api_upstream/temporal/api/cluster/v1/message.proto +83 -0
  135. data/bridge/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto +259 -0
  136. data/bridge/sdk-core/protos/api_upstream/temporal/api/common/v1/message.proto +112 -0
  137. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +46 -0
  138. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/cluster.proto +40 -0
  139. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/command_type.proto +57 -0
  140. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/common.proto +55 -0
  141. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/event_type.proto +168 -0
  142. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto +97 -0
  143. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/namespace.proto +51 -0
  144. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/query.proto +50 -0
  145. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/reset.proto +41 -0
  146. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/schedule.proto +60 -0
  147. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/task_queue.proto +59 -0
  148. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/update.proto +51 -0
  149. data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/workflow.proto +122 -0
  150. data/bridge/sdk-core/protos/api_upstream/temporal/api/errordetails/v1/message.proto +108 -0
  151. data/bridge/sdk-core/protos/api_upstream/temporal/api/failure/v1/message.proto +114 -0
  152. data/bridge/sdk-core/protos/api_upstream/temporal/api/filter/v1/message.proto +56 -0
  153. data/bridge/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto +751 -0
  154. data/bridge/sdk-core/protos/api_upstream/temporal/api/namespace/v1/message.proto +97 -0
  155. data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +161 -0
  156. data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +99 -0
  157. data/bridge/sdk-core/protos/api_upstream/temporal/api/query/v1/message.proto +61 -0
  158. data/bridge/sdk-core/protos/api_upstream/temporal/api/replication/v1/message.proto +55 -0
  159. data/bridge/sdk-core/protos/api_upstream/temporal/api/schedule/v1/message.proto +300 -0
  160. data/bridge/sdk-core/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +108 -0
  161. data/bridge/sdk-core/protos/api_upstream/temporal/api/update/v1/message.proto +46 -0
  162. data/bridge/sdk-core/protos/api_upstream/temporal/api/version/v1/message.proto +59 -0
  163. data/bridge/sdk-core/protos/api_upstream/temporal/api/workflow/v1/message.proto +145 -0
  164. data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +1124 -0
  165. data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +401 -0
  166. data/bridge/sdk-core/protos/grpc/health/v1/health.proto +63 -0
  167. data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_result/activity_result.proto +78 -0
  168. data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_task/activity_task.proto +79 -0
  169. data/bridge/sdk-core/protos/local/temporal/sdk/core/bridge/bridge.proto +210 -0
  170. data/bridge/sdk-core/protos/local/temporal/sdk/core/child_workflow/child_workflow.proto +77 -0
  171. data/bridge/sdk-core/protos/local/temporal/sdk/core/common/common.proto +15 -0
  172. data/bridge/sdk-core/protos/local/temporal/sdk/core/core_interface.proto +30 -0
  173. data/bridge/sdk-core/protos/local/temporal/sdk/core/external_data/external_data.proto +30 -0
  174. data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +261 -0
  175. data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +297 -0
  176. data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto +29 -0
  177. data/bridge/sdk-core/protos/testsrv_upstream/api-linter.yaml +38 -0
  178. data/bridge/sdk-core/protos/testsrv_upstream/buf.yaml +13 -0
  179. data/bridge/sdk-core/protos/testsrv_upstream/dependencies/gogoproto/gogo.proto +141 -0
  180. data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/request_response.proto +63 -0
  181. data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/service.proto +90 -0
  182. data/bridge/sdk-core/rustfmt.toml +1 -0
  183. data/bridge/sdk-core/sdk/Cargo.toml +47 -0
  184. data/bridge/sdk-core/sdk/LICENSE.txt +23 -0
  185. data/bridge/sdk-core/sdk/src/activity_context.rs +230 -0
  186. data/bridge/sdk-core/sdk/src/app_data.rs +37 -0
  187. data/bridge/sdk-core/sdk/src/conversions.rs +8 -0
  188. data/bridge/sdk-core/sdk/src/interceptors.rs +17 -0
  189. data/bridge/sdk-core/sdk/src/lib.rs +792 -0
  190. data/bridge/sdk-core/sdk/src/payload_converter.rs +11 -0
  191. data/bridge/sdk-core/sdk/src/workflow_context/options.rs +295 -0
  192. data/bridge/sdk-core/sdk/src/workflow_context.rs +683 -0
  193. data/bridge/sdk-core/sdk/src/workflow_future.rs +503 -0
  194. data/bridge/sdk-core/sdk-core-protos/Cargo.toml +30 -0
  195. data/bridge/sdk-core/sdk-core-protos/LICENSE.txt +23 -0
  196. data/bridge/sdk-core/sdk-core-protos/build.rs +108 -0
  197. data/bridge/sdk-core/sdk-core-protos/src/constants.rs +7 -0
  198. data/bridge/sdk-core/sdk-core-protos/src/history_builder.rs +497 -0
  199. data/bridge/sdk-core/sdk-core-protos/src/history_info.rs +230 -0
  200. data/bridge/sdk-core/sdk-core-protos/src/lib.rs +1910 -0
  201. data/bridge/sdk-core/sdk-core-protos/src/task_token.rs +38 -0
  202. data/bridge/sdk-core/sdk-core-protos/src/utilities.rs +14 -0
  203. data/bridge/sdk-core/test-utils/Cargo.toml +35 -0
  204. data/bridge/sdk-core/test-utils/src/canned_histories.rs +1579 -0
  205. data/bridge/sdk-core/test-utils/src/histfetch.rs +28 -0
  206. data/bridge/sdk-core/test-utils/src/lib.rs +598 -0
  207. data/bridge/sdk-core/tests/integ_tests/client_tests.rs +36 -0
  208. data/bridge/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +128 -0
  209. data/bridge/sdk-core/tests/integ_tests/heartbeat_tests.rs +218 -0
  210. data/bridge/sdk-core/tests/integ_tests/polling_tests.rs +146 -0
  211. data/bridge/sdk-core/tests/integ_tests/queries_tests.rs +437 -0
  212. data/bridge/sdk-core/tests/integ_tests/visibility_tests.rs +93 -0
  213. data/bridge/sdk-core/tests/integ_tests/workflow_tests/activities.rs +878 -0
  214. data/bridge/sdk-core/tests/integ_tests/workflow_tests/appdata_propagation.rs +61 -0
  215. data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_external.rs +59 -0
  216. data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +58 -0
  217. data/bridge/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +50 -0
  218. data/bridge/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs +60 -0
  219. data/bridge/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +54 -0
  220. data/bridge/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +634 -0
  221. data/bridge/sdk-core/tests/integ_tests/workflow_tests/patches.rs +113 -0
  222. data/bridge/sdk-core/tests/integ_tests/workflow_tests/replay.rs +137 -0
  223. data/bridge/sdk-core/tests/integ_tests/workflow_tests/resets.rs +93 -0
  224. data/bridge/sdk-core/tests/integ_tests/workflow_tests/signals.rs +167 -0
  225. data/bridge/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +99 -0
  226. data/bridge/sdk-core/tests/integ_tests/workflow_tests/timers.rs +131 -0
  227. data/bridge/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +75 -0
  228. data/bridge/sdk-core/tests/integ_tests/workflow_tests.rs +587 -0
  229. data/bridge/sdk-core/tests/load_tests.rs +191 -0
  230. data/bridge/sdk-core/tests/main.rs +111 -0
  231. data/bridge/sdk-core/tests/runner.rs +93 -0
  232. data/bridge/src/connection.rs +167 -0
  233. data/bridge/src/lib.rs +180 -0
  234. data/bridge/src/runtime.rs +47 -0
  235. data/bridge/src/worker.rs +73 -0
  236. data/ext/Rakefile +9 -0
  237. data/lib/bridge.so +0 -0
  238. data/lib/gen/dependencies/gogoproto/gogo_pb.rb +14 -0
  239. data/lib/gen/temporal/api/batch/v1/message_pb.rb +48 -0
  240. data/lib/gen/temporal/api/cluster/v1/message_pb.rb +67 -0
  241. data/lib/gen/temporal/api/command/v1/message_pb.rb +166 -0
  242. data/lib/gen/temporal/api/common/v1/message_pb.rb +69 -0
  243. data/lib/gen/temporal/api/enums/v1/batch_operation_pb.rb +32 -0
  244. data/lib/gen/temporal/api/enums/v1/cluster_pb.rb +26 -0
  245. data/lib/gen/temporal/api/enums/v1/command_type_pb.rb +37 -0
  246. data/lib/gen/temporal/api/enums/v1/common_pb.rb +41 -0
  247. data/lib/gen/temporal/api/enums/v1/event_type_pb.rb +67 -0
  248. data/lib/gen/temporal/api/enums/v1/failed_cause_pb.rb +71 -0
  249. data/lib/gen/temporal/api/enums/v1/namespace_pb.rb +37 -0
  250. data/lib/gen/temporal/api/enums/v1/query_pb.rb +31 -0
  251. data/lib/gen/temporal/api/enums/v1/reset_pb.rb +24 -0
  252. data/lib/gen/temporal/api/enums/v1/schedule_pb.rb +28 -0
  253. data/lib/gen/temporal/api/enums/v1/task_queue_pb.rb +30 -0
  254. data/lib/gen/temporal/api/enums/v1/update_pb.rb +28 -0
  255. data/lib/gen/temporal/api/enums/v1/workflow_pb.rb +89 -0
  256. data/lib/gen/temporal/api/errordetails/v1/message_pb.rb +84 -0
  257. data/lib/gen/temporal/api/failure/v1/message_pb.rb +83 -0
  258. data/lib/gen/temporal/api/filter/v1/message_pb.rb +40 -0
  259. data/lib/gen/temporal/api/history/v1/message_pb.rb +489 -0
  260. data/lib/gen/temporal/api/namespace/v1/message_pb.rb +63 -0
  261. data/lib/gen/temporal/api/operatorservice/v1/request_response_pb.rb +125 -0
  262. data/lib/gen/temporal/api/operatorservice/v1/service_pb.rb +20 -0
  263. data/lib/gen/temporal/api/query/v1/message_pb.rb +38 -0
  264. data/lib/gen/temporal/api/replication/v1/message_pb.rb +37 -0
  265. data/lib/gen/temporal/api/schedule/v1/message_pb.rb +128 -0
  266. data/lib/gen/temporal/api/taskqueue/v1/message_pb.rb +73 -0
  267. data/lib/gen/temporal/api/update/v1/message_pb.rb +26 -0
  268. data/lib/gen/temporal/api/version/v1/message_pb.rb +41 -0
  269. data/lib/gen/temporal/api/workflow/v1/message_pb.rb +110 -0
  270. data/lib/gen/temporal/api/workflowservice/v1/request_response_pb.rb +771 -0
  271. data/lib/gen/temporal/api/workflowservice/v1/service_pb.rb +20 -0
  272. data/lib/gen/temporal/sdk/core/activity_result/activity_result_pb.rb +58 -0
  273. data/lib/gen/temporal/sdk/core/activity_task/activity_task_pb.rb +57 -0
  274. data/lib/gen/temporal/sdk/core/bridge/bridge_pb.rb +222 -0
  275. data/lib/gen/temporal/sdk/core/child_workflow/child_workflow_pb.rb +57 -0
  276. data/lib/gen/temporal/sdk/core/common/common_pb.rb +22 -0
  277. data/lib/gen/temporal/sdk/core/core_interface_pb.rb +34 -0
  278. data/lib/gen/temporal/sdk/core/external_data/external_data_pb.rb +27 -0
  279. data/lib/gen/temporal/sdk/core/workflow_activation/workflow_activation_pb.rb +164 -0
  280. data/lib/gen/temporal/sdk/core/workflow_commands/workflow_commands_pb.rb +192 -0
  281. data/lib/gen/temporal/sdk/core/workflow_completion/workflow_completion_pb.rb +34 -0
  282. data/lib/temporal/bridge.rb +14 -0
  283. data/lib/temporal/client/implementation.rb +339 -0
  284. data/lib/temporal/client/workflow_handle.rb +243 -0
  285. data/lib/temporal/client.rb +144 -0
  286. data/lib/temporal/connection.rb +736 -0
  287. data/lib/temporal/data_converter.rb +150 -0
  288. data/lib/temporal/error/failure.rb +194 -0
  289. data/lib/temporal/error/workflow_failure.rb +17 -0
  290. data/lib/temporal/errors.rb +22 -0
  291. data/lib/temporal/failure_converter/base.rb +26 -0
  292. data/lib/temporal/failure_converter/basic.rb +313 -0
  293. data/lib/temporal/failure_converter.rb +8 -0
  294. data/lib/temporal/interceptor/chain.rb +27 -0
  295. data/lib/temporal/interceptor/client.rb +102 -0
  296. data/lib/temporal/payload_codec/base.rb +32 -0
  297. data/lib/temporal/payload_converter/base.rb +24 -0
  298. data/lib/temporal/payload_converter/bytes.rb +26 -0
  299. data/lib/temporal/payload_converter/composite.rb +47 -0
  300. data/lib/temporal/payload_converter/encoding_base.rb +35 -0
  301. data/lib/temporal/payload_converter/json.rb +25 -0
  302. data/lib/temporal/payload_converter/nil.rb +25 -0
  303. data/lib/temporal/payload_converter.rb +14 -0
  304. data/lib/temporal/retry_policy.rb +82 -0
  305. data/lib/temporal/retry_state.rb +35 -0
  306. data/lib/temporal/runtime.rb +22 -0
  307. data/lib/temporal/timeout_type.rb +29 -0
  308. data/lib/temporal/version.rb +3 -0
  309. data/lib/temporal/workflow/execution_info.rb +54 -0
  310. data/lib/temporal/workflow/execution_status.rb +36 -0
  311. data/lib/temporal/workflow/id_reuse_policy.rb +36 -0
  312. data/lib/temporal/workflow/query_reject_condition.rb +33 -0
  313. data/lib/temporal.rb +8 -0
  314. data/lib/temporalio.rb +3 -0
  315. data/lib/thermite_patch.rb +23 -0
  316. data/temporalio.gemspec +41 -0
  317. metadata +583 -0
@@ -0,0 +1,2070 @@
1
+ use crate::{
2
+ advance_fut, job_assert,
3
+ replay::TestHistoryBuilder,
4
+ test_help::{
5
+ build_fake_worker, build_mock_pollers, build_multihist_mock_sg, canned_histories,
6
+ gen_assert_and_fail, gen_assert_and_reply, hist_to_poll_resp, mock_sdk, mock_sdk_cfg,
7
+ mock_worker, poll_and_reply, poll_and_reply_clears_outstanding_evicts, single_hist_mock_sg,
8
+ test_worker_cfg, FakeWfResponses, MockPollCfg, MocksHolder, ResponseType,
9
+ WorkflowCachingPolicy::{self, AfterEveryReply, NonSticky},
10
+ TEST_Q,
11
+ },
12
+ worker::client::mocks::{mock_manual_workflow_client, mock_workflow_client},
13
+ Worker,
14
+ };
15
+ use futures::{stream, FutureExt};
16
+ use rstest::{fixture, rstest};
17
+ use std::{
18
+ collections::{HashMap, VecDeque},
19
+ sync::{
20
+ atomic::{AtomicU64, Ordering},
21
+ Arc,
22
+ },
23
+ time::Duration,
24
+ };
25
+ use temporal_sdk::{ActivityOptions, CancellableFuture, WfContext};
26
+ use temporal_sdk_core_api::{errors::PollWfError, Worker as WorkerTrait};
27
+ use temporal_sdk_core_protos::{
28
+ coresdk::{
29
+ activity_result::{self as ar, activity_resolution, ActivityResolution},
30
+ workflow_activation::{
31
+ remove_from_cache::EvictionReason, workflow_activation_job, FireTimer, ResolveActivity,
32
+ StartWorkflow, UpdateRandomSeed, WorkflowActivationJob,
33
+ },
34
+ workflow_commands::{
35
+ ActivityCancellationType, CancelTimer, CompleteWorkflowExecution,
36
+ ContinueAsNewWorkflowExecution, FailWorkflowExecution, RequestCancelActivity,
37
+ ScheduleActivity,
38
+ },
39
+ workflow_completion::WorkflowActivationCompletion,
40
+ },
41
+ default_wes_attribs,
42
+ temporal::api::{
43
+ command::v1::command::Attributes,
44
+ common::v1::{Payload, RetryPolicy},
45
+ enums::v1::{EventType, WorkflowTaskFailedCause},
46
+ failure::v1::Failure,
47
+ history::v1::{history_event, TimerFiredEventAttributes},
48
+ workflowservice::v1::{
49
+ GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
50
+ },
51
+ },
52
+ DEFAULT_WORKFLOW_TYPE,
53
+ };
54
+ use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd};
55
+ use tokio::{
56
+ join,
57
+ sync::{Barrier, Semaphore},
58
+ };
59
+
60
+ #[fixture(hist_batches = &[])]
61
+ fn single_timer_setup(hist_batches: &'static [usize]) -> Worker {
62
+ let wfid = "fake_wf_id";
63
+
64
+ let t = canned_histories::single_timer("1");
65
+ build_fake_worker(wfid, t, hist_batches)
66
+ }
67
+
68
+ #[fixture(hist_batches = &[])]
69
+ fn single_activity_setup(hist_batches: &'static [usize]) -> Worker {
70
+ let wfid = "fake_wf_id";
71
+
72
+ let t = canned_histories::single_activity("fake_activity");
73
+ build_fake_worker(wfid, t, hist_batches)
74
+ }
75
+
76
+ #[fixture(hist_batches = &[])]
77
+ fn single_activity_failure_setup(hist_batches: &'static [usize]) -> Worker {
78
+ let wfid = "fake_wf_id";
79
+
80
+ let t = canned_histories::single_failed_activity("fake_activity");
81
+ build_fake_worker(wfid, t, hist_batches)
82
+ }
83
+
84
+ #[rstest]
85
+ #[case::incremental(single_timer_setup(&[1, 2]), NonSticky)]
86
+ #[case::replay(single_timer_setup(&[2]), NonSticky)]
87
+ #[case::incremental_evict(single_timer_setup(&[1, 2]), AfterEveryReply)]
88
+ #[case::replay_evict(single_timer_setup(&[2]), AfterEveryReply)]
89
+ #[tokio::test]
90
+ async fn single_timer(#[case] worker: Worker, #[case] evict: WorkflowCachingPolicy) {
91
+ poll_and_reply(
92
+ &worker,
93
+ evict,
94
+ &[
95
+ gen_assert_and_reply(
96
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
97
+ vec![start_timer_cmd(1, Duration::from_secs(1))],
98
+ ),
99
+ gen_assert_and_reply(
100
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
101
+ vec![CompleteWorkflowExecution { result: None }.into()],
102
+ ),
103
+ ],
104
+ )
105
+ .await;
106
+ }
107
+
108
+ #[rstest(worker,
109
+ case::incremental(single_activity_setup(&[1, 2])),
110
+ case::incremental_activity_failure(single_activity_failure_setup(&[1, 2])),
111
+ case::replay(single_activity_setup(&[2])),
112
+ case::replay_activity_failure(single_activity_failure_setup(&[2]))
113
+ )]
114
+ #[tokio::test]
115
+ async fn single_activity_completion(worker: Worker) {
116
+ poll_and_reply(
117
+ &worker,
118
+ NonSticky,
119
+ &[
120
+ gen_assert_and_reply(
121
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
122
+ vec![ScheduleActivity {
123
+ activity_id: "fake_activity".to_string(),
124
+ ..Default::default()
125
+ }
126
+ .into()],
127
+ ),
128
+ gen_assert_and_reply(
129
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(_)),
130
+ vec![CompleteWorkflowExecution { result: None }.into()],
131
+ ),
132
+ ],
133
+ )
134
+ .await;
135
+ }
136
+
137
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
138
+ #[tokio::test]
139
+ async fn parallel_timer_test_across_wf_bridge(hist_batches: &'static [usize]) {
140
+ let wfid = "fake_wf_id";
141
+ let timer_1_id = 1;
142
+ let timer_2_id = 2;
143
+
144
+ let t = canned_histories::parallel_timer(
145
+ timer_1_id.to_string().as_str(),
146
+ timer_2_id.to_string().as_str(),
147
+ );
148
+ let core = build_fake_worker(wfid, t, hist_batches);
149
+
150
+ poll_and_reply(
151
+ &core,
152
+ NonSticky,
153
+ &[
154
+ gen_assert_and_reply(
155
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
156
+ vec![
157
+ start_timer_cmd(timer_1_id, Duration::from_secs(1)),
158
+ start_timer_cmd(timer_2_id, Duration::from_secs(1)),
159
+ ],
160
+ ),
161
+ gen_assert_and_reply(
162
+ &|res| {
163
+ assert_matches!(
164
+ res.jobs.as_slice(),
165
+ [
166
+ WorkflowActivationJob {
167
+ variant: Some(workflow_activation_job::Variant::FireTimer(
168
+ FireTimer { seq: t1_id }
169
+ )),
170
+ },
171
+ WorkflowActivationJob {
172
+ variant: Some(workflow_activation_job::Variant::FireTimer(
173
+ FireTimer { seq: t2_id }
174
+ )),
175
+ }
176
+ ] => {
177
+ assert_eq!(t1_id, &timer_1_id);
178
+ assert_eq!(t2_id, &timer_2_id);
179
+ }
180
+ );
181
+ },
182
+ vec![CompleteWorkflowExecution { result: None }.into()],
183
+ ),
184
+ ],
185
+ )
186
+ .await;
187
+ }
188
+
189
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
190
+ #[tokio::test]
191
+ async fn timer_cancel(hist_batches: &'static [usize]) {
192
+ let wfid = "fake_wf_id";
193
+ let timer_id = 1;
194
+ let cancel_timer_id = 2;
195
+
196
+ let t = canned_histories::cancel_timer(
197
+ timer_id.to_string().as_str(),
198
+ cancel_timer_id.to_string().as_str(),
199
+ );
200
+ let core = build_fake_worker(wfid, t, hist_batches);
201
+
202
+ poll_and_reply(
203
+ &core,
204
+ NonSticky,
205
+ &[
206
+ gen_assert_and_reply(
207
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
208
+ vec![
209
+ start_timer_cmd(cancel_timer_id, Duration::from_secs(1)),
210
+ start_timer_cmd(timer_id, Duration::from_secs(1)),
211
+ ],
212
+ ),
213
+ gen_assert_and_reply(
214
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
215
+ vec![
216
+ CancelTimer {
217
+ seq: cancel_timer_id,
218
+ }
219
+ .into(),
220
+ CompleteWorkflowExecution { result: None }.into(),
221
+ ],
222
+ ),
223
+ ],
224
+ )
225
+ .await;
226
+ }
227
+
228
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
229
+ #[tokio::test]
230
+ async fn scheduled_activity_cancellation_try_cancel(hist_batches: &'static [usize]) {
231
+ let wfid = "fake_wf_id";
232
+ let activity_seq = 1;
233
+ let activity_id = "fake_activity";
234
+ let signal_id = "signal";
235
+
236
+ let t = canned_histories::cancel_scheduled_activity(activity_id, signal_id);
237
+ let core = build_fake_worker(wfid, t, hist_batches);
238
+
239
+ poll_and_reply(
240
+ &core,
241
+ NonSticky,
242
+ &[
243
+ gen_assert_and_reply(
244
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
245
+ vec![ScheduleActivity {
246
+ seq: activity_seq,
247
+ activity_id: activity_id.to_string(),
248
+ cancellation_type: ActivityCancellationType::TryCancel as i32,
249
+ ..Default::default()
250
+ }
251
+ .into()],
252
+ ),
253
+ gen_assert_and_reply(
254
+ &job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
255
+ vec![RequestCancelActivity { seq: activity_seq }.into()],
256
+ ),
257
+ // Activity is getting resolved right away as we are in the TryCancel mode.
258
+ gen_assert_and_reply(
259
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(_)),
260
+ vec![CompleteWorkflowExecution { result: None }.into()],
261
+ ),
262
+ ],
263
+ )
264
+ .await;
265
+ }
266
+
267
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
268
+ #[tokio::test]
269
+ async fn scheduled_activity_timeout(hist_batches: &'static [usize]) {
270
+ let wfid = "fake_wf_id";
271
+ let activity_seq = 1;
272
+ let activity_id = "fake_activity";
273
+
274
+ let t = canned_histories::scheduled_activity_timeout(activity_id);
275
+ let core = build_fake_worker(wfid, t, hist_batches);
276
+ poll_and_reply(
277
+ &core,
278
+ NonSticky,
279
+ &[
280
+ gen_assert_and_reply(
281
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
282
+ vec![ScheduleActivity {
283
+ seq: activity_seq,
284
+ activity_id: activity_id.to_string(),
285
+ ..Default::default()
286
+ }
287
+ .into()],
288
+ ),
289
+ // Activity is getting resolved right away as it has been timed out.
290
+ gen_assert_and_reply(
291
+ &|res| {
292
+ assert_matches!(
293
+ res.jobs.as_slice(),
294
+ [
295
+ WorkflowActivationJob {
296
+ variant: Some(workflow_activation_job::Variant::ResolveActivity(
297
+ ResolveActivity {
298
+ seq,
299
+ result: Some(ActivityResolution {
300
+ status: Some(activity_resolution::Status::Failed(ar::Failure {
301
+ failure: Some(failure)
302
+ })),
303
+ })
304
+ }
305
+ )),
306
+ }
307
+ ] => {
308
+ assert_eq!(failure.message, "Activity task timed out".to_string());
309
+ assert_eq!(*seq, activity_seq);
310
+ }
311
+ );
312
+ },
313
+ vec![CompleteWorkflowExecution { result: None }.into()],
314
+ ),
315
+ ],
316
+ )
317
+ .await;
318
+ }
319
+
320
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
321
+ #[tokio::test]
322
+ async fn started_activity_timeout(hist_batches: &'static [usize]) {
323
+ let wfid = "fake_wf_id";
324
+ let activity_seq = 1;
325
+
326
+ let t = canned_histories::started_activity_timeout(activity_seq.to_string().as_str());
327
+ let core = build_fake_worker(wfid, t, hist_batches);
328
+
329
+ poll_and_reply(
330
+ &core,
331
+ NonSticky,
332
+ &[
333
+ gen_assert_and_reply(
334
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
335
+ vec![ScheduleActivity {
336
+ seq: activity_seq,
337
+ activity_id: activity_seq.to_string(),
338
+ ..Default::default()
339
+ }
340
+ .into()],
341
+ ),
342
+ // Activity is getting resolved right away as it has been timed out.
343
+ gen_assert_and_reply(
344
+ &|res| {
345
+ assert_matches!(
346
+ res.jobs.as_slice(),
347
+ [
348
+ WorkflowActivationJob {
349
+ variant: Some(workflow_activation_job::Variant::ResolveActivity(
350
+ ResolveActivity {
351
+ seq,
352
+ result: Some(ActivityResolution {
353
+ status: Some(activity_resolution::Status::Failed(ar::Failure {
354
+ failure: Some(failure)
355
+ })),
356
+ })
357
+ }
358
+ )),
359
+ }
360
+ ] => {
361
+ assert_eq!(failure.message, "Activity task timed out".to_string());
362
+ assert_eq!(*seq, activity_seq);
363
+ }
364
+ );
365
+ },
366
+ vec![CompleteWorkflowExecution { result: None }.into()],
367
+ ),
368
+ ],
369
+ )
370
+ .await;
371
+ }
372
+
373
+ #[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
374
+ #[tokio::test]
375
+ async fn cancelled_activity_timeout(hist_batches: &'static [usize]) {
376
+ let wfid = "fake_wf_id";
377
+ let activity_seq = 0;
378
+ let activity_id = "fake_activity";
379
+ let signal_id = "signal";
380
+
381
+ let t = canned_histories::scheduled_cancelled_activity_timeout(activity_id, signal_id);
382
+ let core = build_fake_worker(wfid, t, hist_batches);
383
+
384
+ poll_and_reply(
385
+ &core,
386
+ NonSticky,
387
+ &[
388
+ gen_assert_and_reply(
389
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
390
+ vec![ScheduleActivity {
391
+ seq: activity_seq,
392
+ activity_id: activity_id.to_string(),
393
+ ..Default::default()
394
+ }
395
+ .into()],
396
+ ),
397
+ gen_assert_and_reply(
398
+ &job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
399
+ vec![RequestCancelActivity { seq: activity_seq }.into()],
400
+ ),
401
+ // Activity is resolved right away as it has timed out.
402
+ gen_assert_and_reply(
403
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(
404
+ ResolveActivity {
405
+ seq: _,
406
+ result: Some(ActivityResolution {
407
+ status: Some(activity_resolution::Status::Cancelled(..)),
408
+ })
409
+ }
410
+ )),
411
+ vec![CompleteWorkflowExecution { result: None }.into()],
412
+ ),
413
+ ],
414
+ )
415
+ .await;
416
+ }
417
+
418
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
419
+ #[tokio::test]
420
+ async fn scheduled_activity_cancellation_abandon(hist_batches: &'static [usize]) {
421
+ let wfid = "fake_wf_id";
422
+ let activity_id = 1;
423
+ let signal_id = "signal";
424
+
425
+ let t = canned_histories::cancel_scheduled_activity_abandon(
426
+ activity_id.to_string().as_str(),
427
+ signal_id,
428
+ );
429
+ let core = build_fake_worker(wfid, t, hist_batches);
430
+
431
+ verify_activity_cancellation(&core, activity_id, ActivityCancellationType::Abandon).await;
432
+ }
433
+
434
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
435
+ #[tokio::test]
436
+ async fn started_activity_cancellation_abandon(hist_batches: &'static [usize]) {
437
+ let wfid = "fake_wf_id";
438
+ let activity_id = 1;
439
+ let signal_id = "signal";
440
+
441
+ let t = canned_histories::cancel_started_activity_abandon(
442
+ activity_id.to_string().as_str(),
443
+ signal_id,
444
+ );
445
+ let core = build_fake_worker(wfid, t, hist_batches);
446
+
447
+ verify_activity_cancellation(&core, activity_id, ActivityCancellationType::Abandon).await;
448
+ }
449
+
450
+ #[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
451
+ #[tokio::test]
452
+ async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [usize]) {
453
+ let wfid = "fake_wf_id";
454
+ let wf_type = DEFAULT_WORKFLOW_TYPE;
455
+ let activity_id = "1";
456
+
457
+ let mut t = TestHistoryBuilder::default();
458
+ t.add_by_type(EventType::WorkflowExecutionStarted);
459
+ t.add_full_wf_task();
460
+ let act_scheduled_event_id = t.add_activity_task_scheduled(activity_id);
461
+ let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
462
+ t.add_timer_fired(timer_started_event_id, "1".to_string());
463
+ t.add_full_wf_task();
464
+ let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
465
+ let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
466
+ t.add_activity_task_completed(
467
+ act_scheduled_event_id,
468
+ act_started_event_id,
469
+ Default::default(),
470
+ );
471
+ t.add_full_wf_task();
472
+ t.add_timer_fired(timer_started_event_id, "2".to_string());
473
+ t.add_full_wf_task();
474
+ t.add_workflow_execution_completed();
475
+ let mock = mock_workflow_client();
476
+ let mut worker = mock_sdk(MockPollCfg::from_resp_batches(wfid, t, hist_batches, mock));
477
+
478
+ worker.register_wf(wf_type.to_owned(), |ctx: WfContext| async move {
479
+ let act_fut = ctx.activity(ActivityOptions {
480
+ activity_type: "echo_activity".to_string(),
481
+ start_to_close_timeout: Some(Duration::from_secs(5)),
482
+ cancellation_type: ActivityCancellationType::Abandon,
483
+ ..Default::default()
484
+ });
485
+ ctx.timer(Duration::from_secs(1)).await;
486
+ act_fut.cancel(&ctx);
487
+ ctx.timer(Duration::from_secs(3)).await;
488
+ act_fut.await;
489
+ Ok(().into())
490
+ });
491
+ worker
492
+ .submit_wf(wfid, wf_type, vec![], Default::default())
493
+ .await
494
+ .unwrap();
495
+ worker.run_until_done().await.unwrap();
496
+ }
497
+
498
+ #[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
499
+ #[tokio::test]
500
+ async fn scheduled_activity_cancellation_try_cancel_task_canceled(hist_batches: &'static [usize]) {
501
+ let wfid = "fake_wf_id";
502
+ let activity_id = 1;
503
+ let signal_id = "signal";
504
+
505
+ let t = canned_histories::cancel_scheduled_activity_with_activity_task_cancel(
506
+ activity_id.to_string().as_str(),
507
+ signal_id,
508
+ );
509
+ let core = build_fake_worker(wfid, t, hist_batches);
510
+
511
+ verify_activity_cancellation(&core, activity_id, ActivityCancellationType::TryCancel).await;
512
+ }
513
+
514
+ #[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
515
+ #[tokio::test]
516
+ async fn started_activity_cancellation_try_cancel_task_canceled(hist_batches: &'static [usize]) {
517
+ let wfid = "fake_wf_id";
518
+ let activity_id = 1;
519
+ let signal_id = "signal";
520
+
521
+ let t = canned_histories::cancel_started_activity_with_activity_task_cancel(
522
+ activity_id.to_string().as_str(),
523
+ signal_id,
524
+ );
525
+ let core = build_fake_worker(wfid, t, hist_batches);
526
+
527
+ verify_activity_cancellation(&core, activity_id, ActivityCancellationType::TryCancel).await;
528
+ }
529
+
530
+ /// Verification for try cancel & abandon histories
531
+ async fn verify_activity_cancellation(
532
+ worker: &Worker,
533
+ activity_seq: u32,
534
+ cancel_type: ActivityCancellationType,
535
+ ) {
536
+ poll_and_reply(
537
+ worker,
538
+ NonSticky,
539
+ &[
540
+ gen_assert_and_reply(
541
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
542
+ vec![ScheduleActivity {
543
+ seq: activity_seq,
544
+ activity_id: activity_seq.to_string(),
545
+ cancellation_type: cancel_type as i32,
546
+ ..Default::default()
547
+ }
548
+ .into()],
549
+ ),
550
+ gen_assert_and_reply(
551
+ &job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
552
+ vec![RequestCancelActivity { seq: activity_seq }.into()],
553
+ ),
554
+ // Activity should be resolved right away
555
+ gen_assert_and_reply(
556
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(
557
+ ResolveActivity {
558
+ seq: _,
559
+ result: Some(ActivityResolution {
560
+ status: Some(activity_resolution::Status::Cancelled(..)),
561
+ })
562
+ }
563
+ )),
564
+ vec![CompleteWorkflowExecution { result: None }.into()],
565
+ ),
566
+ ],
567
+ )
568
+ .await;
569
+ }
570
+
571
+ #[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
572
+ #[tokio::test]
573
+ async fn scheduled_activity_cancellation_wait_for_cancellation(hist_batches: &'static [usize]) {
574
+ let wfid = "fake_wf_id";
575
+ let activity_id = 1;
576
+ let signal_id = "signal";
577
+
578
+ let t = canned_histories::cancel_scheduled_activity_with_signal_and_activity_task_cancel(
579
+ activity_id.to_string().as_str(),
580
+ signal_id,
581
+ );
582
+ let core = build_fake_worker(wfid, t, hist_batches);
583
+
584
+ verify_activity_cancellation_wait_for_cancellation(activity_id, &core).await;
585
+ }
586
+
587
+ #[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
588
+ #[tokio::test]
589
+ async fn started_activity_cancellation_wait_for_cancellation(hist_batches: &'static [usize]) {
590
+ let wfid = "fake_wf_id";
591
+ let activity_id = 1;
592
+ let signal_id = "signal";
593
+
594
+ let t = canned_histories::cancel_started_activity_with_signal_and_activity_task_cancel(
595
+ activity_id.to_string().as_str(),
596
+ signal_id,
597
+ );
598
+ let core = build_fake_worker(wfid, t, hist_batches);
599
+
600
+ verify_activity_cancellation_wait_for_cancellation(activity_id, &core).await;
601
+ }
602
+
603
+ async fn verify_activity_cancellation_wait_for_cancellation(activity_id: u32, worker: &Worker) {
604
+ poll_and_reply(
605
+ worker,
606
+ NonSticky,
607
+ &[
608
+ gen_assert_and_reply(
609
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
610
+ vec![ScheduleActivity {
611
+ seq: activity_id,
612
+ activity_id: activity_id.to_string(),
613
+ cancellation_type: ActivityCancellationType::WaitCancellationCompleted as i32,
614
+ ..Default::default()
615
+ }
616
+ .into()],
617
+ ),
618
+ gen_assert_and_reply(
619
+ &job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
620
+ vec![RequestCancelActivity { seq: activity_id }.into()],
621
+ ),
622
+ // Making sure that activity is not resolved until it's cancelled.
623
+ gen_assert_and_reply(
624
+ &job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
625
+ vec![],
626
+ ),
627
+ // Now ActivityTaskCanceled has been processed and activity can be resolved.
628
+ gen_assert_and_reply(
629
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(
630
+ ResolveActivity {
631
+ seq: _,
632
+ result: Some(ActivityResolution {
633
+ status: Some(activity_resolution::Status::Cancelled(..)),
634
+ })
635
+ }
636
+ )),
637
+ vec![CompleteWorkflowExecution { result: None }.into()],
638
+ ),
639
+ ],
640
+ )
641
+ .await;
642
+ }
643
+
644
+ #[tokio::test]
645
+ async fn workflow_update_random_seed_on_workflow_reset() {
646
+ let wfid = "fake_wf_id";
647
+ let new_run_id = "86E39A5F-AE31-4626-BDFE-398EE072D156";
648
+ let timer_1_id = 1;
649
+ let randomness_seed_from_start = AtomicU64::new(0);
650
+
651
+ let t = canned_histories::workflow_fails_with_reset_after_timer(
652
+ timer_1_id.to_string().as_str(),
653
+ new_run_id,
654
+ );
655
+ let core = build_fake_worker(wfid, t, &[2]);
656
+
657
+ poll_and_reply(
658
+ &core,
659
+ NonSticky,
660
+ &[
661
+ gen_assert_and_reply(
662
+ &|res| {
663
+ assert_matches!(
664
+ res.jobs.as_slice(),
665
+ [WorkflowActivationJob {
666
+ variant: Some(workflow_activation_job::Variant::StartWorkflow(
667
+ StartWorkflow{randomness_seed, ..}
668
+ )),
669
+ }] => {
670
+ randomness_seed_from_start.store(*randomness_seed, Ordering::SeqCst);
671
+ }
672
+ );
673
+ },
674
+ vec![start_timer_cmd(timer_1_id, Duration::from_secs(1))],
675
+ ),
676
+ gen_assert_and_reply(
677
+ &|res| {
678
+ assert_matches!(
679
+ res.jobs.as_slice(),
680
+ [WorkflowActivationJob {
681
+ variant: Some(workflow_activation_job::Variant::FireTimer(_),),
682
+ },
683
+ WorkflowActivationJob {
684
+ variant: Some(workflow_activation_job::Variant::UpdateRandomSeed(
685
+ UpdateRandomSeed{randomness_seed})),
686
+ }] => {
687
+ assert_ne!(randomness_seed_from_start.load(Ordering::SeqCst),
688
+ *randomness_seed);
689
+ }
690
+ );
691
+ },
692
+ vec![CompleteWorkflowExecution { result: None }.into()],
693
+ ),
694
+ ],
695
+ )
696
+ .await;
697
+ }
698
+
699
+ #[tokio::test]
700
+ async fn cancel_timer_before_sent_wf_bridge() {
701
+ let wfid = "fake_wf_id";
702
+ let cancel_timer_id = 1;
703
+
704
+ let mut t = TestHistoryBuilder::default();
705
+ t.add_by_type(EventType::WorkflowExecutionStarted);
706
+ t.add_full_wf_task();
707
+ t.add_workflow_execution_completed();
708
+
709
+ let core = build_fake_worker(wfid, t, &[1]);
710
+
711
+ poll_and_reply(
712
+ &core,
713
+ NonSticky,
714
+ &[gen_assert_and_reply(
715
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
716
+ vec![
717
+ start_timer_cmd(cancel_timer_id, Duration::from_secs(1)),
718
+ CancelTimer {
719
+ seq: cancel_timer_id,
720
+ }
721
+ .into(),
722
+ CompleteWorkflowExecution { result: None }.into(),
723
+ ],
724
+ )],
725
+ )
726
+ .await;
727
+ }
728
+
729
+ #[rstest]
730
+ #[case::no_evict_inc(&[1, 2, 2], NonSticky)]
731
+ #[case::no_evict(&[2, 2], NonSticky)]
732
+ #[tokio::test]
733
+ async fn complete_activation_with_failure(
734
+ #[case] batches: &'static [usize],
735
+ #[case] evict: WorkflowCachingPolicy,
736
+ ) {
737
+ let wfid = "fake_wf_id";
738
+ let timer_id = 1;
739
+
740
+ let hist =
741
+ canned_histories::workflow_fails_with_failure_after_timer(timer_id.to_string().as_str());
742
+ let mock_sg = build_multihist_mock_sg(
743
+ vec![FakeWfResponses {
744
+ wf_id: wfid.to_string(),
745
+ hist,
746
+ response_batches: batches.iter().map(Into::into).collect(),
747
+ }],
748
+ true,
749
+ 1,
750
+ );
751
+ let core = mock_worker(mock_sg);
752
+
753
+ poll_and_reply(
754
+ &core,
755
+ evict,
756
+ &[
757
+ gen_assert_and_reply(
758
+ &|_| {},
759
+ vec![start_timer_cmd(timer_id, Duration::from_secs(1))],
760
+ ),
761
+ gen_assert_and_fail(&|_| {}),
762
+ gen_assert_and_reply(
763
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
764
+ vec![CompleteWorkflowExecution { result: None }.into()],
765
+ ),
766
+ ],
767
+ )
768
+ .await;
769
+ core.shutdown().await;
770
+ }
771
+
772
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
773
+ #[tokio::test]
774
+ async fn simple_timer_fail_wf_execution(hist_batches: &'static [usize]) {
775
+ let wfid = "fake_wf_id";
776
+ let timer_id = 1;
777
+
778
+ let t = canned_histories::single_timer(timer_id.to_string().as_str());
779
+ let core = build_fake_worker(wfid, t, hist_batches);
780
+
781
+ poll_and_reply(
782
+ &core,
783
+ NonSticky,
784
+ &[
785
+ gen_assert_and_reply(
786
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
787
+ vec![start_timer_cmd(timer_id, Duration::from_secs(1))],
788
+ ),
789
+ gen_assert_and_reply(
790
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
791
+ vec![FailWorkflowExecution {
792
+ failure: Some(Failure {
793
+ message: "I'm ded".to_string(),
794
+ ..Default::default()
795
+ }),
796
+ }
797
+ .into()],
798
+ ),
799
+ ],
800
+ )
801
+ .await;
802
+ }
803
+
804
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
805
+ #[tokio::test]
806
+ async fn two_signals(hist_batches: &'static [usize]) {
807
+ let wfid = "fake_wf_id";
808
+
809
+ let t = canned_histories::two_signals("sig1", "sig2");
810
+ let core = build_fake_worker(wfid, t, hist_batches);
811
+
812
+ poll_and_reply(
813
+ &core,
814
+ NonSticky,
815
+ &[
816
+ gen_assert_and_reply(
817
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
818
+ // Task is completed with no commands
819
+ vec![],
820
+ ),
821
+ gen_assert_and_reply(
822
+ &job_assert!(
823
+ workflow_activation_job::Variant::SignalWorkflow(_),
824
+ workflow_activation_job::Variant::SignalWorkflow(_)
825
+ ),
826
+ vec![],
827
+ ),
828
+ ],
829
+ )
830
+ .await;
831
+ }
832
+
833
+ #[tokio::test]
834
+ async fn workflow_failures_only_reported_once() {
835
+ let wfid = "fake_wf_id";
836
+ let timer_1 = 1;
837
+ let timer_2 = 2;
838
+
839
+ let hist = canned_histories::workflow_fails_with_failure_two_different_points(
840
+ timer_1.to_string().as_str(),
841
+ timer_2.to_string().as_str(),
842
+ );
843
+ let response_batches = vec![
844
+ 1, 2, // Start then first good reply
845
+ 2, 2, 2, // Poll for every failure
846
+ // Poll again after evicting after second good reply, then two more fails
847
+ 3, 3, 3,
848
+ ];
849
+ let mocks = build_multihist_mock_sg(
850
+ vec![FakeWfResponses {
851
+ wf_id: wfid.to_string(),
852
+ hist,
853
+ response_batches: response_batches.into_iter().map(Into::into).collect(),
854
+ }],
855
+ true,
856
+ // We should only call the server to say we failed twice (once after each success)
857
+ 2,
858
+ );
859
+ let omap = mocks.outstanding_task_map.clone();
860
+ let core = mock_worker(mocks);
861
+
862
+ poll_and_reply_clears_outstanding_evicts(
863
+ &core,
864
+ omap,
865
+ NonSticky,
866
+ &[
867
+ gen_assert_and_reply(
868
+ &|_| {},
869
+ vec![start_timer_cmd(timer_1, Duration::from_secs(1))],
870
+ ),
871
+ // Fail a few times in a row (only one of which should be reported)
872
+ gen_assert_and_fail(&|_| {}),
873
+ gen_assert_and_fail(&|_| {}),
874
+ gen_assert_and_fail(&|_| {}),
875
+ gen_assert_and_reply(
876
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
877
+ vec![start_timer_cmd(timer_2, Duration::from_secs(1))],
878
+ ),
879
+ // Again (a new fail should be reported here)
880
+ gen_assert_and_fail(&|_| {}),
881
+ gen_assert_and_fail(&|_| {}),
882
+ gen_assert_and_reply(
883
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
884
+ vec![CompleteWorkflowExecution { result: None }.into()],
885
+ ),
886
+ ],
887
+ )
888
+ .await;
889
+ }
890
+
891
+ #[tokio::test]
892
+ async fn max_wft_respected() {
893
+ let total_wfs = 100;
894
+ let wf_ids: Vec<_> = (0..total_wfs)
895
+ .into_iter()
896
+ .map(|i| format!("fake-wf-{}", i))
897
+ .collect();
898
+ let hists = wf_ids.iter().map(|wf_id| {
899
+ let hist = canned_histories::single_timer("1");
900
+ FakeWfResponses {
901
+ wf_id: wf_id.to_string(),
902
+ hist,
903
+ response_batches: vec![1.into(), 2.into()],
904
+ }
905
+ });
906
+ let mh = MockPollCfg::new(hists.into_iter().collect(), true, 0);
907
+ let mut worker = mock_sdk_cfg(mh, |cfg| {
908
+ cfg.max_cached_workflows = total_wfs as usize;
909
+ cfg.max_outstanding_workflow_tasks = 1;
910
+ });
911
+ let active_count: &'static _ = Box::leak(Box::new(Semaphore::new(1)));
912
+ worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| async move {
913
+ drop(
914
+ active_count
915
+ .try_acquire()
916
+ .expect("No multiple concurrent workflow tasks!"),
917
+ );
918
+ ctx.timer(Duration::from_secs(1)).await;
919
+ Ok(().into())
920
+ });
921
+
922
+ for wf_id in wf_ids {
923
+ worker
924
+ .submit_wf(wf_id, DEFAULT_WORKFLOW_TYPE, vec![], Default::default())
925
+ .await
926
+ .unwrap();
927
+ }
928
+ worker.run_until_done().await.unwrap();
929
+ }
930
+
931
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[3]))]
932
+ #[tokio::test]
933
+ async fn activity_not_canceled_on_replay_repro(hist_batches: &'static [usize]) {
934
+ let wfid = "fake_wf_id";
935
+ let t = canned_histories::unsent_at_cancel_repro();
936
+ let core = build_fake_worker(wfid, t, hist_batches);
937
+ let activity_id = 1;
938
+
939
+ poll_and_reply(
940
+ &core,
941
+ NonSticky,
942
+ &[
943
+ gen_assert_and_reply(
944
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
945
+ // Start timer and activity
946
+ vec![
947
+ ScheduleActivity {
948
+ seq: activity_id,
949
+ activity_id: activity_id.to_string(),
950
+ cancellation_type: ActivityCancellationType::TryCancel as i32,
951
+ ..Default::default()
952
+ }
953
+ .into(),
954
+ start_timer_cmd(1, Duration::from_secs(1)),
955
+ ],
956
+ ),
957
+ gen_assert_and_reply(
958
+ &job_assert!(workflow_activation_job::Variant::FireTimer(_)),
959
+ vec![RequestCancelActivity { seq: activity_id }.into()],
960
+ ),
961
+ gen_assert_and_reply(
962
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(
963
+ ResolveActivity {
964
+ result: Some(ActivityResolution {
965
+ status: Some(activity_resolution::Status::Cancelled(..)),
966
+ }),
967
+ ..
968
+ }
969
+ )),
970
+ vec![start_timer_cmd(2, Duration::from_secs(1))],
971
+ ),
972
+ ],
973
+ )
974
+ .await;
975
+ }
976
+
977
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[3]))]
978
+ #[tokio::test]
979
+ async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static [usize]) {
980
+ let wfid = "fake_wf_id";
981
+ let t = canned_histories::cancel_not_sent_when_also_complete_repro();
982
+ let core = build_fake_worker(wfid, t, hist_batches);
983
+ let activity_id = 1;
984
+
985
+ poll_and_reply(
986
+ &core,
987
+ NonSticky,
988
+ &[
989
+ gen_assert_and_reply(
990
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
991
+ vec![ScheduleActivity {
992
+ seq: activity_id,
993
+ activity_id: activity_id.to_string(),
994
+ cancellation_type: ActivityCancellationType::TryCancel as i32,
995
+ ..Default::default()
996
+ }
997
+ .into()],
998
+ ),
999
+ gen_assert_and_reply(
1000
+ &job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
1001
+ vec![
1002
+ RequestCancelActivity { seq: activity_id }.into(),
1003
+ start_timer_cmd(2, Duration::from_secs(1)),
1004
+ ],
1005
+ ),
1006
+ gen_assert_and_reply(
1007
+ &job_assert!(workflow_activation_job::Variant::ResolveActivity(
1008
+ ResolveActivity {
1009
+ result: Some(ActivityResolution {
1010
+ status: Some(activity_resolution::Status::Cancelled(..)),
1011
+ }),
1012
+ ..
1013
+ }
1014
+ )),
1015
+ vec![CompleteWorkflowExecution { result: None }.into()],
1016
+ ),
1017
+ ],
1018
+ )
1019
+ .await;
1020
+ }
1021
+
1022
+ #[tokio::test]
1023
+ async fn lots_of_workflows() {
1024
+ let total_wfs = 500;
1025
+ let hists = (0..total_wfs).into_iter().map(|i| {
1026
+ let wf_id = format!("fake-wf-{}", i);
1027
+ let hist = canned_histories::single_timer("1");
1028
+ FakeWfResponses {
1029
+ wf_id,
1030
+ hist,
1031
+ response_batches: vec![1.into(), 2.into()],
1032
+ }
1033
+ });
1034
+ let mut mock = build_multihist_mock_sg(hists, false, 0);
1035
+ mock.make_wft_stream_interminable();
1036
+ let worker = &mock_worker(mock);
1037
+ let completed_count = Arc::new(Semaphore::new(0));
1038
+ let killer = async {
1039
+ let _ = completed_count.acquire_many(total_wfs).await.unwrap();
1040
+ dbg!("Shutdown initted");
1041
+ worker.initiate_shutdown();
1042
+ };
1043
+ let poller = fanout_tasks(5, |_| {
1044
+ let completed_count = completed_count.clone();
1045
+ async move {
1046
+ while let Ok(wft) = worker.poll_workflow_activation().await {
1047
+ let job = &wft.jobs[0];
1048
+ let reply = match job.variant {
1049
+ Some(workflow_activation_job::Variant::StartWorkflow(_)) => {
1050
+ start_timer_cmd(1, Duration::from_secs(1))
1051
+ }
1052
+ Some(workflow_activation_job::Variant::RemoveFromCache(_)) => {
1053
+ worker
1054
+ .complete_workflow_activation(WorkflowActivationCompletion::empty(
1055
+ wft.run_id,
1056
+ ))
1057
+ .await
1058
+ .unwrap();
1059
+ continue;
1060
+ }
1061
+ _ => {
1062
+ completed_count.add_permits(1);
1063
+ CompleteWorkflowExecution { result: None }.into()
1064
+ }
1065
+ };
1066
+ worker
1067
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1068
+ wft.run_id, reply,
1069
+ ))
1070
+ .await
1071
+ .unwrap();
1072
+ }
1073
+ }
1074
+ });
1075
+ join!(killer, poller);
1076
+ worker.shutdown().await;
1077
+ }
1078
+
1079
+ #[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
1080
+ #[tokio::test]
1081
+ async fn wft_timeout_repro(hist_batches: &'static [usize]) {
1082
+ let wfid = "fake_wf_id";
1083
+ let t = canned_histories::wft_timeout_repro();
1084
+ let core = build_fake_worker(wfid, t, hist_batches);
1085
+ let activity_id = 1;
1086
+
1087
+ poll_and_reply(
1088
+ &core,
1089
+ NonSticky,
1090
+ &[
1091
+ gen_assert_and_reply(
1092
+ &job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
1093
+ vec![ScheduleActivity {
1094
+ seq: activity_id,
1095
+ activity_id: activity_id.to_string(),
1096
+ cancellation_type: ActivityCancellationType::TryCancel as i32,
1097
+ ..Default::default()
1098
+ }
1099
+ .into()],
1100
+ ),
1101
+ gen_assert_and_reply(
1102
+ &job_assert!(
1103
+ workflow_activation_job::Variant::SignalWorkflow(_),
1104
+ workflow_activation_job::Variant::SignalWorkflow(_),
1105
+ workflow_activation_job::Variant::ResolveActivity(ResolveActivity {
1106
+ result: Some(ActivityResolution {
1107
+ status: Some(activity_resolution::Status::Completed(..)),
1108
+ }),
1109
+ ..
1110
+ })
1111
+ ),
1112
+ vec![CompleteWorkflowExecution { result: None }.into()],
1113
+ ),
1114
+ ],
1115
+ )
1116
+ .await;
1117
+ }
1118
+
1119
+ #[tokio::test]
1120
+ async fn complete_after_eviction() {
1121
+ let wfid = "fake_wf_id";
1122
+ let t = canned_histories::single_timer("1");
1123
+ let mut mock = mock_workflow_client();
1124
+ mock.expect_complete_workflow_task().times(0);
1125
+ let mock = single_hist_mock_sg(wfid, t, &[2], mock, true);
1126
+ let core = mock_worker(mock);
1127
+
1128
+ let activation = core.poll_workflow_activation().await.unwrap();
1129
+ // We just got start workflow, immediately evict
1130
+ core.request_workflow_eviction(&activation.run_id);
1131
+ // Since we got whole history, we must finish replay before eviction will appear
1132
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1133
+ activation.run_id,
1134
+ start_timer_cmd(1, Duration::from_secs(1)),
1135
+ ))
1136
+ .await
1137
+ .unwrap();
1138
+ let next_activation = core.poll_workflow_activation().await.unwrap();
1139
+ assert_matches!(
1140
+ next_activation.jobs.as_slice(),
1141
+ [WorkflowActivationJob {
1142
+ variant: Some(workflow_activation_job::Variant::FireTimer(_)),
1143
+ },]
1144
+ );
1145
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1146
+ next_activation.run_id,
1147
+ vec![CompleteWorkflowExecution { result: None }.into()],
1148
+ ))
1149
+ .await
1150
+ .unwrap();
1151
+
1152
+ core.shutdown().await;
1153
+ }
1154
+
1155
+ #[tokio::test]
1156
+ async fn sends_appropriate_sticky_task_queue_responses() {
1157
+ // This test verifies that when completions are sent with sticky queues enabled, that they
1158
+ // include the information that tells the server to enqueue the next task on a sticky queue.
1159
+ let wfid = "fake_wf_id";
1160
+ let t = canned_histories::single_timer("1");
1161
+ let mut mock = mock_workflow_client();
1162
+ mock.expect_complete_workflow_task()
1163
+ .withf(|comp| comp.sticky_attributes.is_some())
1164
+ .times(1)
1165
+ .returning(|_| Ok(Default::default()));
1166
+ mock.expect_complete_workflow_task().times(0);
1167
+ let mut mock = single_hist_mock_sg(wfid, t, &[1], mock, false);
1168
+ mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
1169
+ let core = mock_worker(mock);
1170
+
1171
+ let activation = core.poll_workflow_activation().await.unwrap();
1172
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1173
+ activation.run_id,
1174
+ start_timer_cmd(1, Duration::from_secs(1)),
1175
+ ))
1176
+ .await
1177
+ .unwrap();
1178
+ core.shutdown().await;
1179
+ }
1180
+
1181
+ #[tokio::test]
1182
+ async fn new_server_work_while_eviction_outstanding_doesnt_overwrite_activation() {
1183
+ let wfid = "fake_wf_id";
1184
+ let t = canned_histories::single_timer("1");
1185
+ let mock = single_hist_mock_sg(wfid, t, &[1, 2], mock_workflow_client(), false);
1186
+ let taskmap = mock.outstanding_task_map.clone().unwrap();
1187
+ let core = mock_worker(mock);
1188
+
1189
+ // Poll for and complete first workflow task
1190
+ let activation = core.poll_workflow_activation().await.unwrap();
1191
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1192
+ activation.run_id,
1193
+ start_timer_cmd(1, Duration::from_secs(1)),
1194
+ ))
1195
+ .await
1196
+ .unwrap();
1197
+ let evict_act = core.poll_workflow_activation().await.unwrap();
1198
+ assert_matches!(
1199
+ evict_act.jobs.as_slice(),
1200
+ [WorkflowActivationJob {
1201
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1202
+ }]
1203
+ );
1204
+ // Ensure mock has delivered both tasks
1205
+ assert!(taskmap.all_work_delivered());
1206
+ // Now we can complete the evict
1207
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_act.run_id))
1208
+ .await
1209
+ .unwrap();
1210
+ // The task buffered during eviction is applied and we start over
1211
+ let start_again = core.poll_workflow_activation().await.unwrap();
1212
+ assert_matches!(
1213
+ start_again.jobs[0].variant,
1214
+ Some(workflow_activation_job::Variant::StartWorkflow(_))
1215
+ );
1216
+ }
1217
+
1218
+ #[tokio::test]
1219
+ async fn buffered_work_drained_on_shutdown() {
1220
+ let wfid = "fake_wf_id";
1221
+ // Build a one-timer history where first task times out
1222
+ let mut t = TestHistoryBuilder::default();
1223
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1224
+ t.add_workflow_task_scheduled_and_started();
1225
+ // Need to build the first response before adding the timeout events b/c otherwise the history
1226
+ // builder will include them in the first task
1227
+ let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()).resp;
1228
+ t.add_workflow_task_timed_out();
1229
+ t.add_full_wf_task();
1230
+ let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
1231
+ t.add(
1232
+ EventType::TimerFired,
1233
+ history_event::Attributes::TimerFiredEventAttributes(TimerFiredEventAttributes {
1234
+ started_event_id: timer_started_event_id,
1235
+ timer_id: "1".to_string(),
1236
+ }),
1237
+ );
1238
+ t.add_full_wf_task();
1239
+ t.add_workflow_execution_completed();
1240
+
1241
+ let mut tasks = VecDeque::from(vec![resp_1]);
1242
+ // Extend the task list with the now timeout-included version of the task. We add a bunch of
1243
+ // them because the poll loop will spin while new tasks are available and it is buffering them
1244
+ tasks.extend(
1245
+ std::iter::repeat_with(|| {
1246
+ hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp
1247
+ })
1248
+ .take(50),
1249
+ );
1250
+ let mut mock = mock_workflow_client();
1251
+ mock.expect_complete_workflow_task()
1252
+ .returning(|_| Ok(RespondWorkflowTaskCompletedResponse::default()));
1253
+ let mut mock = MocksHolder::from_wft_stream(mock, stream::iter(tasks));
1254
+ // Cache on to avoid being super repetitive
1255
+ mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
1256
+ let core = &mock_worker(mock);
1257
+
1258
+ // Poll for first WFT
1259
+ let act1 = core.poll_workflow_activation().await.unwrap();
1260
+ let poll_fut = async move {
1261
+ // Now poll again, which will start spinning, and buffer the next WFT with timer fired in it
1262
+ // - it won't stop spinning until the first task is complete
1263
+ let t = core.poll_workflow_activation().await.unwrap();
1264
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1265
+ t.run_id,
1266
+ vec![CompleteWorkflowExecution { result: None }.into()],
1267
+ ))
1268
+ .await
1269
+ .unwrap();
1270
+ };
1271
+ let complete_first = async move {
1272
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1273
+ act1.run_id,
1274
+ start_timer_cmd(1, Duration::from_secs(1)),
1275
+ ))
1276
+ .await
1277
+ .unwrap();
1278
+ };
1279
+ join!(poll_fut, complete_first, async {
1280
+ // If the shutdown is sent too too fast, we might not have got a chance to even buffer work
1281
+ tokio::time::sleep(Duration::from_millis(5)).await;
1282
+ core.shutdown().await;
1283
+ });
1284
+ }
1285
+
1286
+ #[tokio::test]
1287
+ async fn fail_wft_then_recover() {
1288
+ let t = canned_histories::long_sequential_timers(1);
1289
+ let mut mh = MockPollCfg::from_resp_batches(
1290
+ "fake_wf_id",
1291
+ t,
1292
+ // We need to deliver all of history twice because of eviction
1293
+ [ResponseType::AllHistory, ResponseType::AllHistory],
1294
+ mock_workflow_client(),
1295
+ );
1296
+ mh.num_expected_fails = 1;
1297
+ mh.expect_fail_wft_matcher =
1298
+ Box::new(|_, cause, _| matches!(cause, WorkflowTaskFailedCause::NonDeterministicError));
1299
+ let mut mock = build_mock_pollers(mh);
1300
+ mock.worker_cfg(|wc| {
1301
+ wc.max_cached_workflows = 2;
1302
+ });
1303
+ let core = mock_worker(mock);
1304
+
1305
+ let act = core.poll_workflow_activation().await.unwrap();
1306
+ // Start an activity instead of a timer, triggering nondeterminism error
1307
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1308
+ act.run_id.clone(),
1309
+ vec![ScheduleActivity {
1310
+ activity_id: "fake_activity".to_string(),
1311
+ ..Default::default()
1312
+ }
1313
+ .into()],
1314
+ ))
1315
+ .await
1316
+ .unwrap();
1317
+ // We must handle an eviction now
1318
+ let evict_act = core.poll_workflow_activation().await.unwrap();
1319
+ assert_eq!(evict_act.run_id, act.run_id);
1320
+ assert_matches!(
1321
+ evict_act.jobs.as_slice(),
1322
+ [WorkflowActivationJob {
1323
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1324
+ }]
1325
+ );
1326
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_act.run_id))
1327
+ .await
1328
+ .unwrap();
1329
+
1330
+ // Workflow starting over, this time issue the right command
1331
+ let act = core.poll_workflow_activation().await.unwrap();
1332
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1333
+ act.run_id,
1334
+ vec![start_timer_cmd(1, Duration::from_secs(1))],
1335
+ ))
1336
+ .await
1337
+ .unwrap();
1338
+ let act = core.poll_workflow_activation().await.unwrap();
1339
+ assert_matches!(
1340
+ act.jobs.as_slice(),
1341
+ [WorkflowActivationJob {
1342
+ variant: Some(workflow_activation_job::Variant::FireTimer(_)),
1343
+ },]
1344
+ );
1345
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1346
+ act.run_id,
1347
+ vec![CompleteWorkflowExecution { result: None }.into()],
1348
+ ))
1349
+ .await
1350
+ .unwrap();
1351
+ core.shutdown().await;
1352
+ }
1353
+
1354
+ #[tokio::test]
1355
+ async fn poll_response_triggers_wf_error() {
1356
+ let mut t = TestHistoryBuilder::default();
1357
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1358
+ // Add this nonsense event here to make applying the poll response fail
1359
+ t.add_external_signal_completed(100);
1360
+ t.add_full_wf_task();
1361
+ t.add_workflow_execution_completed();
1362
+
1363
+ let mh = MockPollCfg::from_resp_batches(
1364
+ "fake_wf_id",
1365
+ t,
1366
+ [ResponseType::AllHistory],
1367
+ mock_workflow_client(),
1368
+ );
1369
+ let mock = build_mock_pollers(mh);
1370
+ let core = mock_worker(mock);
1371
+ // Poll for first WFT, which is immediately an eviction
1372
+ let act = core.poll_workflow_activation().await.unwrap();
1373
+ assert_matches!(
1374
+ act.jobs.as_slice(),
1375
+ [WorkflowActivationJob {
1376
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1377
+ }]
1378
+ );
1379
+ }
1380
+
1381
+ // Verifies we can handle multiple wft timeouts in a row if lang is being very slow in responding
1382
+ #[tokio::test]
1383
+ async fn lang_slower_than_wft_timeouts() {
1384
+ let wfid = "fake_wf_id";
1385
+ let mut t = TestHistoryBuilder::default();
1386
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1387
+ t.add_workflow_task_scheduled_and_started();
1388
+ t.add_workflow_task_timed_out();
1389
+ t.add_full_wf_task();
1390
+ t.add_workflow_execution_completed();
1391
+
1392
+ let mut mock = mock_workflow_client();
1393
+ mock.expect_complete_workflow_task()
1394
+ .times(1)
1395
+ .returning(|_| Err(tonic::Status::not_found("Workflow task not found.")));
1396
+ mock.expect_complete_workflow_task()
1397
+ .times(1)
1398
+ .returning(|_| Ok(Default::default()));
1399
+ let mut mock = single_hist_mock_sg(wfid, t, [1, 1], mock, true);
1400
+ let tasksmap = mock.outstanding_task_map.clone().unwrap();
1401
+ mock.worker_cfg(|wc| {
1402
+ wc.max_cached_workflows = 2;
1403
+ });
1404
+ let core = mock_worker(mock);
1405
+
1406
+ // This completion runs into the workflow task not found error
1407
+ let wf_task = core.poll_workflow_activation().await.unwrap();
1408
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
1409
+ .await
1410
+ .unwrap();
1411
+ // It will get an eviction
1412
+ let wf_task = core.poll_workflow_activation().await.unwrap();
1413
+ assert_matches!(
1414
+ wf_task.jobs.as_slice(),
1415
+ [WorkflowActivationJob {
1416
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1417
+ }]
1418
+ );
1419
+ // Before we complete, unlock the next task from the mock so that we'll see it get buffered.
1420
+ tasksmap.release_run(&wf_task.run_id);
1421
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
1422
+ .await
1423
+ .unwrap();
1424
+ // The buffered WFT should be applied now
1425
+ let start_again = core.poll_workflow_activation().await.unwrap();
1426
+ assert_matches!(
1427
+ start_again.jobs[0].variant,
1428
+ Some(workflow_activation_job::Variant::StartWorkflow(_))
1429
+ );
1430
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1431
+ start_again.run_id,
1432
+ vec![CompleteWorkflowExecution { result: None }.into()],
1433
+ ))
1434
+ .await
1435
+ .unwrap();
1436
+ core.shutdown().await;
1437
+ }
1438
+
1439
+ #[tokio::test]
1440
+ async fn tries_cancel_of_completed_activity() {
1441
+ let mut t = TestHistoryBuilder::default();
1442
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1443
+ t.add_full_wf_task();
1444
+ let scheduled_event_id = t.add_activity_task_scheduled("1");
1445
+ t.add_we_signaled("sig", vec![]);
1446
+ let started_event_id = t.add_activity_task_started(scheduled_event_id);
1447
+ t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
1448
+ t.add_workflow_task_scheduled_and_started();
1449
+
1450
+ let mock = mock_workflow_client();
1451
+ let mut mock = single_hist_mock_sg("fake_wf_id", t, &[1, 2], mock, true);
1452
+ mock.worker_cfg(|cfg| cfg.max_cached_workflows = 1);
1453
+ let core = mock_worker(mock);
1454
+
1455
+ let activation = core.poll_workflow_activation().await.unwrap();
1456
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1457
+ activation.run_id,
1458
+ ScheduleActivity {
1459
+ seq: 1,
1460
+ activity_id: "1".to_string(),
1461
+ ..Default::default()
1462
+ }
1463
+ .into(),
1464
+ ))
1465
+ .await
1466
+ .unwrap();
1467
+ let activation = core.poll_workflow_activation().await.unwrap();
1468
+ assert_matches!(
1469
+ activation.jobs.as_slice(),
1470
+ [
1471
+ WorkflowActivationJob {
1472
+ variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
1473
+ },
1474
+ WorkflowActivationJob {
1475
+ variant: Some(workflow_activation_job::Variant::ResolveActivity(_)),
1476
+ }
1477
+ ]
1478
+ );
1479
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1480
+ activation.run_id,
1481
+ vec![
1482
+ RequestCancelActivity { seq: 1 }.into(),
1483
+ CompleteWorkflowExecution { result: None }.into(),
1484
+ ],
1485
+ ))
1486
+ .await
1487
+ .unwrap();
1488
+
1489
+ core.shutdown().await;
1490
+ }
1491
+
1492
+ #[tokio::test]
1493
+ async fn failing_wft_doesnt_eat_permit_forever() {
1494
+ let mut t = TestHistoryBuilder::default();
1495
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1496
+ t.add_workflow_task_scheduled_and_started();
1497
+
1498
+ let mock = mock_workflow_client();
1499
+ let mut mock = MockPollCfg::from_resp_batches("fake_wf_id", t, [1, 1, 1], mock);
1500
+ mock.num_expected_fails = 1;
1501
+ let mut mock = build_mock_pollers(mock);
1502
+ mock.worker_cfg(|cfg| {
1503
+ cfg.max_cached_workflows = 2;
1504
+ cfg.max_outstanding_workflow_tasks = 2;
1505
+ });
1506
+ let outstanding_mock_tasks = mock.outstanding_task_map.clone();
1507
+ let worker = mock_worker(mock);
1508
+
1509
+ let mut run_id = "".to_string();
1510
+ // Fail twice, verifying a permit is not eaten. We cannot fail the same run more than twice in a
1511
+ // row because we purposefully time out rather than spamming.
1512
+ for _ in 1..=2 {
1513
+ let activation = worker.poll_workflow_activation().await.unwrap();
1514
+ // Issue a nonsense completion that will trigger a WFT failure
1515
+ worker
1516
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1517
+ activation.run_id,
1518
+ RequestCancelActivity { seq: 1 }.into(),
1519
+ ))
1520
+ .await
1521
+ .unwrap();
1522
+ let activation = worker.poll_workflow_activation().await.unwrap();
1523
+ assert_matches!(
1524
+ activation.jobs.as_slice(),
1525
+ [WorkflowActivationJob {
1526
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1527
+ },]
1528
+ );
1529
+ run_id = activation.run_id.clone();
1530
+ worker
1531
+ .complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
1532
+ .await
1533
+ .unwrap();
1534
+ }
1535
+ assert_eq!(worker.outstanding_workflow_tasks().await, 0);
1536
+ // 1 permit is in use because the next task is buffered and has re-used the permit
1537
+ assert_eq!(worker.available_wft_permits().await, 1);
1538
+ // We should be "out of work" because the mock service thinks we didn't complete the last task,
1539
+ // which we didn't, because we don't spam failures. The real server would eventually time out
1540
+ // the task. Mock doesn't understand that, so the WFT permit is released because eventually a
1541
+ // new one will be generated. We manually clear the mock's outstanding task list so the next
1542
+ // poll will work.
1543
+ outstanding_mock_tasks.unwrap().release_run(&run_id);
1544
+ let activation = worker.poll_workflow_activation().await.unwrap();
1545
+ // There should be no change in permits, since this just unbuffered the buffered task
1546
+ assert_eq!(worker.available_wft_permits().await, 1);
1547
+ worker
1548
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1549
+ activation.run_id,
1550
+ CompleteWorkflowExecution { result: None }.into(),
1551
+ ))
1552
+ .await
1553
+ .unwrap();
1554
+ assert_eq!(worker.available_wft_permits().await, 2);
1555
+
1556
+ worker.shutdown().await;
1557
+ }
1558
+
1559
+ #[tokio::test]
1560
+ async fn cache_miss_will_fetch_history() {
1561
+ let mut t = TestHistoryBuilder::default();
1562
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1563
+ t.add_full_wf_task();
1564
+ t.add_we_signaled("sig", vec![]);
1565
+ t.add_full_wf_task();
1566
+ t.add_workflow_execution_completed();
1567
+ let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
1568
+
1569
+ let mut mh = MockPollCfg::from_resp_batches(
1570
+ "fake_wf_id",
1571
+ t,
1572
+ [ResponseType::ToTaskNum(1), ResponseType::OneTask(2)],
1573
+ mock_workflow_client(),
1574
+ );
1575
+ mh.mock_client
1576
+ .expect_get_workflow_execution_history()
1577
+ .times(1)
1578
+ .returning(move |_, _, _| Ok(get_exec_resp.clone()));
1579
+ let mut mock = build_mock_pollers(mh);
1580
+ mock.worker_cfg(|cfg| {
1581
+ cfg.max_cached_workflows = 1;
1582
+ });
1583
+ let worker = mock_worker(mock);
1584
+
1585
+ let activation = worker.poll_workflow_activation().await.unwrap();
1586
+ assert_eq!(activation.history_length, 3);
1587
+ assert_matches!(
1588
+ activation.jobs.as_slice(),
1589
+ [WorkflowActivationJob {
1590
+ variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
1591
+ }]
1592
+ );
1593
+ // Force an eviction (before complete matters, so that we will be sure the eviction is queued
1594
+ // up before the next fake WFT is unlocked)
1595
+ worker.request_wf_eviction(
1596
+ &activation.run_id,
1597
+ "whatever",
1598
+ EvictionReason::LangRequested,
1599
+ );
1600
+ worker
1601
+ .complete_workflow_activation(WorkflowActivationCompletion::empty(&activation.run_id))
1602
+ .await
1603
+ .unwrap();
1604
+ // Handle the eviction, and the restart
1605
+ for i in 1..=2 {
1606
+ let activation = worker.poll_workflow_activation().await.unwrap();
1607
+ assert_eq!(activation.history_length, 3);
1608
+ if i == 1 {
1609
+ assert_matches!(
1610
+ activation.jobs.as_slice(),
1611
+ [WorkflowActivationJob {
1612
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1613
+ }]
1614
+ );
1615
+ } else {
1616
+ assert_matches!(
1617
+ activation.jobs.as_slice(),
1618
+ [WorkflowActivationJob {
1619
+ variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
1620
+ }]
1621
+ );
1622
+ }
1623
+ worker
1624
+ .complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
1625
+ .await
1626
+ .unwrap();
1627
+ }
1628
+ let activation = worker.poll_workflow_activation().await.unwrap();
1629
+ assert_eq!(activation.history_length, 7);
1630
+ assert_matches!(
1631
+ activation.jobs.as_slice(),
1632
+ [WorkflowActivationJob {
1633
+ variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
1634
+ }]
1635
+ );
1636
+ worker
1637
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1638
+ activation.run_id,
1639
+ CompleteWorkflowExecution { result: None }.into(),
1640
+ ))
1641
+ .await
1642
+ .unwrap();
1643
+ assert_eq!(worker.outstanding_workflow_tasks().await, 0);
1644
+ worker.shutdown().await;
1645
+ }
1646
+
1647
+ /// This test verifies that WFTs which come as replies to completing a WFT are properly delivered
1648
+ /// via activation polling.
1649
+ #[tokio::test]
1650
+ async fn tasks_from_completion_are_delivered() {
1651
+ let wfid = "fake_wf_id";
1652
+ let mut t = TestHistoryBuilder::default();
1653
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1654
+ t.add_full_wf_task();
1655
+ t.add_we_signaled("sig", vec![]);
1656
+ t.add_full_wf_task();
1657
+ t.add_workflow_execution_completed();
1658
+
1659
+ let mut mock = mock_workflow_client();
1660
+ let complete_resp = RespondWorkflowTaskCompletedResponse {
1661
+ workflow_task: Some(
1662
+ hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp,
1663
+ ),
1664
+ activity_tasks: vec![],
1665
+ };
1666
+ mock.expect_complete_workflow_task()
1667
+ .times(1)
1668
+ .returning(move |_| Ok(complete_resp.clone()));
1669
+ mock.expect_complete_workflow_task()
1670
+ .times(1)
1671
+ .returning(|_| Ok(Default::default()));
1672
+ let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
1673
+ mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
1674
+ let core = mock_worker(mock);
1675
+
1676
+ let wf_task = core.poll_workflow_activation().await.unwrap();
1677
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
1678
+ .await
1679
+ .unwrap();
1680
+ let wf_task = core.poll_workflow_activation().await.unwrap();
1681
+ assert_matches!(
1682
+ wf_task.jobs.as_slice(),
1683
+ [WorkflowActivationJob {
1684
+ variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
1685
+ },]
1686
+ );
1687
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1688
+ wf_task.run_id,
1689
+ vec![CompleteWorkflowExecution { result: None }.into()],
1690
+ ))
1691
+ .await
1692
+ .unwrap();
1693
+ core.shutdown().await;
1694
+ }
1695
+
1696
+ #[tokio::test]
1697
+ async fn poll_faster_than_complete_wont_overflow_cache() {
1698
+ // Make workflow tasks for 5 different runs
1699
+ let tasks: Vec<_> = (1..=5)
1700
+ .map(|i| FakeWfResponses {
1701
+ wf_id: format!("wf-{}", i),
1702
+ hist: canned_histories::single_timer("1"),
1703
+ response_batches: vec![ResponseType::ToTaskNum(1)],
1704
+ })
1705
+ .collect();
1706
+ let mut mock_client = mock_workflow_client();
1707
+ mock_client
1708
+ .expect_complete_workflow_task()
1709
+ .times(3)
1710
+ .returning(|_| Ok(Default::default()));
1711
+ let mut mock_cfg = MockPollCfg::new(tasks, true, 0);
1712
+ mock_cfg.mock_client = mock_client;
1713
+ let mut mock = build_mock_pollers(mock_cfg);
1714
+ mock.worker_cfg(|wc| {
1715
+ wc.max_cached_workflows = 3;
1716
+ wc.max_outstanding_workflow_tasks = 3;
1717
+ });
1718
+ let core = mock_worker(mock);
1719
+ // Poll 4 times, completing once, such that max tasks are never exceeded
1720
+ let p1 = core.poll_workflow_activation().await.unwrap();
1721
+ let p2 = core.poll_workflow_activation().await.unwrap();
1722
+ let p3 = core.poll_workflow_activation().await.unwrap();
1723
+ for (i, p_res) in [&p1, &p2, &p3].into_iter().enumerate() {
1724
+ assert_matches!(
1725
+ &p_res.jobs[0].variant,
1726
+ Some(workflow_activation_job::Variant::StartWorkflow(sw))
1727
+ if sw.workflow_id == format!("wf-{}", i + 1)
1728
+ );
1729
+ }
1730
+ // Complete first task to free a wft slot. Cache size is at 3
1731
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1732
+ p1.run_id,
1733
+ start_timer_cmd(1, Duration::from_secs(1)),
1734
+ ))
1735
+ .await
1736
+ .unwrap();
1737
+ // Now we're at cache limit. We will poll for a task, discover it is for a new run, issue
1738
+ // an eviction, and buffer the new run task. However, the run we're trying to evict has pending
1739
+ // activations! Thus, we must complete them first before this poll will unblock, and then it
1740
+ // will unblock with the eviciton.
1741
+ let p4 = core.poll_workflow_activation();
1742
+ // Make sure the task gets buffered before we start the complete, so the LRU list is in the
1743
+ // expected order and what we expect to evict will be evicted.
1744
+ advance_fut!(p4);
1745
+ let p4 = async {
1746
+ let p4 = p4.await.unwrap();
1747
+ assert_matches!(
1748
+ &p4.jobs.as_slice(),
1749
+ [WorkflowActivationJob {
1750
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1751
+ }]
1752
+ );
1753
+ p4
1754
+ };
1755
+ let p2_pending_completer = async {
1756
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1757
+ p2.run_id,
1758
+ start_timer_cmd(1, Duration::from_secs(1)),
1759
+ ))
1760
+ .await
1761
+ .unwrap();
1762
+ };
1763
+ let (p4, _) = join!(p4, p2_pending_completer);
1764
+ assert_eq!(core.cached_workflows().await, 3);
1765
+
1766
+ // This poll should also block until the eviction is actually completed
1767
+ let blocking_poll = async {
1768
+ let res = core.poll_workflow_activation().await.unwrap();
1769
+ assert_matches!(
1770
+ &res.jobs[0].variant,
1771
+ Some(workflow_activation_job::Variant::StartWorkflow(sw))
1772
+ if sw.workflow_id == format!("wf-{}", 4)
1773
+ );
1774
+ res
1775
+ };
1776
+ let complete_evict = async {
1777
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(p4.run_id))
1778
+ .await
1779
+ .unwrap();
1780
+ };
1781
+
1782
+ let (_p5, _) = join!(blocking_poll, complete_evict);
1783
+ assert_eq!(core.cached_workflows().await, 3);
1784
+ // The next poll will get an buffer a task for a new run, and generate an eviction for p3 but
1785
+ // that eviction cannot be obtained until we complete the existing outstanding task.
1786
+ let p6 = async {
1787
+ let p6 = core.poll_workflow_activation().await.unwrap();
1788
+ assert_matches!(
1789
+ p6.jobs.as_slice(),
1790
+ [WorkflowActivationJob {
1791
+ variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
1792
+ }]
1793
+ );
1794
+ p6
1795
+ };
1796
+ let completer = async {
1797
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1798
+ p3.run_id,
1799
+ start_timer_cmd(1, Duration::from_secs(1)),
1800
+ ))
1801
+ .await
1802
+ .unwrap();
1803
+ };
1804
+ let (p6, _) = join!(p6, completer);
1805
+ let complete_evict = async {
1806
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(p6.run_id))
1807
+ .await
1808
+ .unwrap();
1809
+ };
1810
+ let blocking_poll = async {
1811
+ // This poll will also block until the last eviction goes through, and when it does it'll
1812
+ // produce the final start workflow task
1813
+ let res = core.poll_workflow_activation().await.unwrap();
1814
+ assert_matches!(
1815
+ &res.jobs[0].variant,
1816
+ Some(workflow_activation_job::Variant::StartWorkflow(sw))
1817
+ if sw.workflow_id == "wf-5"
1818
+ );
1819
+ };
1820
+
1821
+ join!(blocking_poll, complete_evict);
1822
+ // p5 outstanding and final poll outstanding -- hence one permit available
1823
+ assert_eq!(core.available_wft_permits().await, 1);
1824
+ assert_eq!(core.cached_workflows().await, 3);
1825
+ }
1826
+
1827
+ #[tokio::test]
1828
+ async fn eviction_waits_until_replay_finished() {
1829
+ let wfid = "fake_wf_id";
1830
+ let t = canned_histories::long_sequential_timers(3);
1831
+ let mock = mock_workflow_client();
1832
+ let mock = single_hist_mock_sg(wfid, t, &[3], mock, true);
1833
+ let core = mock_worker(mock);
1834
+
1835
+ let activation = core.poll_workflow_activation().await.unwrap();
1836
+ assert_eq!(activation.history_length, 3);
1837
+ // Immediately request eviction after getting start workflow
1838
+ core.request_workflow_eviction(&activation.run_id);
1839
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1840
+ activation.run_id,
1841
+ start_timer_cmd(1, Duration::from_secs(1)),
1842
+ ))
1843
+ .await
1844
+ .unwrap();
1845
+ let t1_fired = core.poll_workflow_activation().await.unwrap();
1846
+ assert_eq!(t1_fired.history_length, 8);
1847
+ assert_matches!(
1848
+ t1_fired.jobs.as_slice(),
1849
+ [WorkflowActivationJob {
1850
+ variant: Some(workflow_activation_job::Variant::FireTimer(_)),
1851
+ }]
1852
+ );
1853
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1854
+ t1_fired.run_id,
1855
+ start_timer_cmd(2, Duration::from_secs(1)),
1856
+ ))
1857
+ .await
1858
+ .unwrap();
1859
+ let t2_fired = core.poll_workflow_activation().await.unwrap();
1860
+ assert_eq!(t2_fired.history_length, 13);
1861
+ assert_matches!(
1862
+ t2_fired.jobs.as_slice(),
1863
+ [WorkflowActivationJob {
1864
+ variant: Some(workflow_activation_job::Variant::FireTimer(_)),
1865
+ }]
1866
+ );
1867
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
1868
+ t2_fired.run_id,
1869
+ vec![CompleteWorkflowExecution { result: None }.into()],
1870
+ ))
1871
+ .await
1872
+ .unwrap();
1873
+
1874
+ core.shutdown().await;
1875
+ }
1876
+
1877
+ #[tokio::test]
1878
+ async fn autocompletes_wft_no_work() {
1879
+ let wfid = "fake_wf_id";
1880
+ let activity_id = "1";
1881
+
1882
+ let mut t = TestHistoryBuilder::default();
1883
+ t.add_by_type(EventType::WorkflowExecutionStarted);
1884
+ t.add_full_wf_task();
1885
+ let scheduled_event_id = t.add_activity_task_scheduled(activity_id);
1886
+ t.add_full_wf_task();
1887
+ t.add_we_signaled("sig1", vec![]);
1888
+ t.add_full_wf_task();
1889
+ let started_event_id = t.add_activity_task_started(scheduled_event_id);
1890
+ t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
1891
+ t.add_full_wf_task();
1892
+ let mock = mock_workflow_client();
1893
+ let mut mock = single_hist_mock_sg(wfid, t, &[1, 2, 3, 4], mock, true);
1894
+ mock.worker_cfg(|w| w.max_cached_workflows = 1);
1895
+ let core = mock_worker(mock);
1896
+
1897
+ let act = core.poll_workflow_activation().await.unwrap();
1898
+ assert_matches!(
1899
+ act.jobs.as_slice(),
1900
+ [WorkflowActivationJob {
1901
+ variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
1902
+ }]
1903
+ );
1904
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1905
+ act.run_id,
1906
+ ScheduleActivity {
1907
+ seq: 1,
1908
+ activity_id: activity_id.to_string(),
1909
+ cancellation_type: ActivityCancellationType::Abandon as i32,
1910
+ ..Default::default()
1911
+ }
1912
+ .into(),
1913
+ ))
1914
+ .await
1915
+ .unwrap();
1916
+ let act = core.poll_workflow_activation().await.unwrap();
1917
+ assert_matches!(
1918
+ act.jobs.as_slice(),
1919
+ [WorkflowActivationJob {
1920
+ variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
1921
+ }]
1922
+ );
1923
+ core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1924
+ act.run_id,
1925
+ RequestCancelActivity { seq: 1 }.into(),
1926
+ ))
1927
+ .await
1928
+ .unwrap();
1929
+ let act = core.poll_workflow_activation().await.unwrap();
1930
+ core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
1931
+ .await
1932
+ .unwrap();
1933
+ // The last task will autocomplete, and thus this will return shutdown since there is no more
1934
+ // work
1935
+ assert_matches!(
1936
+ core.poll_workflow_activation().await.unwrap_err(),
1937
+ PollWfError::ShutDown
1938
+ );
1939
+
1940
+ core.shutdown().await;
1941
+ }
1942
+
1943
+ #[tokio::test]
1944
+ async fn no_race_acquiring_permits() {
1945
+ let wfid = "fake_wf_id";
1946
+ let mut mock_client = mock_manual_workflow_client();
1947
+ // We need to allow two polls to happen by triggering two processing events in the workflow
1948
+ // stream, but then delivering the actual tasks after that
1949
+ let task_barr: &'static Barrier = Box::leak(Box::new(Barrier::new(2)));
1950
+ mock_client
1951
+ .expect_poll_workflow_task()
1952
+ .returning(move |_, _| {
1953
+ let t = canned_histories::single_timer("1");
1954
+ let poll_resp =
1955
+ hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp;
1956
+ async move {
1957
+ task_barr.wait().await;
1958
+ Ok(poll_resp.clone())
1959
+ }
1960
+ .boxed()
1961
+ });
1962
+ mock_client
1963
+ .expect_complete_workflow_task()
1964
+ .returning(|_| async move { Ok(Default::default()) }.boxed());
1965
+
1966
+ let worker = Worker::new_test(
1967
+ test_worker_cfg()
1968
+ .max_outstanding_workflow_tasks(1_usize)
1969
+ .max_cached_workflows(10_usize)
1970
+ .build()
1971
+ .unwrap(),
1972
+ mock_client,
1973
+ );
1974
+
1975
+ // Two polls in a row, both of which will get stuck on the barrier and are only allowed to
1976
+ // proceed after a call which will cause the workflow stream to process an event. Without the
1977
+ // fix, this would've meant the stream though it was OK to poll twice, but once the tasks
1978
+ // are received, it would find there was only one permit.
1979
+ let poll_1_f = async {
1980
+ let r = worker.poll_workflow_activation().await.unwrap();
1981
+ worker
1982
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1983
+ r.run_id,
1984
+ start_timer_cmd(1, Duration::from_secs(1)),
1985
+ ))
1986
+ .await
1987
+ .unwrap();
1988
+ };
1989
+ let poll_2_f = async {
1990
+ let r = worker.poll_workflow_activation().await.unwrap();
1991
+ worker
1992
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
1993
+ r.run_id,
1994
+ start_timer_cmd(1, Duration::from_secs(1)),
1995
+ ))
1996
+ .await
1997
+ .unwrap();
1998
+ };
1999
+ let other_f = async {
2000
+ worker.cached_workflows().await;
2001
+ task_barr.wait().await;
2002
+ worker.cached_workflows().await;
2003
+ task_barr.wait().await;
2004
+ };
2005
+ join!(poll_1_f, poll_2_f, other_f);
2006
+ }
2007
+
2008
+ #[tokio::test]
2009
+ async fn continue_as_new_preserves_some_values() {
2010
+ let wfid = "fake_wf_id";
2011
+ let memo = HashMap::<String, Payload>::from([("enchi".to_string(), b"cat".into())]).into();
2012
+ let search = HashMap::<String, Payload>::from([("noisy".to_string(), b"kitty".into())]).into();
2013
+ let retry_policy = RetryPolicy {
2014
+ backoff_coefficient: 13.37,
2015
+ ..Default::default()
2016
+ };
2017
+ let mut wes_attrs = default_wes_attribs();
2018
+ wes_attrs.memo = Some(memo);
2019
+ wes_attrs.search_attributes = Some(search);
2020
+ wes_attrs.retry_policy = Some(retry_policy);
2021
+ let mut mock_client = mock_workflow_client();
2022
+ let hist = {
2023
+ let mut t = TestHistoryBuilder::default();
2024
+ t.add(
2025
+ EventType::WorkflowExecutionStarted,
2026
+ wes_attrs.clone().into(),
2027
+ );
2028
+ t.add_full_wf_task();
2029
+ t
2030
+ };
2031
+ mock_client
2032
+ .expect_poll_workflow_task()
2033
+ .returning(move |_, _| {
2034
+ Ok(hist_to_poll_resp(
2035
+ &hist,
2036
+ wfid.to_owned(),
2037
+ ResponseType::AllHistory,
2038
+ TEST_Q.to_string(),
2039
+ )
2040
+ .resp)
2041
+ });
2042
+ mock_client
2043
+ .expect_complete_workflow_task()
2044
+ .returning(move |mut c| {
2045
+ let can_cmd = c.commands.pop().unwrap().attributes.unwrap();
2046
+ if let Attributes::ContinueAsNewWorkflowExecutionCommandAttributes(a) = can_cmd {
2047
+ assert_eq!(a.workflow_type.unwrap().name, "meow");
2048
+ assert_eq!(a.memo, wes_attrs.memo);
2049
+ assert_eq!(a.search_attributes, wes_attrs.search_attributes);
2050
+ assert_eq!(a.retry_policy, wes_attrs.retry_policy);
2051
+ } else {
2052
+ panic!("Wrong attributes type");
2053
+ }
2054
+ Ok(Default::default())
2055
+ });
2056
+
2057
+ let worker = Worker::new_test(test_worker_cfg().build().unwrap(), mock_client);
2058
+ let r = worker.poll_workflow_activation().await.unwrap();
2059
+ worker
2060
+ .complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
2061
+ r.run_id,
2062
+ ContinueAsNewWorkflowExecution {
2063
+ workflow_type: "meow".to_string(),
2064
+ ..Default::default()
2065
+ }
2066
+ .into(),
2067
+ ))
2068
+ .await
2069
+ .unwrap();
2070
+ }