temporalio 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/LICENSE +20 -0
- data/README.md +130 -0
- data/bridge/Cargo.lock +2865 -0
- data/bridge/Cargo.toml +26 -0
- data/bridge/sdk-core/ARCHITECTURE.md +76 -0
- data/bridge/sdk-core/Cargo.lock +2606 -0
- data/bridge/sdk-core/Cargo.toml +2 -0
- data/bridge/sdk-core/LICENSE.txt +23 -0
- data/bridge/sdk-core/README.md +107 -0
- data/bridge/sdk-core/arch_docs/diagrams/README.md +10 -0
- data/bridge/sdk-core/arch_docs/diagrams/sticky_queues.puml +40 -0
- data/bridge/sdk-core/arch_docs/diagrams/workflow_internals.svg +1 -0
- data/bridge/sdk-core/arch_docs/sticky_queues.md +51 -0
- data/bridge/sdk-core/bridge-ffi/Cargo.toml +24 -0
- data/bridge/sdk-core/bridge-ffi/LICENSE.txt +23 -0
- data/bridge/sdk-core/bridge-ffi/build.rs +25 -0
- data/bridge/sdk-core/bridge-ffi/include/sdk-core-bridge.h +249 -0
- data/bridge/sdk-core/bridge-ffi/src/lib.rs +825 -0
- data/bridge/sdk-core/bridge-ffi/src/wrappers.rs +211 -0
- data/bridge/sdk-core/client/Cargo.toml +40 -0
- data/bridge/sdk-core/client/LICENSE.txt +23 -0
- data/bridge/sdk-core/client/src/lib.rs +1294 -0
- data/bridge/sdk-core/client/src/metrics.rs +165 -0
- data/bridge/sdk-core/client/src/raw.rs +931 -0
- data/bridge/sdk-core/client/src/retry.rs +674 -0
- data/bridge/sdk-core/client/src/workflow_handle/mod.rs +185 -0
- data/bridge/sdk-core/core/Cargo.toml +116 -0
- data/bridge/sdk-core/core/LICENSE.txt +23 -0
- data/bridge/sdk-core/core/benches/workflow_replay.rs +73 -0
- data/bridge/sdk-core/core/src/abstractions.rs +166 -0
- data/bridge/sdk-core/core/src/core_tests/activity_tasks.rs +911 -0
- data/bridge/sdk-core/core/src/core_tests/child_workflows.rs +221 -0
- data/bridge/sdk-core/core/src/core_tests/determinism.rs +107 -0
- data/bridge/sdk-core/core/src/core_tests/local_activities.rs +515 -0
- data/bridge/sdk-core/core/src/core_tests/mod.rs +100 -0
- data/bridge/sdk-core/core/src/core_tests/queries.rs +736 -0
- data/bridge/sdk-core/core/src/core_tests/replay_flag.rs +65 -0
- data/bridge/sdk-core/core/src/core_tests/workers.rs +259 -0
- data/bridge/sdk-core/core/src/core_tests/workflow_cancels.rs +124 -0
- data/bridge/sdk-core/core/src/core_tests/workflow_tasks.rs +2070 -0
- data/bridge/sdk-core/core/src/ephemeral_server/mod.rs +515 -0
- data/bridge/sdk-core/core/src/lib.rs +175 -0
- data/bridge/sdk-core/core/src/log_export.rs +62 -0
- data/bridge/sdk-core/core/src/pollers/mod.rs +54 -0
- data/bridge/sdk-core/core/src/pollers/poll_buffer.rs +297 -0
- data/bridge/sdk-core/core/src/protosext/mod.rs +428 -0
- data/bridge/sdk-core/core/src/replay/mod.rs +71 -0
- data/bridge/sdk-core/core/src/retry_logic.rs +202 -0
- data/bridge/sdk-core/core/src/telemetry/metrics.rs +383 -0
- data/bridge/sdk-core/core/src/telemetry/mod.rs +412 -0
- data/bridge/sdk-core/core/src/telemetry/prometheus_server.rs +77 -0
- data/bridge/sdk-core/core/src/test_help/mod.rs +875 -0
- data/bridge/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +580 -0
- data/bridge/sdk-core/core/src/worker/activities/local_activities.rs +1042 -0
- data/bridge/sdk-core/core/src/worker/activities.rs +464 -0
- data/bridge/sdk-core/core/src/worker/client/mocks.rs +87 -0
- data/bridge/sdk-core/core/src/worker/client.rs +347 -0
- data/bridge/sdk-core/core/src/worker/mod.rs +566 -0
- data/bridge/sdk-core/core/src/worker/workflow/bridge.rs +37 -0
- data/bridge/sdk-core/core/src/worker/workflow/driven_workflow.rs +110 -0
- data/bridge/sdk-core/core/src/worker/workflow/history_update.rs +458 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs +911 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_external_state_machine.rs +298 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_workflow_state_machine.rs +171 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +860 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/complete_workflow_state_machine.rs +140 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs +161 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/fail_workflow_state_machine.rs +133 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs +1448 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/mod.rs +342 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/mutable_side_effect_state_machine.rs +127 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/patch_state_machine.rs +712 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/side_effect_state_machine.rs +71 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/signal_external_state_machine.rs +443 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/timer_state_machine.rs +439 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs +169 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs +246 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines/local_acts.rs +96 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +1184 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_task_state_machine.rs +277 -0
- data/bridge/sdk-core/core/src/worker/workflow/managed_run/managed_wf_test.rs +198 -0
- data/bridge/sdk-core/core/src/worker/workflow/managed_run.rs +647 -0
- data/bridge/sdk-core/core/src/worker/workflow/mod.rs +1143 -0
- data/bridge/sdk-core/core/src/worker/workflow/run_cache.rs +145 -0
- data/bridge/sdk-core/core/src/worker/workflow/wft_poller.rs +88 -0
- data/bridge/sdk-core/core/src/worker/workflow/workflow_stream.rs +940 -0
- data/bridge/sdk-core/core-api/Cargo.toml +31 -0
- data/bridge/sdk-core/core-api/LICENSE.txt +23 -0
- data/bridge/sdk-core/core-api/src/errors.rs +95 -0
- data/bridge/sdk-core/core-api/src/lib.rs +151 -0
- data/bridge/sdk-core/core-api/src/worker.rs +135 -0
- data/bridge/sdk-core/etc/deps.svg +187 -0
- data/bridge/sdk-core/etc/dynamic-config.yaml +2 -0
- data/bridge/sdk-core/etc/otel-collector-config.yaml +36 -0
- data/bridge/sdk-core/etc/prometheus.yaml +6 -0
- data/bridge/sdk-core/fsm/Cargo.toml +18 -0
- data/bridge/sdk-core/fsm/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/README.md +3 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +27 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +647 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/progress.rs +8 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.rs +18 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.stderr +12 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dynamic_dest_pass.rs +41 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.rs +14 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.stderr +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_arg_pass.rs +32 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_pass.rs +31 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/medium_complex_pass.rs +46 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.rs +29 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +12 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/simple_pass.rs +32 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.rs +18 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.rs +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.rs +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/Cargo.toml +14 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/src/lib.rs +249 -0
- data/bridge/sdk-core/fsm/src/lib.rs +2 -0
- data/bridge/sdk-core/histories/fail_wf_task.bin +0 -0
- data/bridge/sdk-core/histories/timer_workflow_history.bin +0 -0
- data/bridge/sdk-core/integ-with-otel.sh +7 -0
- data/bridge/sdk-core/protos/api_upstream/README.md +9 -0
- data/bridge/sdk-core/protos/api_upstream/api-linter.yaml +40 -0
- data/bridge/sdk-core/protos/api_upstream/buf.yaml +12 -0
- data/bridge/sdk-core/protos/api_upstream/dependencies/gogoproto/gogo.proto +141 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/batch/v1/message.proto +86 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/cluster/v1/message.proto +83 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto +259 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/common/v1/message.proto +112 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +46 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/cluster.proto +40 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/command_type.proto +57 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/common.proto +55 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/event_type.proto +168 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto +97 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/namespace.proto +51 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/query.proto +50 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/reset.proto +41 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/schedule.proto +60 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/task_queue.proto +59 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/update.proto +51 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/workflow.proto +122 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/errordetails/v1/message.proto +108 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/failure/v1/message.proto +114 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/filter/v1/message.proto +56 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto +751 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/namespace/v1/message.proto +97 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +161 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +99 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/query/v1/message.proto +61 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/replication/v1/message.proto +55 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/schedule/v1/message.proto +300 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +108 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/update/v1/message.proto +46 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/version/v1/message.proto +59 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflow/v1/message.proto +145 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +1124 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +401 -0
- data/bridge/sdk-core/protos/grpc/health/v1/health.proto +63 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_result/activity_result.proto +78 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_task/activity_task.proto +79 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/bridge/bridge.proto +210 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/child_workflow/child_workflow.proto +77 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/common/common.proto +15 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/core_interface.proto +30 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/external_data/external_data.proto +30 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +261 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +297 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto +29 -0
- data/bridge/sdk-core/protos/testsrv_upstream/api-linter.yaml +38 -0
- data/bridge/sdk-core/protos/testsrv_upstream/buf.yaml +13 -0
- data/bridge/sdk-core/protos/testsrv_upstream/dependencies/gogoproto/gogo.proto +141 -0
- data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/request_response.proto +63 -0
- data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/service.proto +90 -0
- data/bridge/sdk-core/rustfmt.toml +1 -0
- data/bridge/sdk-core/sdk/Cargo.toml +47 -0
- data/bridge/sdk-core/sdk/LICENSE.txt +23 -0
- data/bridge/sdk-core/sdk/src/activity_context.rs +230 -0
- data/bridge/sdk-core/sdk/src/app_data.rs +37 -0
- data/bridge/sdk-core/sdk/src/conversions.rs +8 -0
- data/bridge/sdk-core/sdk/src/interceptors.rs +17 -0
- data/bridge/sdk-core/sdk/src/lib.rs +792 -0
- data/bridge/sdk-core/sdk/src/payload_converter.rs +11 -0
- data/bridge/sdk-core/sdk/src/workflow_context/options.rs +295 -0
- data/bridge/sdk-core/sdk/src/workflow_context.rs +683 -0
- data/bridge/sdk-core/sdk/src/workflow_future.rs +503 -0
- data/bridge/sdk-core/sdk-core-protos/Cargo.toml +30 -0
- data/bridge/sdk-core/sdk-core-protos/LICENSE.txt +23 -0
- data/bridge/sdk-core/sdk-core-protos/build.rs +108 -0
- data/bridge/sdk-core/sdk-core-protos/src/constants.rs +7 -0
- data/bridge/sdk-core/sdk-core-protos/src/history_builder.rs +497 -0
- data/bridge/sdk-core/sdk-core-protos/src/history_info.rs +230 -0
- data/bridge/sdk-core/sdk-core-protos/src/lib.rs +1910 -0
- data/bridge/sdk-core/sdk-core-protos/src/task_token.rs +38 -0
- data/bridge/sdk-core/sdk-core-protos/src/utilities.rs +14 -0
- data/bridge/sdk-core/test-utils/Cargo.toml +35 -0
- data/bridge/sdk-core/test-utils/src/canned_histories.rs +1579 -0
- data/bridge/sdk-core/test-utils/src/histfetch.rs +28 -0
- data/bridge/sdk-core/test-utils/src/lib.rs +598 -0
- data/bridge/sdk-core/tests/integ_tests/client_tests.rs +36 -0
- data/bridge/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +128 -0
- data/bridge/sdk-core/tests/integ_tests/heartbeat_tests.rs +218 -0
- data/bridge/sdk-core/tests/integ_tests/polling_tests.rs +146 -0
- data/bridge/sdk-core/tests/integ_tests/queries_tests.rs +437 -0
- data/bridge/sdk-core/tests/integ_tests/visibility_tests.rs +93 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/activities.rs +878 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/appdata_propagation.rs +61 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_external.rs +59 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +58 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +50 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs +60 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +54 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +634 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/patches.rs +113 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/replay.rs +137 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/resets.rs +93 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/signals.rs +167 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +99 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/timers.rs +131 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +75 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests.rs +587 -0
- data/bridge/sdk-core/tests/load_tests.rs +191 -0
- data/bridge/sdk-core/tests/main.rs +111 -0
- data/bridge/sdk-core/tests/runner.rs +93 -0
- data/bridge/src/connection.rs +167 -0
- data/bridge/src/lib.rs +180 -0
- data/bridge/src/runtime.rs +47 -0
- data/bridge/src/worker.rs +73 -0
- data/ext/Rakefile +9 -0
- data/lib/bridge.so +0 -0
- data/lib/gen/dependencies/gogoproto/gogo_pb.rb +14 -0
- data/lib/gen/temporal/api/batch/v1/message_pb.rb +48 -0
- data/lib/gen/temporal/api/cluster/v1/message_pb.rb +67 -0
- data/lib/gen/temporal/api/command/v1/message_pb.rb +166 -0
- data/lib/gen/temporal/api/common/v1/message_pb.rb +69 -0
- data/lib/gen/temporal/api/enums/v1/batch_operation_pb.rb +32 -0
- data/lib/gen/temporal/api/enums/v1/cluster_pb.rb +26 -0
- data/lib/gen/temporal/api/enums/v1/command_type_pb.rb +37 -0
- data/lib/gen/temporal/api/enums/v1/common_pb.rb +41 -0
- data/lib/gen/temporal/api/enums/v1/event_type_pb.rb +67 -0
- data/lib/gen/temporal/api/enums/v1/failed_cause_pb.rb +71 -0
- data/lib/gen/temporal/api/enums/v1/namespace_pb.rb +37 -0
- data/lib/gen/temporal/api/enums/v1/query_pb.rb +31 -0
- data/lib/gen/temporal/api/enums/v1/reset_pb.rb +24 -0
- data/lib/gen/temporal/api/enums/v1/schedule_pb.rb +28 -0
- data/lib/gen/temporal/api/enums/v1/task_queue_pb.rb +30 -0
- data/lib/gen/temporal/api/enums/v1/update_pb.rb +28 -0
- data/lib/gen/temporal/api/enums/v1/workflow_pb.rb +89 -0
- data/lib/gen/temporal/api/errordetails/v1/message_pb.rb +84 -0
- data/lib/gen/temporal/api/failure/v1/message_pb.rb +83 -0
- data/lib/gen/temporal/api/filter/v1/message_pb.rb +40 -0
- data/lib/gen/temporal/api/history/v1/message_pb.rb +489 -0
- data/lib/gen/temporal/api/namespace/v1/message_pb.rb +63 -0
- data/lib/gen/temporal/api/operatorservice/v1/request_response_pb.rb +125 -0
- data/lib/gen/temporal/api/operatorservice/v1/service_pb.rb +20 -0
- data/lib/gen/temporal/api/query/v1/message_pb.rb +38 -0
- data/lib/gen/temporal/api/replication/v1/message_pb.rb +37 -0
- data/lib/gen/temporal/api/schedule/v1/message_pb.rb +128 -0
- data/lib/gen/temporal/api/taskqueue/v1/message_pb.rb +73 -0
- data/lib/gen/temporal/api/update/v1/message_pb.rb +26 -0
- data/lib/gen/temporal/api/version/v1/message_pb.rb +41 -0
- data/lib/gen/temporal/api/workflow/v1/message_pb.rb +110 -0
- data/lib/gen/temporal/api/workflowservice/v1/request_response_pb.rb +771 -0
- data/lib/gen/temporal/api/workflowservice/v1/service_pb.rb +20 -0
- data/lib/gen/temporal/sdk/core/activity_result/activity_result_pb.rb +58 -0
- data/lib/gen/temporal/sdk/core/activity_task/activity_task_pb.rb +57 -0
- data/lib/gen/temporal/sdk/core/bridge/bridge_pb.rb +222 -0
- data/lib/gen/temporal/sdk/core/child_workflow/child_workflow_pb.rb +57 -0
- data/lib/gen/temporal/sdk/core/common/common_pb.rb +22 -0
- data/lib/gen/temporal/sdk/core/core_interface_pb.rb +34 -0
- data/lib/gen/temporal/sdk/core/external_data/external_data_pb.rb +27 -0
- data/lib/gen/temporal/sdk/core/workflow_activation/workflow_activation_pb.rb +164 -0
- data/lib/gen/temporal/sdk/core/workflow_commands/workflow_commands_pb.rb +192 -0
- data/lib/gen/temporal/sdk/core/workflow_completion/workflow_completion_pb.rb +34 -0
- data/lib/temporal/bridge.rb +14 -0
- data/lib/temporal/client/implementation.rb +339 -0
- data/lib/temporal/client/workflow_handle.rb +243 -0
- data/lib/temporal/client.rb +144 -0
- data/lib/temporal/connection.rb +736 -0
- data/lib/temporal/data_converter.rb +150 -0
- data/lib/temporal/error/failure.rb +194 -0
- data/lib/temporal/error/workflow_failure.rb +17 -0
- data/lib/temporal/errors.rb +22 -0
- data/lib/temporal/failure_converter/base.rb +26 -0
- data/lib/temporal/failure_converter/basic.rb +313 -0
- data/lib/temporal/failure_converter.rb +8 -0
- data/lib/temporal/interceptor/chain.rb +27 -0
- data/lib/temporal/interceptor/client.rb +102 -0
- data/lib/temporal/payload_codec/base.rb +32 -0
- data/lib/temporal/payload_converter/base.rb +24 -0
- data/lib/temporal/payload_converter/bytes.rb +26 -0
- data/lib/temporal/payload_converter/composite.rb +47 -0
- data/lib/temporal/payload_converter/encoding_base.rb +35 -0
- data/lib/temporal/payload_converter/json.rb +25 -0
- data/lib/temporal/payload_converter/nil.rb +25 -0
- data/lib/temporal/payload_converter.rb +14 -0
- data/lib/temporal/retry_policy.rb +82 -0
- data/lib/temporal/retry_state.rb +35 -0
- data/lib/temporal/runtime.rb +22 -0
- data/lib/temporal/timeout_type.rb +29 -0
- data/lib/temporal/version.rb +3 -0
- data/lib/temporal/workflow/execution_info.rb +54 -0
- data/lib/temporal/workflow/execution_status.rb +36 -0
- data/lib/temporal/workflow/id_reuse_policy.rb +36 -0
- data/lib/temporal/workflow/query_reject_condition.rb +33 -0
- data/lib/temporal.rb +8 -0
- data/lib/temporalio.rb +3 -0
- data/lib/thermite_patch.rb +23 -0
- data/temporalio.gemspec +41 -0
- metadata +583 -0
@@ -0,0 +1,1042 @@
|
|
1
|
+
use crate::{
|
2
|
+
abstractions::{MeteredSemaphore, OwnedMeteredSemPermit},
|
3
|
+
protosext::ValidScheduleLA,
|
4
|
+
retry_logic::RetryPolicyExt,
|
5
|
+
MetricsContext, TaskToken,
|
6
|
+
};
|
7
|
+
use parking_lot::Mutex;
|
8
|
+
use std::{
|
9
|
+
collections::HashMap,
|
10
|
+
fmt::{Debug, Formatter},
|
11
|
+
time::{Duration, Instant, SystemTime},
|
12
|
+
};
|
13
|
+
use temporal_sdk_core_protos::{
|
14
|
+
coresdk::{
|
15
|
+
activity_result::{Cancellation, Failure as ActFail, Success},
|
16
|
+
activity_task::{activity_task, ActivityCancelReason, ActivityTask, Cancel, Start},
|
17
|
+
},
|
18
|
+
temporal::api::{common::v1::WorkflowExecution, enums::v1::TimeoutType},
|
19
|
+
};
|
20
|
+
use tokio::{
|
21
|
+
sync::{
|
22
|
+
mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
|
23
|
+
Notify,
|
24
|
+
},
|
25
|
+
task::JoinHandle,
|
26
|
+
time::sleep,
|
27
|
+
};
|
28
|
+
use tokio_util::sync::CancellationToken;
|
29
|
+
|
30
|
+
#[allow(clippy::large_enum_variant)] // Timeouts are relatively rare
|
31
|
+
#[derive(Debug)]
|
32
|
+
pub(crate) enum DispatchOrTimeoutLA {
|
33
|
+
/// Send the activity task to lang
|
34
|
+
Dispatch(ActivityTask),
|
35
|
+
/// Notify the machines (and maybe lang) that this LA has timed out
|
36
|
+
Timeout {
|
37
|
+
run_id: String,
|
38
|
+
resolution: LocalActivityResolution,
|
39
|
+
task: Option<ActivityTask>,
|
40
|
+
},
|
41
|
+
}
|
42
|
+
|
43
|
+
#[derive(Debug)]
|
44
|
+
pub(crate) struct LocalInFlightActInfo {
|
45
|
+
pub la_info: NewLocalAct,
|
46
|
+
pub dispatch_time: Instant,
|
47
|
+
pub attempt: u32,
|
48
|
+
_permit: OwnedMeteredSemPermit,
|
49
|
+
}
|
50
|
+
|
51
|
+
#[derive(Debug, Clone)]
|
52
|
+
pub(crate) enum LocalActivityExecutionResult {
|
53
|
+
Completed(Success),
|
54
|
+
Failed(ActFail),
|
55
|
+
TimedOut(ActFail),
|
56
|
+
Cancelled(Cancellation),
|
57
|
+
}
|
58
|
+
impl LocalActivityExecutionResult {
|
59
|
+
pub(crate) fn empty_cancel() -> Self {
|
60
|
+
Self::Cancelled(Cancellation::from_details(None))
|
61
|
+
}
|
62
|
+
pub(crate) fn timeout(tt: TimeoutType) -> Self {
|
63
|
+
Self::TimedOut(ActFail::timeout(tt))
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
#[derive(Debug, Clone)]
|
68
|
+
pub(crate) struct LocalActivityResolution {
|
69
|
+
pub seq: u32,
|
70
|
+
pub result: LocalActivityExecutionResult,
|
71
|
+
pub runtime: Duration,
|
72
|
+
pub attempt: u32,
|
73
|
+
pub backoff: Option<prost_types::Duration>,
|
74
|
+
pub original_schedule_time: Option<SystemTime>,
|
75
|
+
}
|
76
|
+
|
77
|
+
#[derive(Clone)]
|
78
|
+
pub(crate) struct NewLocalAct {
|
79
|
+
pub schedule_cmd: ValidScheduleLA,
|
80
|
+
pub workflow_type: String,
|
81
|
+
pub workflow_exec_info: WorkflowExecution,
|
82
|
+
pub schedule_time: SystemTime,
|
83
|
+
}
|
84
|
+
impl Debug for NewLocalAct {
|
85
|
+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
86
|
+
write!(
|
87
|
+
f,
|
88
|
+
"LocalActivity({}, {})",
|
89
|
+
self.schedule_cmd.seq, self.schedule_cmd.activity_type
|
90
|
+
)
|
91
|
+
}
|
92
|
+
}
|
93
|
+
|
94
|
+
#[derive(Debug, derive_more::From)]
|
95
|
+
#[allow(clippy::large_enum_variant)]
|
96
|
+
pub(crate) enum LocalActRequest {
|
97
|
+
New(NewLocalAct),
|
98
|
+
Cancel(ExecutingLAId),
|
99
|
+
}
|
100
|
+
|
101
|
+
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
|
102
|
+
pub(crate) struct ExecutingLAId {
|
103
|
+
pub run_id: String,
|
104
|
+
pub seq_num: u32,
|
105
|
+
}
|
106
|
+
|
107
|
+
pub(crate) struct LocalActivityManager {
|
108
|
+
/// Just so we can provide activity tasks the same namespace as the worker
|
109
|
+
namespace: String,
|
110
|
+
/// Constrains number of currently executing local activities
|
111
|
+
semaphore: MeteredSemaphore,
|
112
|
+
/// Sink for new activity execution requests
|
113
|
+
act_req_tx: UnboundedSender<NewOrRetry>,
|
114
|
+
/// Cancels need a different queue since they should be taken first, and don't take a permit
|
115
|
+
cancels_req_tx: UnboundedSender<CancelOrTimeout>,
|
116
|
+
/// Wakes every time a complete is processed
|
117
|
+
complete_notify: Notify,
|
118
|
+
|
119
|
+
rcvs: tokio::sync::Mutex<RcvChans>,
|
120
|
+
shutdown_complete_tok: CancellationToken,
|
121
|
+
dat: Mutex<LAMData>,
|
122
|
+
}
|
123
|
+
|
124
|
+
struct LAMData {
|
125
|
+
/// Activities that have been issued to lang but not yet completed
|
126
|
+
outstanding_activity_tasks: HashMap<TaskToken, LocalInFlightActInfo>,
|
127
|
+
id_to_tt: HashMap<ExecutingLAId, TaskToken>,
|
128
|
+
/// Tasks for activities which are currently backing off. May be used to cancel retrying them.
|
129
|
+
backing_off_tasks: HashMap<ExecutingLAId, JoinHandle<()>>,
|
130
|
+
/// Tasks for timing out activities which are currently in the queue or dispatched.
|
131
|
+
timeout_tasks: HashMap<ExecutingLAId, TimeoutBag>,
|
132
|
+
next_tt_num: u32,
|
133
|
+
}
|
134
|
+
|
135
|
+
impl LAMData {
|
136
|
+
fn gen_next_token(&mut self) -> TaskToken {
|
137
|
+
self.next_tt_num += 1;
|
138
|
+
TaskToken::new_local_activity_token(self.next_tt_num.to_le_bytes())
|
139
|
+
}
|
140
|
+
}
|
141
|
+
|
142
|
+
impl LocalActivityManager {
|
143
|
+
pub(crate) fn new(
|
144
|
+
max_concurrent: usize,
|
145
|
+
namespace: String,
|
146
|
+
metrics_context: MetricsContext,
|
147
|
+
) -> Self {
|
148
|
+
let (act_req_tx, act_req_rx) = unbounded_channel();
|
149
|
+
let (cancels_req_tx, cancels_req_rx) = unbounded_channel();
|
150
|
+
let shutdown_complete_tok = CancellationToken::new();
|
151
|
+
Self {
|
152
|
+
namespace,
|
153
|
+
semaphore: MeteredSemaphore::new(
|
154
|
+
max_concurrent,
|
155
|
+
metrics_context,
|
156
|
+
MetricsContext::available_task_slots,
|
157
|
+
),
|
158
|
+
act_req_tx,
|
159
|
+
cancels_req_tx,
|
160
|
+
complete_notify: Notify::new(),
|
161
|
+
rcvs: tokio::sync::Mutex::new(RcvChans {
|
162
|
+
act_req_rx,
|
163
|
+
cancels_req_rx,
|
164
|
+
shutdown: shutdown_complete_tok.clone(),
|
165
|
+
}),
|
166
|
+
shutdown_complete_tok,
|
167
|
+
dat: Mutex::new(LAMData {
|
168
|
+
outstanding_activity_tasks: Default::default(),
|
169
|
+
id_to_tt: Default::default(),
|
170
|
+
backing_off_tasks: Default::default(),
|
171
|
+
timeout_tasks: Default::default(),
|
172
|
+
next_tt_num: 0,
|
173
|
+
}),
|
174
|
+
}
|
175
|
+
}
|
176
|
+
|
177
|
+
#[cfg(test)]
|
178
|
+
fn test(max_concurrent: usize) -> Self {
|
179
|
+
Self::new(
|
180
|
+
max_concurrent,
|
181
|
+
"fake_ns".to_string(),
|
182
|
+
MetricsContext::default(),
|
183
|
+
)
|
184
|
+
}
|
185
|
+
|
186
|
+
#[cfg(test)]
|
187
|
+
pub(crate) fn num_outstanding(&self) -> usize {
|
188
|
+
self.dat.lock().outstanding_activity_tasks.len()
|
189
|
+
}
|
190
|
+
|
191
|
+
#[cfg(test)]
|
192
|
+
fn num_in_backoff(&self) -> usize {
|
193
|
+
self.dat.lock().backing_off_tasks.len()
|
194
|
+
}
|
195
|
+
|
196
|
+
pub(crate) fn enqueue(
|
197
|
+
&self,
|
198
|
+
reqs: impl IntoIterator<Item = LocalActRequest>,
|
199
|
+
) -> Vec<LocalActivityResolution> {
|
200
|
+
let mut immediate_resolutions = vec![];
|
201
|
+
for req in reqs {
|
202
|
+
debug!(local_activity = ?req, "Queuing local activity");
|
203
|
+
match req {
|
204
|
+
LocalActRequest::New(act) => {
|
205
|
+
let id = ExecutingLAId {
|
206
|
+
run_id: act.workflow_exec_info.run_id.clone(),
|
207
|
+
seq_num: act.schedule_cmd.seq,
|
208
|
+
};
|
209
|
+
let mut dlock = self.dat.lock();
|
210
|
+
if dlock.id_to_tt.contains_key(&id) {
|
211
|
+
// Do not queue local activities which are in fact already executing.
|
212
|
+
// This can happen during evictions.
|
213
|
+
debug!("Tried to queue already-executing local activity {:?}", &id);
|
214
|
+
continue;
|
215
|
+
}
|
216
|
+
// Pre-generate and insert the task token now, before we may or may not dispatch
|
217
|
+
// the activity, so we can enforce idempotency. Prevents two identical LAs
|
218
|
+
// ending up in the queue at once.
|
219
|
+
let tt = dlock.gen_next_token();
|
220
|
+
dlock.id_to_tt.insert(id.clone(), tt);
|
221
|
+
|
222
|
+
// Set up timeouts for the new activity
|
223
|
+
match TimeoutBag::new(&act, self.cancels_req_tx.clone()) {
|
224
|
+
Ok(tb) => {
|
225
|
+
dlock.timeout_tasks.insert(id, tb);
|
226
|
+
|
227
|
+
self.act_req_tx
|
228
|
+
.send(NewOrRetry::New(act))
|
229
|
+
.expect("Receive half of LA request channel cannot be dropped");
|
230
|
+
}
|
231
|
+
Err(res) => immediate_resolutions.push(res),
|
232
|
+
}
|
233
|
+
}
|
234
|
+
LocalActRequest::Cancel(id) => {
|
235
|
+
let mut dlock = self.dat.lock();
|
236
|
+
|
237
|
+
// First check if this ID is currently backing off, if so abort the backoff
|
238
|
+
// task
|
239
|
+
if let Some(t) = dlock.backing_off_tasks.remove(&id) {
|
240
|
+
t.abort();
|
241
|
+
immediate_resolutions.push(LocalActivityResolution {
|
242
|
+
seq: id.seq_num,
|
243
|
+
result: LocalActivityExecutionResult::Cancelled(
|
244
|
+
Cancellation::from_details(None),
|
245
|
+
),
|
246
|
+
runtime: Duration::from_secs(0),
|
247
|
+
attempt: 0,
|
248
|
+
backoff: None,
|
249
|
+
original_schedule_time: None,
|
250
|
+
});
|
251
|
+
continue;
|
252
|
+
}
|
253
|
+
|
254
|
+
if let Some(tt) = dlock.id_to_tt.get(&id) {
|
255
|
+
self.cancels_req_tx
|
256
|
+
.send(CancelOrTimeout::Cancel(ActivityTask {
|
257
|
+
task_token: tt.0.clone(),
|
258
|
+
variant: Some(activity_task::Variant::Cancel(Cancel {
|
259
|
+
reason: ActivityCancelReason::Cancelled as i32,
|
260
|
+
})),
|
261
|
+
}))
|
262
|
+
.expect("Receive half of LA cancel channel cannot be dropped");
|
263
|
+
}
|
264
|
+
}
|
265
|
+
}
|
266
|
+
}
|
267
|
+
immediate_resolutions
|
268
|
+
}
|
269
|
+
|
270
|
+
/// Returns the next pending local-activity related action, or None if shutdown has initiated
|
271
|
+
/// and there are no more remaining actions to take.
|
272
|
+
pub(crate) async fn next_pending(&self) -> Option<DispatchOrTimeoutLA> {
|
273
|
+
let (new_or_retry, permit) = match self.rcvs.lock().await.next(&self.semaphore).await? {
|
274
|
+
NewOrCancel::Cancel(c) => {
|
275
|
+
return match c {
|
276
|
+
CancelOrTimeout::Cancel(c) => Some(DispatchOrTimeoutLA::Dispatch(c)),
|
277
|
+
CancelOrTimeout::Timeout {
|
278
|
+
run_id,
|
279
|
+
resolution,
|
280
|
+
dispatch_cancel,
|
281
|
+
} => {
|
282
|
+
let task = if dispatch_cancel {
|
283
|
+
let tt = self
|
284
|
+
.dat
|
285
|
+
.lock()
|
286
|
+
.id_to_tt
|
287
|
+
.get(&ExecutingLAId {
|
288
|
+
run_id: run_id.clone(),
|
289
|
+
seq_num: resolution.seq,
|
290
|
+
})
|
291
|
+
.map(Clone::clone);
|
292
|
+
if let Some(task_token) = tt {
|
293
|
+
self.complete(&task_token, &resolution.result);
|
294
|
+
Some(ActivityTask {
|
295
|
+
task_token: task_token.0,
|
296
|
+
variant: Some(activity_task::Variant::Cancel(Cancel {
|
297
|
+
reason: ActivityCancelReason::TimedOut as i32,
|
298
|
+
})),
|
299
|
+
})
|
300
|
+
} else {
|
301
|
+
None
|
302
|
+
}
|
303
|
+
} else {
|
304
|
+
None
|
305
|
+
};
|
306
|
+
Some(DispatchOrTimeoutLA::Timeout {
|
307
|
+
run_id,
|
308
|
+
resolution,
|
309
|
+
task,
|
310
|
+
})
|
311
|
+
}
|
312
|
+
};
|
313
|
+
}
|
314
|
+
NewOrCancel::New(n, perm) => (n, perm),
|
315
|
+
};
|
316
|
+
|
317
|
+
// It is important that there are no await points after receiving from the channel, as
|
318
|
+
// it would mean dropping this future would cause us to drop the activity request.
|
319
|
+
let (new_la, attempt) = match new_or_retry {
|
320
|
+
NewOrRetry::New(n) => {
|
321
|
+
let explicit_attempt_num_or_1 = n.schedule_cmd.attempt.max(1);
|
322
|
+
(n, explicit_attempt_num_or_1)
|
323
|
+
}
|
324
|
+
NewOrRetry::Retry { in_flight, attempt } => (in_flight, attempt),
|
325
|
+
};
|
326
|
+
let orig = new_la.clone();
|
327
|
+
let id = ExecutingLAId {
|
328
|
+
run_id: new_la.workflow_exec_info.run_id.clone(),
|
329
|
+
seq_num: new_la.schedule_cmd.seq,
|
330
|
+
};
|
331
|
+
let sa = new_la.schedule_cmd;
|
332
|
+
|
333
|
+
let mut dat = self.dat.lock();
|
334
|
+
// If this request originated from a local backoff task, clear the entry for it. We
|
335
|
+
// don't await the handle because we know it must already be done, and there's no
|
336
|
+
// meaningful value.
|
337
|
+
dat.backing_off_tasks.remove(&id);
|
338
|
+
|
339
|
+
// If this task sat in the queue for too long, return a timeout for it instead
|
340
|
+
if let Some(s2s) = sa.schedule_to_start_timeout.as_ref() {
|
341
|
+
let sat_for = new_la.schedule_time.elapsed().unwrap_or_default();
|
342
|
+
if sat_for > *s2s {
|
343
|
+
return Some(DispatchOrTimeoutLA::Timeout {
|
344
|
+
run_id: new_la.workflow_exec_info.run_id,
|
345
|
+
resolution: LocalActivityResolution {
|
346
|
+
seq: sa.seq,
|
347
|
+
result: LocalActivityExecutionResult::timeout(TimeoutType::ScheduleToStart),
|
348
|
+
runtime: sat_for,
|
349
|
+
attempt,
|
350
|
+
backoff: None,
|
351
|
+
original_schedule_time: Some(new_la.schedule_time),
|
352
|
+
},
|
353
|
+
task: None,
|
354
|
+
});
|
355
|
+
}
|
356
|
+
}
|
357
|
+
|
358
|
+
let tt = dat
|
359
|
+
.id_to_tt
|
360
|
+
.get(&id)
|
361
|
+
.expect("Task token must exist")
|
362
|
+
.clone();
|
363
|
+
dat.outstanding_activity_tasks.insert(
|
364
|
+
tt.clone(),
|
365
|
+
LocalInFlightActInfo {
|
366
|
+
la_info: orig,
|
367
|
+
dispatch_time: Instant::now(),
|
368
|
+
attempt,
|
369
|
+
_permit: permit,
|
370
|
+
},
|
371
|
+
);
|
372
|
+
if let Some(to) = dat.timeout_tasks.get_mut(&id) {
|
373
|
+
to.mark_started();
|
374
|
+
}
|
375
|
+
|
376
|
+
let (schedule_to_close, start_to_close) = sa.close_timeouts.into_sched_and_start();
|
377
|
+
Some(DispatchOrTimeoutLA::Dispatch(ActivityTask {
|
378
|
+
task_token: tt.0,
|
379
|
+
variant: Some(activity_task::Variant::Start(Start {
|
380
|
+
workflow_namespace: self.namespace.clone(),
|
381
|
+
workflow_type: new_la.workflow_type,
|
382
|
+
workflow_execution: Some(new_la.workflow_exec_info),
|
383
|
+
activity_id: sa.activity_id,
|
384
|
+
activity_type: sa.activity_type,
|
385
|
+
header_fields: sa.headers,
|
386
|
+
input: sa.arguments,
|
387
|
+
heartbeat_details: vec![],
|
388
|
+
scheduled_time: Some(new_la.schedule_time.into()),
|
389
|
+
current_attempt_scheduled_time: Some(new_la.schedule_time.into()),
|
390
|
+
started_time: Some(SystemTime::now().into()),
|
391
|
+
attempt,
|
392
|
+
schedule_to_close_timeout: schedule_to_close.and_then(|d| d.try_into().ok()),
|
393
|
+
start_to_close_timeout: start_to_close.and_then(|d| d.try_into().ok()),
|
394
|
+
heartbeat_timeout: None,
|
395
|
+
retry_policy: Some(sa.retry_policy),
|
396
|
+
is_local: true,
|
397
|
+
})),
|
398
|
+
}))
|
399
|
+
}
|
400
|
+
|
401
|
+
/// Mark a local activity as having completed (pass, fail, or cancelled)
|
402
|
+
pub(crate) fn complete(
|
403
|
+
&self,
|
404
|
+
task_token: &TaskToken,
|
405
|
+
status: &LocalActivityExecutionResult,
|
406
|
+
) -> LACompleteAction {
|
407
|
+
let mut dlock = self.dat.lock();
|
408
|
+
if let Some(info) = dlock.outstanding_activity_tasks.remove(task_token) {
|
409
|
+
let exec_id = ExecutingLAId {
|
410
|
+
run_id: info.la_info.workflow_exec_info.run_id.clone(),
|
411
|
+
seq_num: info.la_info.schedule_cmd.seq,
|
412
|
+
};
|
413
|
+
dlock.id_to_tt.remove(&exec_id);
|
414
|
+
|
415
|
+
match status {
|
416
|
+
LocalActivityExecutionResult::Completed(_)
|
417
|
+
| LocalActivityExecutionResult::TimedOut(_)
|
418
|
+
| LocalActivityExecutionResult::Cancelled { .. } => {
|
419
|
+
// Timeouts are included in this branch since they are not retried
|
420
|
+
self.complete_notify.notify_one();
|
421
|
+
LACompleteAction::Report(info)
|
422
|
+
}
|
423
|
+
LocalActivityExecutionResult::Failed(f) => {
|
424
|
+
if let Some(backoff_dur) = info.la_info.schedule_cmd.retry_policy.should_retry(
|
425
|
+
info.attempt as usize,
|
426
|
+
f.failure
|
427
|
+
.as_ref()
|
428
|
+
.and_then(|f| f.maybe_application_failure()),
|
429
|
+
) {
|
430
|
+
let will_use_timer =
|
431
|
+
backoff_dur > info.la_info.schedule_cmd.local_retry_threshold;
|
432
|
+
debug!(run_id = %info.la_info.workflow_exec_info.run_id,
|
433
|
+
seq_num = %info.la_info.schedule_cmd.seq,
|
434
|
+
attempt = %info.attempt,
|
435
|
+
will_use_timer,
|
436
|
+
"Local activity failed, will retry after backing off for {:?}",
|
437
|
+
backoff_dur
|
438
|
+
);
|
439
|
+
if will_use_timer {
|
440
|
+
// We want this to be reported, as the workflow will mark this
|
441
|
+
// failure down, then start a timer for backoff.
|
442
|
+
return LACompleteAction::LangDoesTimerBackoff(
|
443
|
+
backoff_dur.try_into().expect("backoff fits into proto"),
|
444
|
+
info,
|
445
|
+
);
|
446
|
+
}
|
447
|
+
// Immediately create a new task token for the to-be-retried LA
|
448
|
+
let tt = dlock.gen_next_token();
|
449
|
+
dlock.id_to_tt.insert(exec_id.clone(), tt);
|
450
|
+
|
451
|
+
// Send the retry request after waiting the backoff duration
|
452
|
+
let send_chan = self.act_req_tx.clone();
|
453
|
+
let jh = tokio::spawn(async move {
|
454
|
+
tokio::time::sleep(backoff_dur).await;
|
455
|
+
|
456
|
+
send_chan
|
457
|
+
.send(NewOrRetry::Retry {
|
458
|
+
in_flight: info.la_info,
|
459
|
+
attempt: info.attempt + 1,
|
460
|
+
})
|
461
|
+
.expect("Receive half of LA request channel cannot be dropped");
|
462
|
+
});
|
463
|
+
dlock.backing_off_tasks.insert(exec_id, jh);
|
464
|
+
|
465
|
+
LACompleteAction::WillBeRetried
|
466
|
+
} else {
|
467
|
+
LACompleteAction::Report(info)
|
468
|
+
}
|
469
|
+
}
|
470
|
+
}
|
471
|
+
} else {
|
472
|
+
LACompleteAction::Untracked
|
473
|
+
}
|
474
|
+
}
|
475
|
+
|
476
|
+
pub(crate) async fn shutdown_and_wait_all_finished(&self) {
|
477
|
+
while !self.dat.lock().outstanding_activity_tasks.is_empty() {
|
478
|
+
self.complete_notify.notified().await;
|
479
|
+
}
|
480
|
+
self.shutdown_complete_tok.cancel();
|
481
|
+
}
|
482
|
+
}
|
483
|
+
|
484
|
+
#[derive(Debug)]
|
485
|
+
#[allow(clippy::large_enum_variant)] // Most will be reported
|
486
|
+
pub(crate) enum LACompleteAction {
|
487
|
+
/// Caller should report the status to the workflow
|
488
|
+
Report(LocalInFlightActInfo),
|
489
|
+
/// Lang needs to be told to do the schedule-a-timer-then-rerun hack
|
490
|
+
LangDoesTimerBackoff(prost_types::Duration, LocalInFlightActInfo),
|
491
|
+
/// The activity will be re-enqueued for another attempt (and so status should not be reported
|
492
|
+
/// to the workflow)
|
493
|
+
WillBeRetried,
|
494
|
+
/// The activity was unknown
|
495
|
+
Untracked,
|
496
|
+
}
|
497
|
+
|
498
|
+
#[derive(Debug)]
|
499
|
+
enum NewOrRetry {
|
500
|
+
New(NewLocalAct),
|
501
|
+
Retry {
|
502
|
+
in_flight: NewLocalAct,
|
503
|
+
attempt: u32,
|
504
|
+
},
|
505
|
+
}
|
506
|
+
|
507
|
+
#[allow(clippy::large_enum_variant)]
|
508
|
+
#[derive(Debug, Clone)]
|
509
|
+
enum CancelOrTimeout {
|
510
|
+
Cancel(ActivityTask),
|
511
|
+
Timeout {
|
512
|
+
run_id: String,
|
513
|
+
resolution: LocalActivityResolution,
|
514
|
+
dispatch_cancel: bool,
|
515
|
+
},
|
516
|
+
}
|
517
|
+
|
518
|
+
enum NewOrCancel {
|
519
|
+
New(NewOrRetry, OwnedMeteredSemPermit),
|
520
|
+
Cancel(CancelOrTimeout),
|
521
|
+
}
|
522
|
+
|
523
|
+
struct RcvChans {
|
524
|
+
/// Activities that need to be executed by lang
|
525
|
+
act_req_rx: UnboundedReceiver<NewOrRetry>,
|
526
|
+
/// Cancels to send to lang or apply internally
|
527
|
+
cancels_req_rx: UnboundedReceiver<CancelOrTimeout>,
|
528
|
+
shutdown: CancellationToken,
|
529
|
+
}
|
530
|
+
|
531
|
+
impl RcvChans {
|
532
|
+
async fn next(&mut self, new_sem: &MeteredSemaphore) -> Option<NewOrCancel> {
|
533
|
+
tokio::select! {
|
534
|
+
cancel = async { self.cancels_req_rx.recv().await } => {
|
535
|
+
Some(NewOrCancel::Cancel(cancel.expect("Send halves of LA manager are not dropped")))
|
536
|
+
}
|
537
|
+
(maybe_new_or_retry, perm) = async {
|
538
|
+
// Wait for a permit to take a task and forget it. Permits are removed until a
|
539
|
+
// completion.
|
540
|
+
let perm = new_sem.acquire_owned().await.expect("is never closed");
|
541
|
+
(self.act_req_rx.recv().await, perm)
|
542
|
+
} => Some(NewOrCancel::New(
|
543
|
+
maybe_new_or_retry.expect("Send halves of LA manager are not dropped"), perm
|
544
|
+
)),
|
545
|
+
_ = self.shutdown.cancelled() => None
|
546
|
+
}
|
547
|
+
}
|
548
|
+
}
|
549
|
+
|
550
|
+
struct TimeoutBag {
|
551
|
+
sched_to_close_handle: JoinHandle<()>,
|
552
|
+
start_to_close_dur_and_dat: Option<(Duration, CancelOrTimeout)>,
|
553
|
+
start_to_close_handle: Option<JoinHandle<()>>,
|
554
|
+
cancel_chan: UnboundedSender<CancelOrTimeout>,
|
555
|
+
}
|
556
|
+
|
557
|
+
impl TimeoutBag {
|
558
|
+
/// Create new timeout tasks for the provided local activity. This must be called as soon
|
559
|
+
/// as request to schedule it arrives.
|
560
|
+
///
|
561
|
+
/// Returns error in the event the activity is *already* timed out
|
562
|
+
fn new(
|
563
|
+
new_la: &NewLocalAct,
|
564
|
+
cancel_chan: UnboundedSender<CancelOrTimeout>,
|
565
|
+
) -> Result<TimeoutBag, LocalActivityResolution> {
|
566
|
+
let (schedule_to_close, start_to_close) =
|
567
|
+
new_la.schedule_cmd.close_timeouts.into_sched_and_start();
|
568
|
+
|
569
|
+
let resolution = LocalActivityResolution {
|
570
|
+
seq: new_la.schedule_cmd.seq,
|
571
|
+
result: LocalActivityExecutionResult::timeout(TimeoutType::ScheduleToClose),
|
572
|
+
runtime: Default::default(),
|
573
|
+
attempt: new_la.schedule_cmd.attempt,
|
574
|
+
backoff: None,
|
575
|
+
original_schedule_time: Some(new_la.schedule_time),
|
576
|
+
};
|
577
|
+
// Remove any time already elapsed since the scheduling time
|
578
|
+
let schedule_to_close = schedule_to_close
|
579
|
+
.map(|s2c| s2c.saturating_sub(new_la.schedule_time.elapsed().unwrap_or_default()));
|
580
|
+
if let Some(ref s2c) = schedule_to_close {
|
581
|
+
if s2c.is_zero() {
|
582
|
+
return Err(resolution);
|
583
|
+
}
|
584
|
+
}
|
585
|
+
let timeout_dat = CancelOrTimeout::Timeout {
|
586
|
+
run_id: new_la.workflow_exec_info.run_id.clone(),
|
587
|
+
resolution,
|
588
|
+
dispatch_cancel: true,
|
589
|
+
};
|
590
|
+
let start_to_close_dur_and_dat = start_to_close.map(|d| (d, timeout_dat.clone()));
|
591
|
+
let fut_dat = schedule_to_close.map(|s2c| (s2c, timeout_dat));
|
592
|
+
|
593
|
+
let cancel_chan_clone = cancel_chan.clone();
|
594
|
+
let scheduling = tokio::spawn(async move {
|
595
|
+
if let Some((timeout, dat)) = fut_dat {
|
596
|
+
sleep(timeout).await;
|
597
|
+
cancel_chan_clone
|
598
|
+
.send(dat)
|
599
|
+
.expect("receive half not dropped");
|
600
|
+
}
|
601
|
+
});
|
602
|
+
Ok(TimeoutBag {
|
603
|
+
sched_to_close_handle: scheduling,
|
604
|
+
start_to_close_dur_and_dat,
|
605
|
+
start_to_close_handle: None,
|
606
|
+
cancel_chan,
|
607
|
+
})
|
608
|
+
}
|
609
|
+
|
610
|
+
/// Must be called once the associated local activity has been started / dispatched to lang.
|
611
|
+
fn mark_started(&mut self) {
|
612
|
+
if let Some((start_to_close, mut dat)) = self.start_to_close_dur_and_dat.take() {
|
613
|
+
let started_t = Instant::now();
|
614
|
+
let cchan = self.cancel_chan.clone();
|
615
|
+
self.start_to_close_handle = Some(tokio::spawn(async move {
|
616
|
+
sleep(start_to_close).await;
|
617
|
+
if let CancelOrTimeout::Timeout { resolution, .. } = &mut dat {
|
618
|
+
resolution.result =
|
619
|
+
LocalActivityExecutionResult::timeout(TimeoutType::StartToClose);
|
620
|
+
resolution.runtime = started_t.elapsed();
|
621
|
+
}
|
622
|
+
|
623
|
+
cchan.send(dat).expect("receive half not dropped");
|
624
|
+
}));
|
625
|
+
}
|
626
|
+
}
|
627
|
+
}
|
628
|
+
|
629
|
+
impl Drop for TimeoutBag {
|
630
|
+
fn drop(&mut self) {
|
631
|
+
self.sched_to_close_handle.abort();
|
632
|
+
if let Some(x) = self.start_to_close_handle.as_ref() {
|
633
|
+
x.abort()
|
634
|
+
}
|
635
|
+
}
|
636
|
+
}
|
637
|
+
|
638
|
+
#[cfg(test)]
|
639
|
+
mod tests {
|
640
|
+
use super::*;
|
641
|
+
use crate::{prost_dur, protosext::LACloseTimeouts};
|
642
|
+
use temporal_sdk_core_protos::temporal::api::{
|
643
|
+
common::v1::RetryPolicy,
|
644
|
+
failure::v1::{failure::FailureInfo, ApplicationFailureInfo, Failure},
|
645
|
+
};
|
646
|
+
use tokio::{sync::mpsc::error::TryRecvError, task::yield_now};
|
647
|
+
|
648
|
+
impl DispatchOrTimeoutLA {
|
649
|
+
fn unwrap(self) -> ActivityTask {
|
650
|
+
match self {
|
651
|
+
DispatchOrTimeoutLA::Dispatch(t) => t,
|
652
|
+
DispatchOrTimeoutLA::Timeout { .. } => {
|
653
|
+
panic!("Timeout returned when expected a task")
|
654
|
+
}
|
655
|
+
}
|
656
|
+
}
|
657
|
+
}
|
658
|
+
|
659
|
+
#[tokio::test]
|
660
|
+
async fn max_concurrent_respected() {
|
661
|
+
let lam = LocalActivityManager::test(1);
|
662
|
+
lam.enqueue((1..=50).map(|i| {
|
663
|
+
NewLocalAct {
|
664
|
+
schedule_cmd: ValidScheduleLA {
|
665
|
+
seq: i,
|
666
|
+
activity_id: i.to_string(),
|
667
|
+
..Default::default()
|
668
|
+
},
|
669
|
+
workflow_type: "".to_string(),
|
670
|
+
workflow_exec_info: Default::default(),
|
671
|
+
schedule_time: SystemTime::now(),
|
672
|
+
}
|
673
|
+
.into()
|
674
|
+
}));
|
675
|
+
for i in 1..=50 {
|
676
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
677
|
+
assert_matches!(
|
678
|
+
next.variant.unwrap(),
|
679
|
+
activity_task::Variant::Start(Start {activity_id, ..})
|
680
|
+
if activity_id == i.to_string()
|
681
|
+
);
|
682
|
+
let next_tt = TaskToken(next.task_token);
|
683
|
+
let complete_branch = async {
|
684
|
+
lam.complete(
|
685
|
+
&next_tt,
|
686
|
+
&LocalActivityExecutionResult::Completed(Default::default()),
|
687
|
+
)
|
688
|
+
};
|
689
|
+
tokio::select! {
|
690
|
+
// Next call will not resolve until we complete the first
|
691
|
+
_ = lam.next_pending() => {
|
692
|
+
panic!("Branch must not be selected")
|
693
|
+
}
|
694
|
+
_ = complete_branch => {}
|
695
|
+
}
|
696
|
+
}
|
697
|
+
}
|
698
|
+
|
699
|
+
#[tokio::test]
|
700
|
+
async fn no_work_doesnt_deadlock_with_complete() {
|
701
|
+
let lam = LocalActivityManager::test(5);
|
702
|
+
lam.enqueue([NewLocalAct {
|
703
|
+
schedule_cmd: ValidScheduleLA {
|
704
|
+
seq: 1,
|
705
|
+
activity_id: 1.to_string(),
|
706
|
+
..Default::default()
|
707
|
+
},
|
708
|
+
workflow_type: "".to_string(),
|
709
|
+
workflow_exec_info: Default::default(),
|
710
|
+
schedule_time: SystemTime::now(),
|
711
|
+
}
|
712
|
+
.into()]);
|
713
|
+
|
714
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
715
|
+
let tt = TaskToken(next.task_token);
|
716
|
+
tokio::select! {
|
717
|
+
biased;
|
718
|
+
|
719
|
+
_ = lam.next_pending() => {
|
720
|
+
panic!("Complete branch must win")
|
721
|
+
}
|
722
|
+
_ = async {
|
723
|
+
// Spin until the receive lock has been grabbed by the call to pending, to ensure
|
724
|
+
// it's advanced enough
|
725
|
+
while lam.rcvs.try_lock().is_ok() { yield_now().await; }
|
726
|
+
lam.complete(&tt, &LocalActivityExecutionResult::Completed(Default::default()));
|
727
|
+
} => (),
|
728
|
+
};
|
729
|
+
}
|
730
|
+
|
731
|
+
#[tokio::test]
|
732
|
+
async fn can_cancel_in_flight() {
|
733
|
+
let lam = LocalActivityManager::test(5);
|
734
|
+
lam.enqueue([NewLocalAct {
|
735
|
+
schedule_cmd: ValidScheduleLA {
|
736
|
+
seq: 1,
|
737
|
+
activity_id: 1.to_string(),
|
738
|
+
..Default::default()
|
739
|
+
},
|
740
|
+
workflow_type: "".to_string(),
|
741
|
+
workflow_exec_info: WorkflowExecution {
|
742
|
+
workflow_id: "".to_string(),
|
743
|
+
run_id: "run_id".to_string(),
|
744
|
+
},
|
745
|
+
schedule_time: SystemTime::now(),
|
746
|
+
}
|
747
|
+
.into()]);
|
748
|
+
lam.next_pending().await.unwrap().unwrap();
|
749
|
+
|
750
|
+
lam.enqueue([LocalActRequest::Cancel(ExecutingLAId {
|
751
|
+
run_id: "run_id".to_string(),
|
752
|
+
seq_num: 1,
|
753
|
+
})]);
|
754
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
755
|
+
assert_matches!(next.variant.unwrap(), activity_task::Variant::Cancel(_));
|
756
|
+
}
|
757
|
+
|
758
|
+
#[tokio::test]
|
759
|
+
async fn respects_timer_backoff_threshold() {
|
760
|
+
let lam = LocalActivityManager::test(1);
|
761
|
+
lam.enqueue([NewLocalAct {
|
762
|
+
schedule_cmd: ValidScheduleLA {
|
763
|
+
seq: 1,
|
764
|
+
activity_id: 1.to_string(),
|
765
|
+
attempt: 5,
|
766
|
+
retry_policy: RetryPolicy {
|
767
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
768
|
+
backoff_coefficient: 10.0,
|
769
|
+
maximum_interval: Some(prost_dur!(from_secs(10))),
|
770
|
+
maximum_attempts: 10,
|
771
|
+
non_retryable_error_types: vec![],
|
772
|
+
},
|
773
|
+
local_retry_threshold: Duration::from_secs(5),
|
774
|
+
..Default::default()
|
775
|
+
},
|
776
|
+
workflow_type: "".to_string(),
|
777
|
+
workflow_exec_info: Default::default(),
|
778
|
+
schedule_time: SystemTime::now(),
|
779
|
+
}
|
780
|
+
.into()]);
|
781
|
+
|
782
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
783
|
+
let tt = TaskToken(next.task_token);
|
784
|
+
let res = lam.complete(
|
785
|
+
&tt,
|
786
|
+
&LocalActivityExecutionResult::Failed(Default::default()),
|
787
|
+
);
|
788
|
+
assert_matches!(res, LACompleteAction::LangDoesTimerBackoff(dur, info)
|
789
|
+
if dur.seconds == 10 && info.attempt == 5
|
790
|
+
)
|
791
|
+
}
|
792
|
+
|
793
|
+
#[tokio::test]
|
794
|
+
async fn respects_non_retryable_error_types() {
|
795
|
+
let lam = LocalActivityManager::test(1);
|
796
|
+
lam.enqueue([NewLocalAct {
|
797
|
+
schedule_cmd: ValidScheduleLA {
|
798
|
+
seq: 1,
|
799
|
+
activity_id: "1".to_string(),
|
800
|
+
attempt: 1,
|
801
|
+
retry_policy: RetryPolicy {
|
802
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
803
|
+
backoff_coefficient: 10.0,
|
804
|
+
maximum_interval: Some(prost_dur!(from_secs(10))),
|
805
|
+
maximum_attempts: 10,
|
806
|
+
non_retryable_error_types: vec!["TestError".to_string()],
|
807
|
+
},
|
808
|
+
local_retry_threshold: Duration::from_secs(5),
|
809
|
+
..Default::default()
|
810
|
+
},
|
811
|
+
workflow_type: "".to_string(),
|
812
|
+
workflow_exec_info: Default::default(),
|
813
|
+
schedule_time: SystemTime::now(),
|
814
|
+
}
|
815
|
+
.into()]);
|
816
|
+
|
817
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
818
|
+
let tt = TaskToken(next.task_token);
|
819
|
+
let res = lam.complete(
|
820
|
+
&tt,
|
821
|
+
&LocalActivityExecutionResult::Failed(ActFail {
|
822
|
+
failure: Some(Failure {
|
823
|
+
failure_info: Some(FailureInfo::ApplicationFailureInfo(
|
824
|
+
ApplicationFailureInfo {
|
825
|
+
r#type: "TestError".to_string(),
|
826
|
+
non_retryable: false,
|
827
|
+
..Default::default()
|
828
|
+
},
|
829
|
+
)),
|
830
|
+
..Default::default()
|
831
|
+
}),
|
832
|
+
}),
|
833
|
+
);
|
834
|
+
assert_matches!(res, LACompleteAction::Report(_));
|
835
|
+
}
|
836
|
+
|
837
|
+
#[tokio::test]
|
838
|
+
async fn can_cancel_during_local_backoff() {
|
839
|
+
let lam = LocalActivityManager::test(1);
|
840
|
+
lam.enqueue([NewLocalAct {
|
841
|
+
schedule_cmd: ValidScheduleLA {
|
842
|
+
seq: 1,
|
843
|
+
activity_id: 1.to_string(),
|
844
|
+
attempt: 5,
|
845
|
+
retry_policy: RetryPolicy {
|
846
|
+
initial_interval: Some(prost_dur!(from_secs(10))),
|
847
|
+
backoff_coefficient: 1.0,
|
848
|
+
maximum_interval: Some(prost_dur!(from_secs(10))),
|
849
|
+
maximum_attempts: 10,
|
850
|
+
non_retryable_error_types: vec![],
|
851
|
+
},
|
852
|
+
local_retry_threshold: Duration::from_secs(500),
|
853
|
+
..Default::default()
|
854
|
+
},
|
855
|
+
workflow_type: "".to_string(),
|
856
|
+
workflow_exec_info: WorkflowExecution {
|
857
|
+
workflow_id: "".to_string(),
|
858
|
+
run_id: "run_id".to_string(),
|
859
|
+
},
|
860
|
+
schedule_time: SystemTime::now(),
|
861
|
+
}
|
862
|
+
.into()]);
|
863
|
+
|
864
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
865
|
+
let tt = TaskToken(next.task_token);
|
866
|
+
lam.complete(
|
867
|
+
&tt,
|
868
|
+
&LocalActivityExecutionResult::Failed(Default::default()),
|
869
|
+
);
|
870
|
+
// Cancel the activity, which is performing local backoff
|
871
|
+
let immediate_res = lam.enqueue([LocalActRequest::Cancel(ExecutingLAId {
|
872
|
+
run_id: "run_id".to_string(),
|
873
|
+
seq_num: 1,
|
874
|
+
})]);
|
875
|
+
// It should not be present in the backoff tasks
|
876
|
+
assert_eq!(lam.num_in_backoff(), 0);
|
877
|
+
assert_eq!(lam.num_outstanding(), 0);
|
878
|
+
// It should return a resolution to cancel
|
879
|
+
assert_eq!(immediate_res.len(), 1);
|
880
|
+
assert_matches!(
|
881
|
+
immediate_res[0].result,
|
882
|
+
LocalActivityExecutionResult::Cancelled { .. }
|
883
|
+
);
|
884
|
+
}
|
885
|
+
|
886
|
+
#[tokio::test]
|
887
|
+
async fn local_backoff_clears_handle_map_when_started() {
|
888
|
+
let lam = LocalActivityManager::test(1);
|
889
|
+
lam.enqueue([NewLocalAct {
|
890
|
+
schedule_cmd: ValidScheduleLA {
|
891
|
+
seq: 1,
|
892
|
+
activity_id: 1.to_string(),
|
893
|
+
attempt: 5,
|
894
|
+
retry_policy: RetryPolicy {
|
895
|
+
initial_interval: Some(prost_dur!(from_millis(10))),
|
896
|
+
backoff_coefficient: 1.0,
|
897
|
+
..Default::default()
|
898
|
+
},
|
899
|
+
local_retry_threshold: Duration::from_secs(500),
|
900
|
+
..Default::default()
|
901
|
+
},
|
902
|
+
workflow_type: "".to_string(),
|
903
|
+
workflow_exec_info: WorkflowExecution {
|
904
|
+
workflow_id: "".to_string(),
|
905
|
+
run_id: "run_id".to_string(),
|
906
|
+
},
|
907
|
+
schedule_time: SystemTime::now(),
|
908
|
+
}
|
909
|
+
.into()]);
|
910
|
+
|
911
|
+
let next = lam.next_pending().await.unwrap().unwrap();
|
912
|
+
let tt = TaskToken(next.task_token);
|
913
|
+
lam.complete(
|
914
|
+
&tt,
|
915
|
+
&LocalActivityExecutionResult::Failed(Default::default()),
|
916
|
+
);
|
917
|
+
lam.next_pending().await.unwrap().unwrap();
|
918
|
+
assert_eq!(lam.num_in_backoff(), 0);
|
919
|
+
assert_eq!(lam.num_outstanding(), 1);
|
920
|
+
}
|
921
|
+
|
922
|
+
#[tokio::test]
|
923
|
+
async fn sched_to_start_timeout() {
|
924
|
+
let lam = LocalActivityManager::test(1);
|
925
|
+
let timeout = Duration::from_millis(100);
|
926
|
+
lam.enqueue([NewLocalAct {
|
927
|
+
schedule_cmd: ValidScheduleLA {
|
928
|
+
seq: 1,
|
929
|
+
activity_id: 1.to_string(),
|
930
|
+
attempt: 5,
|
931
|
+
retry_policy: RetryPolicy {
|
932
|
+
initial_interval: Some(prost_dur!(from_millis(10))),
|
933
|
+
backoff_coefficient: 1.0,
|
934
|
+
..Default::default()
|
935
|
+
},
|
936
|
+
local_retry_threshold: Duration::from_secs(500),
|
937
|
+
schedule_to_start_timeout: Some(timeout),
|
938
|
+
..Default::default()
|
939
|
+
},
|
940
|
+
workflow_type: "".to_string(),
|
941
|
+
workflow_exec_info: WorkflowExecution {
|
942
|
+
workflow_id: "".to_string(),
|
943
|
+
run_id: "run_id".to_string(),
|
944
|
+
},
|
945
|
+
schedule_time: SystemTime::now(),
|
946
|
+
}
|
947
|
+
.into()]);
|
948
|
+
|
949
|
+
// Wait more than the timeout before grabbing the task
|
950
|
+
sleep(timeout + Duration::from_millis(10)).await;
|
951
|
+
|
952
|
+
assert_matches!(
|
953
|
+
lam.next_pending().await.unwrap(),
|
954
|
+
DispatchOrTimeoutLA::Timeout { .. }
|
955
|
+
);
|
956
|
+
assert_eq!(lam.num_in_backoff(), 0);
|
957
|
+
assert_eq!(lam.num_outstanding(), 0);
|
958
|
+
}
|
959
|
+
|
960
|
+
#[rstest::rstest]
|
961
|
+
#[case::schedule(true)]
|
962
|
+
#[case::start(false)]
|
963
|
+
#[tokio::test]
|
964
|
+
async fn local_x_to_close_timeout(#[case] is_schedule: bool) {
|
965
|
+
let lam = LocalActivityManager::test(1);
|
966
|
+
let timeout = Duration::from_millis(100);
|
967
|
+
let close_timeouts = if is_schedule {
|
968
|
+
LACloseTimeouts::ScheduleOnly(timeout)
|
969
|
+
} else {
|
970
|
+
LACloseTimeouts::StartOnly(timeout)
|
971
|
+
};
|
972
|
+
lam.enqueue([NewLocalAct {
|
973
|
+
schedule_cmd: ValidScheduleLA {
|
974
|
+
seq: 1,
|
975
|
+
activity_id: 1.to_string(),
|
976
|
+
attempt: 5,
|
977
|
+
retry_policy: RetryPolicy {
|
978
|
+
initial_interval: Some(prost_dur!(from_millis(10))),
|
979
|
+
backoff_coefficient: 1.0,
|
980
|
+
..Default::default()
|
981
|
+
},
|
982
|
+
local_retry_threshold: Duration::from_secs(500),
|
983
|
+
close_timeouts,
|
984
|
+
..Default::default()
|
985
|
+
},
|
986
|
+
workflow_type: "".to_string(),
|
987
|
+
workflow_exec_info: WorkflowExecution {
|
988
|
+
workflow_id: "".to_string(),
|
989
|
+
run_id: "run_id".to_string(),
|
990
|
+
},
|
991
|
+
schedule_time: SystemTime::now(),
|
992
|
+
}
|
993
|
+
.into()]);
|
994
|
+
|
995
|
+
lam.next_pending().await.unwrap().unwrap();
|
996
|
+
assert_eq!(lam.num_in_backoff(), 0);
|
997
|
+
assert_eq!(lam.num_outstanding(), 1);
|
998
|
+
|
999
|
+
sleep(timeout + Duration::from_millis(10)).await;
|
1000
|
+
assert_matches!(
|
1001
|
+
lam.next_pending().await.unwrap(),
|
1002
|
+
DispatchOrTimeoutLA::Timeout { .. }
|
1003
|
+
);
|
1004
|
+
assert_eq!(lam.num_outstanding(), 0);
|
1005
|
+
}
|
1006
|
+
|
1007
|
+
#[tokio::test]
|
1008
|
+
async fn idempotency_enforced() {
|
1009
|
+
let lam = LocalActivityManager::test(10);
|
1010
|
+
let new_la = NewLocalAct {
|
1011
|
+
schedule_cmd: ValidScheduleLA {
|
1012
|
+
seq: 1,
|
1013
|
+
activity_id: 1.to_string(),
|
1014
|
+
..Default::default()
|
1015
|
+
},
|
1016
|
+
workflow_type: "".to_string(),
|
1017
|
+
workflow_exec_info: WorkflowExecution {
|
1018
|
+
workflow_id: "".to_string(),
|
1019
|
+
run_id: "run_id".to_string(),
|
1020
|
+
},
|
1021
|
+
schedule_time: SystemTime::now(),
|
1022
|
+
};
|
1023
|
+
// Verify only one will get queued
|
1024
|
+
lam.enqueue([new_la.clone().into(), new_la.clone().into()]);
|
1025
|
+
lam.next_pending().await.unwrap().unwrap();
|
1026
|
+
assert_eq!(lam.num_outstanding(), 1);
|
1027
|
+
// There should be nothing else in the queue
|
1028
|
+
assert_eq!(
|
1029
|
+
lam.rcvs.lock().await.act_req_rx.try_recv().unwrap_err(),
|
1030
|
+
TryRecvError::Empty
|
1031
|
+
);
|
1032
|
+
|
1033
|
+
// Verify that if we now enqueue the same act again, after the task is outstanding, we still
|
1034
|
+
// don't add it.
|
1035
|
+
lam.enqueue([new_la.into()]);
|
1036
|
+
assert_eq!(lam.num_outstanding(), 1);
|
1037
|
+
assert_eq!(
|
1038
|
+
lam.rcvs.lock().await.act_req_rx.try_recv().unwrap_err(),
|
1039
|
+
TryRecvError::Empty
|
1040
|
+
);
|
1041
|
+
}
|
1042
|
+
}
|