temporalio 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +3 -0
- data/LICENSE +20 -0
- data/README.md +130 -0
- data/bridge/Cargo.lock +2865 -0
- data/bridge/Cargo.toml +26 -0
- data/bridge/sdk-core/ARCHITECTURE.md +76 -0
- data/bridge/sdk-core/Cargo.lock +2606 -0
- data/bridge/sdk-core/Cargo.toml +2 -0
- data/bridge/sdk-core/LICENSE.txt +23 -0
- data/bridge/sdk-core/README.md +107 -0
- data/bridge/sdk-core/arch_docs/diagrams/README.md +10 -0
- data/bridge/sdk-core/arch_docs/diagrams/sticky_queues.puml +40 -0
- data/bridge/sdk-core/arch_docs/diagrams/workflow_internals.svg +1 -0
- data/bridge/sdk-core/arch_docs/sticky_queues.md +51 -0
- data/bridge/sdk-core/bridge-ffi/Cargo.toml +24 -0
- data/bridge/sdk-core/bridge-ffi/LICENSE.txt +23 -0
- data/bridge/sdk-core/bridge-ffi/build.rs +25 -0
- data/bridge/sdk-core/bridge-ffi/include/sdk-core-bridge.h +249 -0
- data/bridge/sdk-core/bridge-ffi/src/lib.rs +825 -0
- data/bridge/sdk-core/bridge-ffi/src/wrappers.rs +211 -0
- data/bridge/sdk-core/client/Cargo.toml +40 -0
- data/bridge/sdk-core/client/LICENSE.txt +23 -0
- data/bridge/sdk-core/client/src/lib.rs +1294 -0
- data/bridge/sdk-core/client/src/metrics.rs +165 -0
- data/bridge/sdk-core/client/src/raw.rs +931 -0
- data/bridge/sdk-core/client/src/retry.rs +674 -0
- data/bridge/sdk-core/client/src/workflow_handle/mod.rs +185 -0
- data/bridge/sdk-core/core/Cargo.toml +116 -0
- data/bridge/sdk-core/core/LICENSE.txt +23 -0
- data/bridge/sdk-core/core/benches/workflow_replay.rs +73 -0
- data/bridge/sdk-core/core/src/abstractions.rs +166 -0
- data/bridge/sdk-core/core/src/core_tests/activity_tasks.rs +911 -0
- data/bridge/sdk-core/core/src/core_tests/child_workflows.rs +221 -0
- data/bridge/sdk-core/core/src/core_tests/determinism.rs +107 -0
- data/bridge/sdk-core/core/src/core_tests/local_activities.rs +515 -0
- data/bridge/sdk-core/core/src/core_tests/mod.rs +100 -0
- data/bridge/sdk-core/core/src/core_tests/queries.rs +736 -0
- data/bridge/sdk-core/core/src/core_tests/replay_flag.rs +65 -0
- data/bridge/sdk-core/core/src/core_tests/workers.rs +259 -0
- data/bridge/sdk-core/core/src/core_tests/workflow_cancels.rs +124 -0
- data/bridge/sdk-core/core/src/core_tests/workflow_tasks.rs +2070 -0
- data/bridge/sdk-core/core/src/ephemeral_server/mod.rs +515 -0
- data/bridge/sdk-core/core/src/lib.rs +175 -0
- data/bridge/sdk-core/core/src/log_export.rs +62 -0
- data/bridge/sdk-core/core/src/pollers/mod.rs +54 -0
- data/bridge/sdk-core/core/src/pollers/poll_buffer.rs +297 -0
- data/bridge/sdk-core/core/src/protosext/mod.rs +428 -0
- data/bridge/sdk-core/core/src/replay/mod.rs +71 -0
- data/bridge/sdk-core/core/src/retry_logic.rs +202 -0
- data/bridge/sdk-core/core/src/telemetry/metrics.rs +383 -0
- data/bridge/sdk-core/core/src/telemetry/mod.rs +412 -0
- data/bridge/sdk-core/core/src/telemetry/prometheus_server.rs +77 -0
- data/bridge/sdk-core/core/src/test_help/mod.rs +875 -0
- data/bridge/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +580 -0
- data/bridge/sdk-core/core/src/worker/activities/local_activities.rs +1042 -0
- data/bridge/sdk-core/core/src/worker/activities.rs +464 -0
- data/bridge/sdk-core/core/src/worker/client/mocks.rs +87 -0
- data/bridge/sdk-core/core/src/worker/client.rs +347 -0
- data/bridge/sdk-core/core/src/worker/mod.rs +566 -0
- data/bridge/sdk-core/core/src/worker/workflow/bridge.rs +37 -0
- data/bridge/sdk-core/core/src/worker/workflow/driven_workflow.rs +110 -0
- data/bridge/sdk-core/core/src/worker/workflow/history_update.rs +458 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs +911 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_external_state_machine.rs +298 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_workflow_state_machine.rs +171 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +860 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/complete_workflow_state_machine.rs +140 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs +161 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/fail_workflow_state_machine.rs +133 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs +1448 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/mod.rs +342 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/mutable_side_effect_state_machine.rs +127 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/patch_state_machine.rs +712 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/side_effect_state_machine.rs +71 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/signal_external_state_machine.rs +443 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/timer_state_machine.rs +439 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs +169 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs +246 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines/local_acts.rs +96 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +1184 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_task_state_machine.rs +277 -0
- data/bridge/sdk-core/core/src/worker/workflow/managed_run/managed_wf_test.rs +198 -0
- data/bridge/sdk-core/core/src/worker/workflow/managed_run.rs +647 -0
- data/bridge/sdk-core/core/src/worker/workflow/mod.rs +1143 -0
- data/bridge/sdk-core/core/src/worker/workflow/run_cache.rs +145 -0
- data/bridge/sdk-core/core/src/worker/workflow/wft_poller.rs +88 -0
- data/bridge/sdk-core/core/src/worker/workflow/workflow_stream.rs +940 -0
- data/bridge/sdk-core/core-api/Cargo.toml +31 -0
- data/bridge/sdk-core/core-api/LICENSE.txt +23 -0
- data/bridge/sdk-core/core-api/src/errors.rs +95 -0
- data/bridge/sdk-core/core-api/src/lib.rs +151 -0
- data/bridge/sdk-core/core-api/src/worker.rs +135 -0
- data/bridge/sdk-core/etc/deps.svg +187 -0
- data/bridge/sdk-core/etc/dynamic-config.yaml +2 -0
- data/bridge/sdk-core/etc/otel-collector-config.yaml +36 -0
- data/bridge/sdk-core/etc/prometheus.yaml +6 -0
- data/bridge/sdk-core/fsm/Cargo.toml +18 -0
- data/bridge/sdk-core/fsm/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/README.md +3 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +27 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +647 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/progress.rs +8 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.rs +18 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.stderr +12 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dynamic_dest_pass.rs +41 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.rs +14 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.stderr +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_arg_pass.rs +32 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_pass.rs +31 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/medium_complex_pass.rs +46 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.rs +29 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +12 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/simple_pass.rs +32 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.rs +18 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.rs +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.rs +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/Cargo.toml +14 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/src/lib.rs +249 -0
- data/bridge/sdk-core/fsm/src/lib.rs +2 -0
- data/bridge/sdk-core/histories/fail_wf_task.bin +0 -0
- data/bridge/sdk-core/histories/timer_workflow_history.bin +0 -0
- data/bridge/sdk-core/integ-with-otel.sh +7 -0
- data/bridge/sdk-core/protos/api_upstream/README.md +9 -0
- data/bridge/sdk-core/protos/api_upstream/api-linter.yaml +40 -0
- data/bridge/sdk-core/protos/api_upstream/buf.yaml +12 -0
- data/bridge/sdk-core/protos/api_upstream/dependencies/gogoproto/gogo.proto +141 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/batch/v1/message.proto +86 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/cluster/v1/message.proto +83 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto +259 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/common/v1/message.proto +112 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +46 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/cluster.proto +40 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/command_type.proto +57 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/common.proto +55 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/event_type.proto +168 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto +97 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/namespace.proto +51 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/query.proto +50 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/reset.proto +41 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/schedule.proto +60 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/task_queue.proto +59 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/update.proto +51 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/workflow.proto +122 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/errordetails/v1/message.proto +108 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/failure/v1/message.proto +114 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/filter/v1/message.proto +56 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto +751 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/namespace/v1/message.proto +97 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +161 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +99 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/query/v1/message.proto +61 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/replication/v1/message.proto +55 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/schedule/v1/message.proto +300 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +108 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/update/v1/message.proto +46 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/version/v1/message.proto +59 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflow/v1/message.proto +145 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +1124 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +401 -0
- data/bridge/sdk-core/protos/grpc/health/v1/health.proto +63 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_result/activity_result.proto +78 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_task/activity_task.proto +79 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/bridge/bridge.proto +210 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/child_workflow/child_workflow.proto +77 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/common/common.proto +15 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/core_interface.proto +30 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/external_data/external_data.proto +30 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +261 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +297 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto +29 -0
- data/bridge/sdk-core/protos/testsrv_upstream/api-linter.yaml +38 -0
- data/bridge/sdk-core/protos/testsrv_upstream/buf.yaml +13 -0
- data/bridge/sdk-core/protos/testsrv_upstream/dependencies/gogoproto/gogo.proto +141 -0
- data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/request_response.proto +63 -0
- data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/service.proto +90 -0
- data/bridge/sdk-core/rustfmt.toml +1 -0
- data/bridge/sdk-core/sdk/Cargo.toml +47 -0
- data/bridge/sdk-core/sdk/LICENSE.txt +23 -0
- data/bridge/sdk-core/sdk/src/activity_context.rs +230 -0
- data/bridge/sdk-core/sdk/src/app_data.rs +37 -0
- data/bridge/sdk-core/sdk/src/conversions.rs +8 -0
- data/bridge/sdk-core/sdk/src/interceptors.rs +17 -0
- data/bridge/sdk-core/sdk/src/lib.rs +792 -0
- data/bridge/sdk-core/sdk/src/payload_converter.rs +11 -0
- data/bridge/sdk-core/sdk/src/workflow_context/options.rs +295 -0
- data/bridge/sdk-core/sdk/src/workflow_context.rs +683 -0
- data/bridge/sdk-core/sdk/src/workflow_future.rs +503 -0
- data/bridge/sdk-core/sdk-core-protos/Cargo.toml +30 -0
- data/bridge/sdk-core/sdk-core-protos/LICENSE.txt +23 -0
- data/bridge/sdk-core/sdk-core-protos/build.rs +108 -0
- data/bridge/sdk-core/sdk-core-protos/src/constants.rs +7 -0
- data/bridge/sdk-core/sdk-core-protos/src/history_builder.rs +497 -0
- data/bridge/sdk-core/sdk-core-protos/src/history_info.rs +230 -0
- data/bridge/sdk-core/sdk-core-protos/src/lib.rs +1910 -0
- data/bridge/sdk-core/sdk-core-protos/src/task_token.rs +38 -0
- data/bridge/sdk-core/sdk-core-protos/src/utilities.rs +14 -0
- data/bridge/sdk-core/test-utils/Cargo.toml +35 -0
- data/bridge/sdk-core/test-utils/src/canned_histories.rs +1579 -0
- data/bridge/sdk-core/test-utils/src/histfetch.rs +28 -0
- data/bridge/sdk-core/test-utils/src/lib.rs +598 -0
- data/bridge/sdk-core/tests/integ_tests/client_tests.rs +36 -0
- data/bridge/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +128 -0
- data/bridge/sdk-core/tests/integ_tests/heartbeat_tests.rs +218 -0
- data/bridge/sdk-core/tests/integ_tests/polling_tests.rs +146 -0
- data/bridge/sdk-core/tests/integ_tests/queries_tests.rs +437 -0
- data/bridge/sdk-core/tests/integ_tests/visibility_tests.rs +93 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/activities.rs +878 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/appdata_propagation.rs +61 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_external.rs +59 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +58 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +50 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs +60 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +54 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +634 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/patches.rs +113 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/replay.rs +137 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/resets.rs +93 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/signals.rs +167 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +99 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/timers.rs +131 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +75 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests.rs +587 -0
- data/bridge/sdk-core/tests/load_tests.rs +191 -0
- data/bridge/sdk-core/tests/main.rs +111 -0
- data/bridge/sdk-core/tests/runner.rs +93 -0
- data/bridge/src/connection.rs +167 -0
- data/bridge/src/lib.rs +180 -0
- data/bridge/src/runtime.rs +47 -0
- data/bridge/src/worker.rs +73 -0
- data/ext/Rakefile +9 -0
- data/lib/bridge.so +0 -0
- data/lib/gen/dependencies/gogoproto/gogo_pb.rb +14 -0
- data/lib/gen/temporal/api/batch/v1/message_pb.rb +48 -0
- data/lib/gen/temporal/api/cluster/v1/message_pb.rb +67 -0
- data/lib/gen/temporal/api/command/v1/message_pb.rb +166 -0
- data/lib/gen/temporal/api/common/v1/message_pb.rb +69 -0
- data/lib/gen/temporal/api/enums/v1/batch_operation_pb.rb +32 -0
- data/lib/gen/temporal/api/enums/v1/cluster_pb.rb +26 -0
- data/lib/gen/temporal/api/enums/v1/command_type_pb.rb +37 -0
- data/lib/gen/temporal/api/enums/v1/common_pb.rb +41 -0
- data/lib/gen/temporal/api/enums/v1/event_type_pb.rb +67 -0
- data/lib/gen/temporal/api/enums/v1/failed_cause_pb.rb +71 -0
- data/lib/gen/temporal/api/enums/v1/namespace_pb.rb +37 -0
- data/lib/gen/temporal/api/enums/v1/query_pb.rb +31 -0
- data/lib/gen/temporal/api/enums/v1/reset_pb.rb +24 -0
- data/lib/gen/temporal/api/enums/v1/schedule_pb.rb +28 -0
- data/lib/gen/temporal/api/enums/v1/task_queue_pb.rb +30 -0
- data/lib/gen/temporal/api/enums/v1/update_pb.rb +28 -0
- data/lib/gen/temporal/api/enums/v1/workflow_pb.rb +89 -0
- data/lib/gen/temporal/api/errordetails/v1/message_pb.rb +84 -0
- data/lib/gen/temporal/api/failure/v1/message_pb.rb +83 -0
- data/lib/gen/temporal/api/filter/v1/message_pb.rb +40 -0
- data/lib/gen/temporal/api/history/v1/message_pb.rb +489 -0
- data/lib/gen/temporal/api/namespace/v1/message_pb.rb +63 -0
- data/lib/gen/temporal/api/operatorservice/v1/request_response_pb.rb +125 -0
- data/lib/gen/temporal/api/operatorservice/v1/service_pb.rb +20 -0
- data/lib/gen/temporal/api/query/v1/message_pb.rb +38 -0
- data/lib/gen/temporal/api/replication/v1/message_pb.rb +37 -0
- data/lib/gen/temporal/api/schedule/v1/message_pb.rb +128 -0
- data/lib/gen/temporal/api/taskqueue/v1/message_pb.rb +73 -0
- data/lib/gen/temporal/api/update/v1/message_pb.rb +26 -0
- data/lib/gen/temporal/api/version/v1/message_pb.rb +41 -0
- data/lib/gen/temporal/api/workflow/v1/message_pb.rb +110 -0
- data/lib/gen/temporal/api/workflowservice/v1/request_response_pb.rb +771 -0
- data/lib/gen/temporal/api/workflowservice/v1/service_pb.rb +20 -0
- data/lib/gen/temporal/sdk/core/activity_result/activity_result_pb.rb +58 -0
- data/lib/gen/temporal/sdk/core/activity_task/activity_task_pb.rb +57 -0
- data/lib/gen/temporal/sdk/core/bridge/bridge_pb.rb +222 -0
- data/lib/gen/temporal/sdk/core/child_workflow/child_workflow_pb.rb +57 -0
- data/lib/gen/temporal/sdk/core/common/common_pb.rb +22 -0
- data/lib/gen/temporal/sdk/core/core_interface_pb.rb +34 -0
- data/lib/gen/temporal/sdk/core/external_data/external_data_pb.rb +27 -0
- data/lib/gen/temporal/sdk/core/workflow_activation/workflow_activation_pb.rb +164 -0
- data/lib/gen/temporal/sdk/core/workflow_commands/workflow_commands_pb.rb +192 -0
- data/lib/gen/temporal/sdk/core/workflow_completion/workflow_completion_pb.rb +34 -0
- data/lib/temporal/bridge.rb +14 -0
- data/lib/temporal/client/implementation.rb +339 -0
- data/lib/temporal/client/workflow_handle.rb +243 -0
- data/lib/temporal/client.rb +144 -0
- data/lib/temporal/connection.rb +736 -0
- data/lib/temporal/data_converter.rb +150 -0
- data/lib/temporal/error/failure.rb +194 -0
- data/lib/temporal/error/workflow_failure.rb +17 -0
- data/lib/temporal/errors.rb +22 -0
- data/lib/temporal/failure_converter/base.rb +26 -0
- data/lib/temporal/failure_converter/basic.rb +313 -0
- data/lib/temporal/failure_converter.rb +8 -0
- data/lib/temporal/interceptor/chain.rb +27 -0
- data/lib/temporal/interceptor/client.rb +102 -0
- data/lib/temporal/payload_codec/base.rb +32 -0
- data/lib/temporal/payload_converter/base.rb +24 -0
- data/lib/temporal/payload_converter/bytes.rb +26 -0
- data/lib/temporal/payload_converter/composite.rb +47 -0
- data/lib/temporal/payload_converter/encoding_base.rb +35 -0
- data/lib/temporal/payload_converter/json.rb +25 -0
- data/lib/temporal/payload_converter/nil.rb +25 -0
- data/lib/temporal/payload_converter.rb +14 -0
- data/lib/temporal/retry_policy.rb +82 -0
- data/lib/temporal/retry_state.rb +35 -0
- data/lib/temporal/runtime.rb +22 -0
- data/lib/temporal/timeout_type.rb +29 -0
- data/lib/temporal/version.rb +3 -0
- data/lib/temporal/workflow/execution_info.rb +54 -0
- data/lib/temporal/workflow/execution_status.rb +36 -0
- data/lib/temporal/workflow/id_reuse_policy.rb +36 -0
- data/lib/temporal/workflow/query_reject_condition.rb +33 -0
- data/lib/temporal.rb +8 -0
- data/lib/temporalio.rb +3 -0
- data/lib/thermite_patch.rb +23 -0
- data/temporalio.gemspec +41 -0
- metadata +583 -0
@@ -0,0 +1,2070 @@
|
|
1
|
+
use crate::{
|
2
|
+
advance_fut, job_assert,
|
3
|
+
replay::TestHistoryBuilder,
|
4
|
+
test_help::{
|
5
|
+
build_fake_worker, build_mock_pollers, build_multihist_mock_sg, canned_histories,
|
6
|
+
gen_assert_and_fail, gen_assert_and_reply, hist_to_poll_resp, mock_sdk, mock_sdk_cfg,
|
7
|
+
mock_worker, poll_and_reply, poll_and_reply_clears_outstanding_evicts, single_hist_mock_sg,
|
8
|
+
test_worker_cfg, FakeWfResponses, MockPollCfg, MocksHolder, ResponseType,
|
9
|
+
WorkflowCachingPolicy::{self, AfterEveryReply, NonSticky},
|
10
|
+
TEST_Q,
|
11
|
+
},
|
12
|
+
worker::client::mocks::{mock_manual_workflow_client, mock_workflow_client},
|
13
|
+
Worker,
|
14
|
+
};
|
15
|
+
use futures::{stream, FutureExt};
|
16
|
+
use rstest::{fixture, rstest};
|
17
|
+
use std::{
|
18
|
+
collections::{HashMap, VecDeque},
|
19
|
+
sync::{
|
20
|
+
atomic::{AtomicU64, Ordering},
|
21
|
+
Arc,
|
22
|
+
},
|
23
|
+
time::Duration,
|
24
|
+
};
|
25
|
+
use temporal_sdk::{ActivityOptions, CancellableFuture, WfContext};
|
26
|
+
use temporal_sdk_core_api::{errors::PollWfError, Worker as WorkerTrait};
|
27
|
+
use temporal_sdk_core_protos::{
|
28
|
+
coresdk::{
|
29
|
+
activity_result::{self as ar, activity_resolution, ActivityResolution},
|
30
|
+
workflow_activation::{
|
31
|
+
remove_from_cache::EvictionReason, workflow_activation_job, FireTimer, ResolveActivity,
|
32
|
+
StartWorkflow, UpdateRandomSeed, WorkflowActivationJob,
|
33
|
+
},
|
34
|
+
workflow_commands::{
|
35
|
+
ActivityCancellationType, CancelTimer, CompleteWorkflowExecution,
|
36
|
+
ContinueAsNewWorkflowExecution, FailWorkflowExecution, RequestCancelActivity,
|
37
|
+
ScheduleActivity,
|
38
|
+
},
|
39
|
+
workflow_completion::WorkflowActivationCompletion,
|
40
|
+
},
|
41
|
+
default_wes_attribs,
|
42
|
+
temporal::api::{
|
43
|
+
command::v1::command::Attributes,
|
44
|
+
common::v1::{Payload, RetryPolicy},
|
45
|
+
enums::v1::{EventType, WorkflowTaskFailedCause},
|
46
|
+
failure::v1::Failure,
|
47
|
+
history::v1::{history_event, TimerFiredEventAttributes},
|
48
|
+
workflowservice::v1::{
|
49
|
+
GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
|
50
|
+
},
|
51
|
+
},
|
52
|
+
DEFAULT_WORKFLOW_TYPE,
|
53
|
+
};
|
54
|
+
use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd};
|
55
|
+
use tokio::{
|
56
|
+
join,
|
57
|
+
sync::{Barrier, Semaphore},
|
58
|
+
};
|
59
|
+
|
60
|
+
#[fixture(hist_batches = &[])]
|
61
|
+
fn single_timer_setup(hist_batches: &'static [usize]) -> Worker {
|
62
|
+
let wfid = "fake_wf_id";
|
63
|
+
|
64
|
+
let t = canned_histories::single_timer("1");
|
65
|
+
build_fake_worker(wfid, t, hist_batches)
|
66
|
+
}
|
67
|
+
|
68
|
+
#[fixture(hist_batches = &[])]
|
69
|
+
fn single_activity_setup(hist_batches: &'static [usize]) -> Worker {
|
70
|
+
let wfid = "fake_wf_id";
|
71
|
+
|
72
|
+
let t = canned_histories::single_activity("fake_activity");
|
73
|
+
build_fake_worker(wfid, t, hist_batches)
|
74
|
+
}
|
75
|
+
|
76
|
+
#[fixture(hist_batches = &[])]
|
77
|
+
fn single_activity_failure_setup(hist_batches: &'static [usize]) -> Worker {
|
78
|
+
let wfid = "fake_wf_id";
|
79
|
+
|
80
|
+
let t = canned_histories::single_failed_activity("fake_activity");
|
81
|
+
build_fake_worker(wfid, t, hist_batches)
|
82
|
+
}
|
83
|
+
|
84
|
+
#[rstest]
|
85
|
+
#[case::incremental(single_timer_setup(&[1, 2]), NonSticky)]
|
86
|
+
#[case::replay(single_timer_setup(&[2]), NonSticky)]
|
87
|
+
#[case::incremental_evict(single_timer_setup(&[1, 2]), AfterEveryReply)]
|
88
|
+
#[case::replay_evict(single_timer_setup(&[2]), AfterEveryReply)]
|
89
|
+
#[tokio::test]
|
90
|
+
async fn single_timer(#[case] worker: Worker, #[case] evict: WorkflowCachingPolicy) {
|
91
|
+
poll_and_reply(
|
92
|
+
&worker,
|
93
|
+
evict,
|
94
|
+
&[
|
95
|
+
gen_assert_and_reply(
|
96
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
97
|
+
vec![start_timer_cmd(1, Duration::from_secs(1))],
|
98
|
+
),
|
99
|
+
gen_assert_and_reply(
|
100
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
101
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
102
|
+
),
|
103
|
+
],
|
104
|
+
)
|
105
|
+
.await;
|
106
|
+
}
|
107
|
+
|
108
|
+
#[rstest(worker,
|
109
|
+
case::incremental(single_activity_setup(&[1, 2])),
|
110
|
+
case::incremental_activity_failure(single_activity_failure_setup(&[1, 2])),
|
111
|
+
case::replay(single_activity_setup(&[2])),
|
112
|
+
case::replay_activity_failure(single_activity_failure_setup(&[2]))
|
113
|
+
)]
|
114
|
+
#[tokio::test]
|
115
|
+
async fn single_activity_completion(worker: Worker) {
|
116
|
+
poll_and_reply(
|
117
|
+
&worker,
|
118
|
+
NonSticky,
|
119
|
+
&[
|
120
|
+
gen_assert_and_reply(
|
121
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
122
|
+
vec![ScheduleActivity {
|
123
|
+
activity_id: "fake_activity".to_string(),
|
124
|
+
..Default::default()
|
125
|
+
}
|
126
|
+
.into()],
|
127
|
+
),
|
128
|
+
gen_assert_and_reply(
|
129
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(_)),
|
130
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
131
|
+
),
|
132
|
+
],
|
133
|
+
)
|
134
|
+
.await;
|
135
|
+
}
|
136
|
+
|
137
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
138
|
+
#[tokio::test]
|
139
|
+
async fn parallel_timer_test_across_wf_bridge(hist_batches: &'static [usize]) {
|
140
|
+
let wfid = "fake_wf_id";
|
141
|
+
let timer_1_id = 1;
|
142
|
+
let timer_2_id = 2;
|
143
|
+
|
144
|
+
let t = canned_histories::parallel_timer(
|
145
|
+
timer_1_id.to_string().as_str(),
|
146
|
+
timer_2_id.to_string().as_str(),
|
147
|
+
);
|
148
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
149
|
+
|
150
|
+
poll_and_reply(
|
151
|
+
&core,
|
152
|
+
NonSticky,
|
153
|
+
&[
|
154
|
+
gen_assert_and_reply(
|
155
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
156
|
+
vec![
|
157
|
+
start_timer_cmd(timer_1_id, Duration::from_secs(1)),
|
158
|
+
start_timer_cmd(timer_2_id, Duration::from_secs(1)),
|
159
|
+
],
|
160
|
+
),
|
161
|
+
gen_assert_and_reply(
|
162
|
+
&|res| {
|
163
|
+
assert_matches!(
|
164
|
+
res.jobs.as_slice(),
|
165
|
+
[
|
166
|
+
WorkflowActivationJob {
|
167
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(
|
168
|
+
FireTimer { seq: t1_id }
|
169
|
+
)),
|
170
|
+
},
|
171
|
+
WorkflowActivationJob {
|
172
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(
|
173
|
+
FireTimer { seq: t2_id }
|
174
|
+
)),
|
175
|
+
}
|
176
|
+
] => {
|
177
|
+
assert_eq!(t1_id, &timer_1_id);
|
178
|
+
assert_eq!(t2_id, &timer_2_id);
|
179
|
+
}
|
180
|
+
);
|
181
|
+
},
|
182
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
183
|
+
),
|
184
|
+
],
|
185
|
+
)
|
186
|
+
.await;
|
187
|
+
}
|
188
|
+
|
189
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
190
|
+
#[tokio::test]
|
191
|
+
async fn timer_cancel(hist_batches: &'static [usize]) {
|
192
|
+
let wfid = "fake_wf_id";
|
193
|
+
let timer_id = 1;
|
194
|
+
let cancel_timer_id = 2;
|
195
|
+
|
196
|
+
let t = canned_histories::cancel_timer(
|
197
|
+
timer_id.to_string().as_str(),
|
198
|
+
cancel_timer_id.to_string().as_str(),
|
199
|
+
);
|
200
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
201
|
+
|
202
|
+
poll_and_reply(
|
203
|
+
&core,
|
204
|
+
NonSticky,
|
205
|
+
&[
|
206
|
+
gen_assert_and_reply(
|
207
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
208
|
+
vec![
|
209
|
+
start_timer_cmd(cancel_timer_id, Duration::from_secs(1)),
|
210
|
+
start_timer_cmd(timer_id, Duration::from_secs(1)),
|
211
|
+
],
|
212
|
+
),
|
213
|
+
gen_assert_and_reply(
|
214
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
215
|
+
vec![
|
216
|
+
CancelTimer {
|
217
|
+
seq: cancel_timer_id,
|
218
|
+
}
|
219
|
+
.into(),
|
220
|
+
CompleteWorkflowExecution { result: None }.into(),
|
221
|
+
],
|
222
|
+
),
|
223
|
+
],
|
224
|
+
)
|
225
|
+
.await;
|
226
|
+
}
|
227
|
+
|
228
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
229
|
+
#[tokio::test]
|
230
|
+
async fn scheduled_activity_cancellation_try_cancel(hist_batches: &'static [usize]) {
|
231
|
+
let wfid = "fake_wf_id";
|
232
|
+
let activity_seq = 1;
|
233
|
+
let activity_id = "fake_activity";
|
234
|
+
let signal_id = "signal";
|
235
|
+
|
236
|
+
let t = canned_histories::cancel_scheduled_activity(activity_id, signal_id);
|
237
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
238
|
+
|
239
|
+
poll_and_reply(
|
240
|
+
&core,
|
241
|
+
NonSticky,
|
242
|
+
&[
|
243
|
+
gen_assert_and_reply(
|
244
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
245
|
+
vec![ScheduleActivity {
|
246
|
+
seq: activity_seq,
|
247
|
+
activity_id: activity_id.to_string(),
|
248
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
249
|
+
..Default::default()
|
250
|
+
}
|
251
|
+
.into()],
|
252
|
+
),
|
253
|
+
gen_assert_and_reply(
|
254
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
255
|
+
vec![RequestCancelActivity { seq: activity_seq }.into()],
|
256
|
+
),
|
257
|
+
// Activity is getting resolved right away as we are in the TryCancel mode.
|
258
|
+
gen_assert_and_reply(
|
259
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(_)),
|
260
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
261
|
+
),
|
262
|
+
],
|
263
|
+
)
|
264
|
+
.await;
|
265
|
+
}
|
266
|
+
|
267
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
268
|
+
#[tokio::test]
|
269
|
+
async fn scheduled_activity_timeout(hist_batches: &'static [usize]) {
|
270
|
+
let wfid = "fake_wf_id";
|
271
|
+
let activity_seq = 1;
|
272
|
+
let activity_id = "fake_activity";
|
273
|
+
|
274
|
+
let t = canned_histories::scheduled_activity_timeout(activity_id);
|
275
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
276
|
+
poll_and_reply(
|
277
|
+
&core,
|
278
|
+
NonSticky,
|
279
|
+
&[
|
280
|
+
gen_assert_and_reply(
|
281
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
282
|
+
vec![ScheduleActivity {
|
283
|
+
seq: activity_seq,
|
284
|
+
activity_id: activity_id.to_string(),
|
285
|
+
..Default::default()
|
286
|
+
}
|
287
|
+
.into()],
|
288
|
+
),
|
289
|
+
// Activity is getting resolved right away as it has been timed out.
|
290
|
+
gen_assert_and_reply(
|
291
|
+
&|res| {
|
292
|
+
assert_matches!(
|
293
|
+
res.jobs.as_slice(),
|
294
|
+
[
|
295
|
+
WorkflowActivationJob {
|
296
|
+
variant: Some(workflow_activation_job::Variant::ResolveActivity(
|
297
|
+
ResolveActivity {
|
298
|
+
seq,
|
299
|
+
result: Some(ActivityResolution {
|
300
|
+
status: Some(activity_resolution::Status::Failed(ar::Failure {
|
301
|
+
failure: Some(failure)
|
302
|
+
})),
|
303
|
+
})
|
304
|
+
}
|
305
|
+
)),
|
306
|
+
}
|
307
|
+
] => {
|
308
|
+
assert_eq!(failure.message, "Activity task timed out".to_string());
|
309
|
+
assert_eq!(*seq, activity_seq);
|
310
|
+
}
|
311
|
+
);
|
312
|
+
},
|
313
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
314
|
+
),
|
315
|
+
],
|
316
|
+
)
|
317
|
+
.await;
|
318
|
+
}
|
319
|
+
|
320
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
321
|
+
#[tokio::test]
|
322
|
+
async fn started_activity_timeout(hist_batches: &'static [usize]) {
|
323
|
+
let wfid = "fake_wf_id";
|
324
|
+
let activity_seq = 1;
|
325
|
+
|
326
|
+
let t = canned_histories::started_activity_timeout(activity_seq.to_string().as_str());
|
327
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
328
|
+
|
329
|
+
poll_and_reply(
|
330
|
+
&core,
|
331
|
+
NonSticky,
|
332
|
+
&[
|
333
|
+
gen_assert_and_reply(
|
334
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
335
|
+
vec![ScheduleActivity {
|
336
|
+
seq: activity_seq,
|
337
|
+
activity_id: activity_seq.to_string(),
|
338
|
+
..Default::default()
|
339
|
+
}
|
340
|
+
.into()],
|
341
|
+
),
|
342
|
+
// Activity is getting resolved right away as it has been timed out.
|
343
|
+
gen_assert_and_reply(
|
344
|
+
&|res| {
|
345
|
+
assert_matches!(
|
346
|
+
res.jobs.as_slice(),
|
347
|
+
[
|
348
|
+
WorkflowActivationJob {
|
349
|
+
variant: Some(workflow_activation_job::Variant::ResolveActivity(
|
350
|
+
ResolveActivity {
|
351
|
+
seq,
|
352
|
+
result: Some(ActivityResolution {
|
353
|
+
status: Some(activity_resolution::Status::Failed(ar::Failure {
|
354
|
+
failure: Some(failure)
|
355
|
+
})),
|
356
|
+
})
|
357
|
+
}
|
358
|
+
)),
|
359
|
+
}
|
360
|
+
] => {
|
361
|
+
assert_eq!(failure.message, "Activity task timed out".to_string());
|
362
|
+
assert_eq!(*seq, activity_seq);
|
363
|
+
}
|
364
|
+
);
|
365
|
+
},
|
366
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
367
|
+
),
|
368
|
+
],
|
369
|
+
)
|
370
|
+
.await;
|
371
|
+
}
|
372
|
+
|
373
|
+
#[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
|
374
|
+
#[tokio::test]
|
375
|
+
async fn cancelled_activity_timeout(hist_batches: &'static [usize]) {
|
376
|
+
let wfid = "fake_wf_id";
|
377
|
+
let activity_seq = 0;
|
378
|
+
let activity_id = "fake_activity";
|
379
|
+
let signal_id = "signal";
|
380
|
+
|
381
|
+
let t = canned_histories::scheduled_cancelled_activity_timeout(activity_id, signal_id);
|
382
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
383
|
+
|
384
|
+
poll_and_reply(
|
385
|
+
&core,
|
386
|
+
NonSticky,
|
387
|
+
&[
|
388
|
+
gen_assert_and_reply(
|
389
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
390
|
+
vec![ScheduleActivity {
|
391
|
+
seq: activity_seq,
|
392
|
+
activity_id: activity_id.to_string(),
|
393
|
+
..Default::default()
|
394
|
+
}
|
395
|
+
.into()],
|
396
|
+
),
|
397
|
+
gen_assert_and_reply(
|
398
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
399
|
+
vec![RequestCancelActivity { seq: activity_seq }.into()],
|
400
|
+
),
|
401
|
+
// Activity is resolved right away as it has timed out.
|
402
|
+
gen_assert_and_reply(
|
403
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
404
|
+
ResolveActivity {
|
405
|
+
seq: _,
|
406
|
+
result: Some(ActivityResolution {
|
407
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
408
|
+
})
|
409
|
+
}
|
410
|
+
)),
|
411
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
412
|
+
),
|
413
|
+
],
|
414
|
+
)
|
415
|
+
.await;
|
416
|
+
}
|
417
|
+
|
418
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
419
|
+
#[tokio::test]
|
420
|
+
async fn scheduled_activity_cancellation_abandon(hist_batches: &'static [usize]) {
|
421
|
+
let wfid = "fake_wf_id";
|
422
|
+
let activity_id = 1;
|
423
|
+
let signal_id = "signal";
|
424
|
+
|
425
|
+
let t = canned_histories::cancel_scheduled_activity_abandon(
|
426
|
+
activity_id.to_string().as_str(),
|
427
|
+
signal_id,
|
428
|
+
);
|
429
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
430
|
+
|
431
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::Abandon).await;
|
432
|
+
}
|
433
|
+
|
434
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
435
|
+
#[tokio::test]
|
436
|
+
async fn started_activity_cancellation_abandon(hist_batches: &'static [usize]) {
|
437
|
+
let wfid = "fake_wf_id";
|
438
|
+
let activity_id = 1;
|
439
|
+
let signal_id = "signal";
|
440
|
+
|
441
|
+
let t = canned_histories::cancel_started_activity_abandon(
|
442
|
+
activity_id.to_string().as_str(),
|
443
|
+
signal_id,
|
444
|
+
);
|
445
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
446
|
+
|
447
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::Abandon).await;
|
448
|
+
}
|
449
|
+
|
450
|
+
#[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
|
451
|
+
#[tokio::test]
|
452
|
+
async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [usize]) {
|
453
|
+
let wfid = "fake_wf_id";
|
454
|
+
let wf_type = DEFAULT_WORKFLOW_TYPE;
|
455
|
+
let activity_id = "1";
|
456
|
+
|
457
|
+
let mut t = TestHistoryBuilder::default();
|
458
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
459
|
+
t.add_full_wf_task();
|
460
|
+
let act_scheduled_event_id = t.add_activity_task_scheduled(activity_id);
|
461
|
+
let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
|
462
|
+
t.add_timer_fired(timer_started_event_id, "1".to_string());
|
463
|
+
t.add_full_wf_task();
|
464
|
+
let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
|
465
|
+
let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
|
466
|
+
t.add_activity_task_completed(
|
467
|
+
act_scheduled_event_id,
|
468
|
+
act_started_event_id,
|
469
|
+
Default::default(),
|
470
|
+
);
|
471
|
+
t.add_full_wf_task();
|
472
|
+
t.add_timer_fired(timer_started_event_id, "2".to_string());
|
473
|
+
t.add_full_wf_task();
|
474
|
+
t.add_workflow_execution_completed();
|
475
|
+
let mock = mock_workflow_client();
|
476
|
+
let mut worker = mock_sdk(MockPollCfg::from_resp_batches(wfid, t, hist_batches, mock));
|
477
|
+
|
478
|
+
worker.register_wf(wf_type.to_owned(), |ctx: WfContext| async move {
|
479
|
+
let act_fut = ctx.activity(ActivityOptions {
|
480
|
+
activity_type: "echo_activity".to_string(),
|
481
|
+
start_to_close_timeout: Some(Duration::from_secs(5)),
|
482
|
+
cancellation_type: ActivityCancellationType::Abandon,
|
483
|
+
..Default::default()
|
484
|
+
});
|
485
|
+
ctx.timer(Duration::from_secs(1)).await;
|
486
|
+
act_fut.cancel(&ctx);
|
487
|
+
ctx.timer(Duration::from_secs(3)).await;
|
488
|
+
act_fut.await;
|
489
|
+
Ok(().into())
|
490
|
+
});
|
491
|
+
worker
|
492
|
+
.submit_wf(wfid, wf_type, vec![], Default::default())
|
493
|
+
.await
|
494
|
+
.unwrap();
|
495
|
+
worker.run_until_done().await.unwrap();
|
496
|
+
}
|
497
|
+
|
498
|
+
#[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
|
499
|
+
#[tokio::test]
|
500
|
+
async fn scheduled_activity_cancellation_try_cancel_task_canceled(hist_batches: &'static [usize]) {
|
501
|
+
let wfid = "fake_wf_id";
|
502
|
+
let activity_id = 1;
|
503
|
+
let signal_id = "signal";
|
504
|
+
|
505
|
+
let t = canned_histories::cancel_scheduled_activity_with_activity_task_cancel(
|
506
|
+
activity_id.to_string().as_str(),
|
507
|
+
signal_id,
|
508
|
+
);
|
509
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
510
|
+
|
511
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::TryCancel).await;
|
512
|
+
}
|
513
|
+
|
514
|
+
#[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
|
515
|
+
#[tokio::test]
|
516
|
+
async fn started_activity_cancellation_try_cancel_task_canceled(hist_batches: &'static [usize]) {
|
517
|
+
let wfid = "fake_wf_id";
|
518
|
+
let activity_id = 1;
|
519
|
+
let signal_id = "signal";
|
520
|
+
|
521
|
+
let t = canned_histories::cancel_started_activity_with_activity_task_cancel(
|
522
|
+
activity_id.to_string().as_str(),
|
523
|
+
signal_id,
|
524
|
+
);
|
525
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
526
|
+
|
527
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::TryCancel).await;
|
528
|
+
}
|
529
|
+
|
530
|
+
/// Verification for try cancel & abandon histories
|
531
|
+
async fn verify_activity_cancellation(
|
532
|
+
worker: &Worker,
|
533
|
+
activity_seq: u32,
|
534
|
+
cancel_type: ActivityCancellationType,
|
535
|
+
) {
|
536
|
+
poll_and_reply(
|
537
|
+
worker,
|
538
|
+
NonSticky,
|
539
|
+
&[
|
540
|
+
gen_assert_and_reply(
|
541
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
542
|
+
vec![ScheduleActivity {
|
543
|
+
seq: activity_seq,
|
544
|
+
activity_id: activity_seq.to_string(),
|
545
|
+
cancellation_type: cancel_type as i32,
|
546
|
+
..Default::default()
|
547
|
+
}
|
548
|
+
.into()],
|
549
|
+
),
|
550
|
+
gen_assert_and_reply(
|
551
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
552
|
+
vec![RequestCancelActivity { seq: activity_seq }.into()],
|
553
|
+
),
|
554
|
+
// Activity should be resolved right away
|
555
|
+
gen_assert_and_reply(
|
556
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
557
|
+
ResolveActivity {
|
558
|
+
seq: _,
|
559
|
+
result: Some(ActivityResolution {
|
560
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
561
|
+
})
|
562
|
+
}
|
563
|
+
)),
|
564
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
565
|
+
),
|
566
|
+
],
|
567
|
+
)
|
568
|
+
.await;
|
569
|
+
}
|
570
|
+
|
571
|
+
#[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
|
572
|
+
#[tokio::test]
|
573
|
+
async fn scheduled_activity_cancellation_wait_for_cancellation(hist_batches: &'static [usize]) {
|
574
|
+
let wfid = "fake_wf_id";
|
575
|
+
let activity_id = 1;
|
576
|
+
let signal_id = "signal";
|
577
|
+
|
578
|
+
let t = canned_histories::cancel_scheduled_activity_with_signal_and_activity_task_cancel(
|
579
|
+
activity_id.to_string().as_str(),
|
580
|
+
signal_id,
|
581
|
+
);
|
582
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
583
|
+
|
584
|
+
verify_activity_cancellation_wait_for_cancellation(activity_id, &core).await;
|
585
|
+
}
|
586
|
+
|
587
|
+
#[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
|
588
|
+
#[tokio::test]
|
589
|
+
async fn started_activity_cancellation_wait_for_cancellation(hist_batches: &'static [usize]) {
|
590
|
+
let wfid = "fake_wf_id";
|
591
|
+
let activity_id = 1;
|
592
|
+
let signal_id = "signal";
|
593
|
+
|
594
|
+
let t = canned_histories::cancel_started_activity_with_signal_and_activity_task_cancel(
|
595
|
+
activity_id.to_string().as_str(),
|
596
|
+
signal_id,
|
597
|
+
);
|
598
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
599
|
+
|
600
|
+
verify_activity_cancellation_wait_for_cancellation(activity_id, &core).await;
|
601
|
+
}
|
602
|
+
|
603
|
+
async fn verify_activity_cancellation_wait_for_cancellation(activity_id: u32, worker: &Worker) {
|
604
|
+
poll_and_reply(
|
605
|
+
worker,
|
606
|
+
NonSticky,
|
607
|
+
&[
|
608
|
+
gen_assert_and_reply(
|
609
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
610
|
+
vec![ScheduleActivity {
|
611
|
+
seq: activity_id,
|
612
|
+
activity_id: activity_id.to_string(),
|
613
|
+
cancellation_type: ActivityCancellationType::WaitCancellationCompleted as i32,
|
614
|
+
..Default::default()
|
615
|
+
}
|
616
|
+
.into()],
|
617
|
+
),
|
618
|
+
gen_assert_and_reply(
|
619
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
620
|
+
vec![RequestCancelActivity { seq: activity_id }.into()],
|
621
|
+
),
|
622
|
+
// Making sure that activity is not resolved until it's cancelled.
|
623
|
+
gen_assert_and_reply(
|
624
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
625
|
+
vec![],
|
626
|
+
),
|
627
|
+
// Now ActivityTaskCanceled has been processed and activity can be resolved.
|
628
|
+
gen_assert_and_reply(
|
629
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
630
|
+
ResolveActivity {
|
631
|
+
seq: _,
|
632
|
+
result: Some(ActivityResolution {
|
633
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
634
|
+
})
|
635
|
+
}
|
636
|
+
)),
|
637
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
638
|
+
),
|
639
|
+
],
|
640
|
+
)
|
641
|
+
.await;
|
642
|
+
}
|
643
|
+
|
644
|
+
#[tokio::test]
|
645
|
+
async fn workflow_update_random_seed_on_workflow_reset() {
|
646
|
+
let wfid = "fake_wf_id";
|
647
|
+
let new_run_id = "86E39A5F-AE31-4626-BDFE-398EE072D156";
|
648
|
+
let timer_1_id = 1;
|
649
|
+
let randomness_seed_from_start = AtomicU64::new(0);
|
650
|
+
|
651
|
+
let t = canned_histories::workflow_fails_with_reset_after_timer(
|
652
|
+
timer_1_id.to_string().as_str(),
|
653
|
+
new_run_id,
|
654
|
+
);
|
655
|
+
let core = build_fake_worker(wfid, t, &[2]);
|
656
|
+
|
657
|
+
poll_and_reply(
|
658
|
+
&core,
|
659
|
+
NonSticky,
|
660
|
+
&[
|
661
|
+
gen_assert_and_reply(
|
662
|
+
&|res| {
|
663
|
+
assert_matches!(
|
664
|
+
res.jobs.as_slice(),
|
665
|
+
[WorkflowActivationJob {
|
666
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(
|
667
|
+
StartWorkflow{randomness_seed, ..}
|
668
|
+
)),
|
669
|
+
}] => {
|
670
|
+
randomness_seed_from_start.store(*randomness_seed, Ordering::SeqCst);
|
671
|
+
}
|
672
|
+
);
|
673
|
+
},
|
674
|
+
vec![start_timer_cmd(timer_1_id, Duration::from_secs(1))],
|
675
|
+
),
|
676
|
+
gen_assert_and_reply(
|
677
|
+
&|res| {
|
678
|
+
assert_matches!(
|
679
|
+
res.jobs.as_slice(),
|
680
|
+
[WorkflowActivationJob {
|
681
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_),),
|
682
|
+
},
|
683
|
+
WorkflowActivationJob {
|
684
|
+
variant: Some(workflow_activation_job::Variant::UpdateRandomSeed(
|
685
|
+
UpdateRandomSeed{randomness_seed})),
|
686
|
+
}] => {
|
687
|
+
assert_ne!(randomness_seed_from_start.load(Ordering::SeqCst),
|
688
|
+
*randomness_seed);
|
689
|
+
}
|
690
|
+
);
|
691
|
+
},
|
692
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
693
|
+
),
|
694
|
+
],
|
695
|
+
)
|
696
|
+
.await;
|
697
|
+
}
|
698
|
+
|
699
|
+
#[tokio::test]
|
700
|
+
async fn cancel_timer_before_sent_wf_bridge() {
|
701
|
+
let wfid = "fake_wf_id";
|
702
|
+
let cancel_timer_id = 1;
|
703
|
+
|
704
|
+
let mut t = TestHistoryBuilder::default();
|
705
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
706
|
+
t.add_full_wf_task();
|
707
|
+
t.add_workflow_execution_completed();
|
708
|
+
|
709
|
+
let core = build_fake_worker(wfid, t, &[1]);
|
710
|
+
|
711
|
+
poll_and_reply(
|
712
|
+
&core,
|
713
|
+
NonSticky,
|
714
|
+
&[gen_assert_and_reply(
|
715
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
716
|
+
vec![
|
717
|
+
start_timer_cmd(cancel_timer_id, Duration::from_secs(1)),
|
718
|
+
CancelTimer {
|
719
|
+
seq: cancel_timer_id,
|
720
|
+
}
|
721
|
+
.into(),
|
722
|
+
CompleteWorkflowExecution { result: None }.into(),
|
723
|
+
],
|
724
|
+
)],
|
725
|
+
)
|
726
|
+
.await;
|
727
|
+
}
|
728
|
+
|
729
|
+
#[rstest]
|
730
|
+
#[case::no_evict_inc(&[1, 2, 2], NonSticky)]
|
731
|
+
#[case::no_evict(&[2, 2], NonSticky)]
|
732
|
+
#[tokio::test]
|
733
|
+
async fn complete_activation_with_failure(
|
734
|
+
#[case] batches: &'static [usize],
|
735
|
+
#[case] evict: WorkflowCachingPolicy,
|
736
|
+
) {
|
737
|
+
let wfid = "fake_wf_id";
|
738
|
+
let timer_id = 1;
|
739
|
+
|
740
|
+
let hist =
|
741
|
+
canned_histories::workflow_fails_with_failure_after_timer(timer_id.to_string().as_str());
|
742
|
+
let mock_sg = build_multihist_mock_sg(
|
743
|
+
vec![FakeWfResponses {
|
744
|
+
wf_id: wfid.to_string(),
|
745
|
+
hist,
|
746
|
+
response_batches: batches.iter().map(Into::into).collect(),
|
747
|
+
}],
|
748
|
+
true,
|
749
|
+
1,
|
750
|
+
);
|
751
|
+
let core = mock_worker(mock_sg);
|
752
|
+
|
753
|
+
poll_and_reply(
|
754
|
+
&core,
|
755
|
+
evict,
|
756
|
+
&[
|
757
|
+
gen_assert_and_reply(
|
758
|
+
&|_| {},
|
759
|
+
vec![start_timer_cmd(timer_id, Duration::from_secs(1))],
|
760
|
+
),
|
761
|
+
gen_assert_and_fail(&|_| {}),
|
762
|
+
gen_assert_and_reply(
|
763
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
764
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
765
|
+
),
|
766
|
+
],
|
767
|
+
)
|
768
|
+
.await;
|
769
|
+
core.shutdown().await;
|
770
|
+
}
|
771
|
+
|
772
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
773
|
+
#[tokio::test]
|
774
|
+
async fn simple_timer_fail_wf_execution(hist_batches: &'static [usize]) {
|
775
|
+
let wfid = "fake_wf_id";
|
776
|
+
let timer_id = 1;
|
777
|
+
|
778
|
+
let t = canned_histories::single_timer(timer_id.to_string().as_str());
|
779
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
780
|
+
|
781
|
+
poll_and_reply(
|
782
|
+
&core,
|
783
|
+
NonSticky,
|
784
|
+
&[
|
785
|
+
gen_assert_and_reply(
|
786
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
787
|
+
vec![start_timer_cmd(timer_id, Duration::from_secs(1))],
|
788
|
+
),
|
789
|
+
gen_assert_and_reply(
|
790
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
791
|
+
vec![FailWorkflowExecution {
|
792
|
+
failure: Some(Failure {
|
793
|
+
message: "I'm ded".to_string(),
|
794
|
+
..Default::default()
|
795
|
+
}),
|
796
|
+
}
|
797
|
+
.into()],
|
798
|
+
),
|
799
|
+
],
|
800
|
+
)
|
801
|
+
.await;
|
802
|
+
}
|
803
|
+
|
804
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
805
|
+
#[tokio::test]
|
806
|
+
async fn two_signals(hist_batches: &'static [usize]) {
|
807
|
+
let wfid = "fake_wf_id";
|
808
|
+
|
809
|
+
let t = canned_histories::two_signals("sig1", "sig2");
|
810
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
811
|
+
|
812
|
+
poll_and_reply(
|
813
|
+
&core,
|
814
|
+
NonSticky,
|
815
|
+
&[
|
816
|
+
gen_assert_and_reply(
|
817
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
818
|
+
// Task is completed with no commands
|
819
|
+
vec![],
|
820
|
+
),
|
821
|
+
gen_assert_and_reply(
|
822
|
+
&job_assert!(
|
823
|
+
workflow_activation_job::Variant::SignalWorkflow(_),
|
824
|
+
workflow_activation_job::Variant::SignalWorkflow(_)
|
825
|
+
),
|
826
|
+
vec![],
|
827
|
+
),
|
828
|
+
],
|
829
|
+
)
|
830
|
+
.await;
|
831
|
+
}
|
832
|
+
|
833
|
+
#[tokio::test]
|
834
|
+
async fn workflow_failures_only_reported_once() {
|
835
|
+
let wfid = "fake_wf_id";
|
836
|
+
let timer_1 = 1;
|
837
|
+
let timer_2 = 2;
|
838
|
+
|
839
|
+
let hist = canned_histories::workflow_fails_with_failure_two_different_points(
|
840
|
+
timer_1.to_string().as_str(),
|
841
|
+
timer_2.to_string().as_str(),
|
842
|
+
);
|
843
|
+
let response_batches = vec![
|
844
|
+
1, 2, // Start then first good reply
|
845
|
+
2, 2, 2, // Poll for every failure
|
846
|
+
// Poll again after evicting after second good reply, then two more fails
|
847
|
+
3, 3, 3,
|
848
|
+
];
|
849
|
+
let mocks = build_multihist_mock_sg(
|
850
|
+
vec![FakeWfResponses {
|
851
|
+
wf_id: wfid.to_string(),
|
852
|
+
hist,
|
853
|
+
response_batches: response_batches.into_iter().map(Into::into).collect(),
|
854
|
+
}],
|
855
|
+
true,
|
856
|
+
// We should only call the server to say we failed twice (once after each success)
|
857
|
+
2,
|
858
|
+
);
|
859
|
+
let omap = mocks.outstanding_task_map.clone();
|
860
|
+
let core = mock_worker(mocks);
|
861
|
+
|
862
|
+
poll_and_reply_clears_outstanding_evicts(
|
863
|
+
&core,
|
864
|
+
omap,
|
865
|
+
NonSticky,
|
866
|
+
&[
|
867
|
+
gen_assert_and_reply(
|
868
|
+
&|_| {},
|
869
|
+
vec![start_timer_cmd(timer_1, Duration::from_secs(1))],
|
870
|
+
),
|
871
|
+
// Fail a few times in a row (only one of which should be reported)
|
872
|
+
gen_assert_and_fail(&|_| {}),
|
873
|
+
gen_assert_and_fail(&|_| {}),
|
874
|
+
gen_assert_and_fail(&|_| {}),
|
875
|
+
gen_assert_and_reply(
|
876
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
877
|
+
vec![start_timer_cmd(timer_2, Duration::from_secs(1))],
|
878
|
+
),
|
879
|
+
// Again (a new fail should be reported here)
|
880
|
+
gen_assert_and_fail(&|_| {}),
|
881
|
+
gen_assert_and_fail(&|_| {}),
|
882
|
+
gen_assert_and_reply(
|
883
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
884
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
885
|
+
),
|
886
|
+
],
|
887
|
+
)
|
888
|
+
.await;
|
889
|
+
}
|
890
|
+
|
891
|
+
#[tokio::test]
|
892
|
+
async fn max_wft_respected() {
|
893
|
+
let total_wfs = 100;
|
894
|
+
let wf_ids: Vec<_> = (0..total_wfs)
|
895
|
+
.into_iter()
|
896
|
+
.map(|i| format!("fake-wf-{}", i))
|
897
|
+
.collect();
|
898
|
+
let hists = wf_ids.iter().map(|wf_id| {
|
899
|
+
let hist = canned_histories::single_timer("1");
|
900
|
+
FakeWfResponses {
|
901
|
+
wf_id: wf_id.to_string(),
|
902
|
+
hist,
|
903
|
+
response_batches: vec![1.into(), 2.into()],
|
904
|
+
}
|
905
|
+
});
|
906
|
+
let mh = MockPollCfg::new(hists.into_iter().collect(), true, 0);
|
907
|
+
let mut worker = mock_sdk_cfg(mh, |cfg| {
|
908
|
+
cfg.max_cached_workflows = total_wfs as usize;
|
909
|
+
cfg.max_outstanding_workflow_tasks = 1;
|
910
|
+
});
|
911
|
+
let active_count: &'static _ = Box::leak(Box::new(Semaphore::new(1)));
|
912
|
+
worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| async move {
|
913
|
+
drop(
|
914
|
+
active_count
|
915
|
+
.try_acquire()
|
916
|
+
.expect("No multiple concurrent workflow tasks!"),
|
917
|
+
);
|
918
|
+
ctx.timer(Duration::from_secs(1)).await;
|
919
|
+
Ok(().into())
|
920
|
+
});
|
921
|
+
|
922
|
+
for wf_id in wf_ids {
|
923
|
+
worker
|
924
|
+
.submit_wf(wf_id, DEFAULT_WORKFLOW_TYPE, vec![], Default::default())
|
925
|
+
.await
|
926
|
+
.unwrap();
|
927
|
+
}
|
928
|
+
worker.run_until_done().await.unwrap();
|
929
|
+
}
|
930
|
+
|
931
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[3]))]
|
932
|
+
#[tokio::test]
|
933
|
+
async fn activity_not_canceled_on_replay_repro(hist_batches: &'static [usize]) {
|
934
|
+
let wfid = "fake_wf_id";
|
935
|
+
let t = canned_histories::unsent_at_cancel_repro();
|
936
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
937
|
+
let activity_id = 1;
|
938
|
+
|
939
|
+
poll_and_reply(
|
940
|
+
&core,
|
941
|
+
NonSticky,
|
942
|
+
&[
|
943
|
+
gen_assert_and_reply(
|
944
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
945
|
+
// Start timer and activity
|
946
|
+
vec![
|
947
|
+
ScheduleActivity {
|
948
|
+
seq: activity_id,
|
949
|
+
activity_id: activity_id.to_string(),
|
950
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
951
|
+
..Default::default()
|
952
|
+
}
|
953
|
+
.into(),
|
954
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
955
|
+
],
|
956
|
+
),
|
957
|
+
gen_assert_and_reply(
|
958
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
959
|
+
vec![RequestCancelActivity { seq: activity_id }.into()],
|
960
|
+
),
|
961
|
+
gen_assert_and_reply(
|
962
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
963
|
+
ResolveActivity {
|
964
|
+
result: Some(ActivityResolution {
|
965
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
966
|
+
}),
|
967
|
+
..
|
968
|
+
}
|
969
|
+
)),
|
970
|
+
vec![start_timer_cmd(2, Duration::from_secs(1))],
|
971
|
+
),
|
972
|
+
],
|
973
|
+
)
|
974
|
+
.await;
|
975
|
+
}
|
976
|
+
|
977
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[3]))]
|
978
|
+
#[tokio::test]
|
979
|
+
async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static [usize]) {
|
980
|
+
let wfid = "fake_wf_id";
|
981
|
+
let t = canned_histories::cancel_not_sent_when_also_complete_repro();
|
982
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
983
|
+
let activity_id = 1;
|
984
|
+
|
985
|
+
poll_and_reply(
|
986
|
+
&core,
|
987
|
+
NonSticky,
|
988
|
+
&[
|
989
|
+
gen_assert_and_reply(
|
990
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
991
|
+
vec![ScheduleActivity {
|
992
|
+
seq: activity_id,
|
993
|
+
activity_id: activity_id.to_string(),
|
994
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
995
|
+
..Default::default()
|
996
|
+
}
|
997
|
+
.into()],
|
998
|
+
),
|
999
|
+
gen_assert_and_reply(
|
1000
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1001
|
+
vec![
|
1002
|
+
RequestCancelActivity { seq: activity_id }.into(),
|
1003
|
+
start_timer_cmd(2, Duration::from_secs(1)),
|
1004
|
+
],
|
1005
|
+
),
|
1006
|
+
gen_assert_and_reply(
|
1007
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
1008
|
+
ResolveActivity {
|
1009
|
+
result: Some(ActivityResolution {
|
1010
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
1011
|
+
}),
|
1012
|
+
..
|
1013
|
+
}
|
1014
|
+
)),
|
1015
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1016
|
+
),
|
1017
|
+
],
|
1018
|
+
)
|
1019
|
+
.await;
|
1020
|
+
}
|
1021
|
+
|
1022
|
+
#[tokio::test]
|
1023
|
+
async fn lots_of_workflows() {
|
1024
|
+
let total_wfs = 500;
|
1025
|
+
let hists = (0..total_wfs).into_iter().map(|i| {
|
1026
|
+
let wf_id = format!("fake-wf-{}", i);
|
1027
|
+
let hist = canned_histories::single_timer("1");
|
1028
|
+
FakeWfResponses {
|
1029
|
+
wf_id,
|
1030
|
+
hist,
|
1031
|
+
response_batches: vec![1.into(), 2.into()],
|
1032
|
+
}
|
1033
|
+
});
|
1034
|
+
let mut mock = build_multihist_mock_sg(hists, false, 0);
|
1035
|
+
mock.make_wft_stream_interminable();
|
1036
|
+
let worker = &mock_worker(mock);
|
1037
|
+
let completed_count = Arc::new(Semaphore::new(0));
|
1038
|
+
let killer = async {
|
1039
|
+
let _ = completed_count.acquire_many(total_wfs).await.unwrap();
|
1040
|
+
dbg!("Shutdown initted");
|
1041
|
+
worker.initiate_shutdown();
|
1042
|
+
};
|
1043
|
+
let poller = fanout_tasks(5, |_| {
|
1044
|
+
let completed_count = completed_count.clone();
|
1045
|
+
async move {
|
1046
|
+
while let Ok(wft) = worker.poll_workflow_activation().await {
|
1047
|
+
let job = &wft.jobs[0];
|
1048
|
+
let reply = match job.variant {
|
1049
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_)) => {
|
1050
|
+
start_timer_cmd(1, Duration::from_secs(1))
|
1051
|
+
}
|
1052
|
+
Some(workflow_activation_job::Variant::RemoveFromCache(_)) => {
|
1053
|
+
worker
|
1054
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(
|
1055
|
+
wft.run_id,
|
1056
|
+
))
|
1057
|
+
.await
|
1058
|
+
.unwrap();
|
1059
|
+
continue;
|
1060
|
+
}
|
1061
|
+
_ => {
|
1062
|
+
completed_count.add_permits(1);
|
1063
|
+
CompleteWorkflowExecution { result: None }.into()
|
1064
|
+
}
|
1065
|
+
};
|
1066
|
+
worker
|
1067
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1068
|
+
wft.run_id, reply,
|
1069
|
+
))
|
1070
|
+
.await
|
1071
|
+
.unwrap();
|
1072
|
+
}
|
1073
|
+
}
|
1074
|
+
});
|
1075
|
+
join!(killer, poller);
|
1076
|
+
worker.shutdown().await;
|
1077
|
+
}
|
1078
|
+
|
1079
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
1080
|
+
#[tokio::test]
|
1081
|
+
async fn wft_timeout_repro(hist_batches: &'static [usize]) {
|
1082
|
+
let wfid = "fake_wf_id";
|
1083
|
+
let t = canned_histories::wft_timeout_repro();
|
1084
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
1085
|
+
let activity_id = 1;
|
1086
|
+
|
1087
|
+
poll_and_reply(
|
1088
|
+
&core,
|
1089
|
+
NonSticky,
|
1090
|
+
&[
|
1091
|
+
gen_assert_and_reply(
|
1092
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
1093
|
+
vec![ScheduleActivity {
|
1094
|
+
seq: activity_id,
|
1095
|
+
activity_id: activity_id.to_string(),
|
1096
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
1097
|
+
..Default::default()
|
1098
|
+
}
|
1099
|
+
.into()],
|
1100
|
+
),
|
1101
|
+
gen_assert_and_reply(
|
1102
|
+
&job_assert!(
|
1103
|
+
workflow_activation_job::Variant::SignalWorkflow(_),
|
1104
|
+
workflow_activation_job::Variant::SignalWorkflow(_),
|
1105
|
+
workflow_activation_job::Variant::ResolveActivity(ResolveActivity {
|
1106
|
+
result: Some(ActivityResolution {
|
1107
|
+
status: Some(activity_resolution::Status::Completed(..)),
|
1108
|
+
}),
|
1109
|
+
..
|
1110
|
+
})
|
1111
|
+
),
|
1112
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1113
|
+
),
|
1114
|
+
],
|
1115
|
+
)
|
1116
|
+
.await;
|
1117
|
+
}
|
1118
|
+
|
1119
|
+
#[tokio::test]
|
1120
|
+
async fn complete_after_eviction() {
|
1121
|
+
let wfid = "fake_wf_id";
|
1122
|
+
let t = canned_histories::single_timer("1");
|
1123
|
+
let mut mock = mock_workflow_client();
|
1124
|
+
mock.expect_complete_workflow_task().times(0);
|
1125
|
+
let mock = single_hist_mock_sg(wfid, t, &[2], mock, true);
|
1126
|
+
let core = mock_worker(mock);
|
1127
|
+
|
1128
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1129
|
+
// We just got start workflow, immediately evict
|
1130
|
+
core.request_workflow_eviction(&activation.run_id);
|
1131
|
+
// Since we got whole history, we must finish replay before eviction will appear
|
1132
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1133
|
+
activation.run_id,
|
1134
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1135
|
+
))
|
1136
|
+
.await
|
1137
|
+
.unwrap();
|
1138
|
+
let next_activation = core.poll_workflow_activation().await.unwrap();
|
1139
|
+
assert_matches!(
|
1140
|
+
next_activation.jobs.as_slice(),
|
1141
|
+
[WorkflowActivationJob {
|
1142
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1143
|
+
},]
|
1144
|
+
);
|
1145
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1146
|
+
next_activation.run_id,
|
1147
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1148
|
+
))
|
1149
|
+
.await
|
1150
|
+
.unwrap();
|
1151
|
+
|
1152
|
+
core.shutdown().await;
|
1153
|
+
}
|
1154
|
+
|
1155
|
+
#[tokio::test]
|
1156
|
+
async fn sends_appropriate_sticky_task_queue_responses() {
|
1157
|
+
// This test verifies that when completions are sent with sticky queues enabled, that they
|
1158
|
+
// include the information that tells the server to enqueue the next task on a sticky queue.
|
1159
|
+
let wfid = "fake_wf_id";
|
1160
|
+
let t = canned_histories::single_timer("1");
|
1161
|
+
let mut mock = mock_workflow_client();
|
1162
|
+
mock.expect_complete_workflow_task()
|
1163
|
+
.withf(|comp| comp.sticky_attributes.is_some())
|
1164
|
+
.times(1)
|
1165
|
+
.returning(|_| Ok(Default::default()));
|
1166
|
+
mock.expect_complete_workflow_task().times(0);
|
1167
|
+
let mut mock = single_hist_mock_sg(wfid, t, &[1], mock, false);
|
1168
|
+
mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
|
1169
|
+
let core = mock_worker(mock);
|
1170
|
+
|
1171
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1172
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1173
|
+
activation.run_id,
|
1174
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1175
|
+
))
|
1176
|
+
.await
|
1177
|
+
.unwrap();
|
1178
|
+
core.shutdown().await;
|
1179
|
+
}
|
1180
|
+
|
1181
|
+
#[tokio::test]
|
1182
|
+
async fn new_server_work_while_eviction_outstanding_doesnt_overwrite_activation() {
|
1183
|
+
let wfid = "fake_wf_id";
|
1184
|
+
let t = canned_histories::single_timer("1");
|
1185
|
+
let mock = single_hist_mock_sg(wfid, t, &[1, 2], mock_workflow_client(), false);
|
1186
|
+
let taskmap = mock.outstanding_task_map.clone().unwrap();
|
1187
|
+
let core = mock_worker(mock);
|
1188
|
+
|
1189
|
+
// Poll for and complete first workflow task
|
1190
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1191
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1192
|
+
activation.run_id,
|
1193
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1194
|
+
))
|
1195
|
+
.await
|
1196
|
+
.unwrap();
|
1197
|
+
let evict_act = core.poll_workflow_activation().await.unwrap();
|
1198
|
+
assert_matches!(
|
1199
|
+
evict_act.jobs.as_slice(),
|
1200
|
+
[WorkflowActivationJob {
|
1201
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1202
|
+
}]
|
1203
|
+
);
|
1204
|
+
// Ensure mock has delivered both tasks
|
1205
|
+
assert!(taskmap.all_work_delivered());
|
1206
|
+
// Now we can complete the evict
|
1207
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_act.run_id))
|
1208
|
+
.await
|
1209
|
+
.unwrap();
|
1210
|
+
// The task buffered during eviction is applied and we start over
|
1211
|
+
let start_again = core.poll_workflow_activation().await.unwrap();
|
1212
|
+
assert_matches!(
|
1213
|
+
start_again.jobs[0].variant,
|
1214
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_))
|
1215
|
+
);
|
1216
|
+
}
|
1217
|
+
|
1218
|
+
#[tokio::test]
|
1219
|
+
async fn buffered_work_drained_on_shutdown() {
|
1220
|
+
let wfid = "fake_wf_id";
|
1221
|
+
// Build a one-timer history where first task times out
|
1222
|
+
let mut t = TestHistoryBuilder::default();
|
1223
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1224
|
+
t.add_workflow_task_scheduled_and_started();
|
1225
|
+
// Need to build the first response before adding the timeout events b/c otherwise the history
|
1226
|
+
// builder will include them in the first task
|
1227
|
+
let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()).resp;
|
1228
|
+
t.add_workflow_task_timed_out();
|
1229
|
+
t.add_full_wf_task();
|
1230
|
+
let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
|
1231
|
+
t.add(
|
1232
|
+
EventType::TimerFired,
|
1233
|
+
history_event::Attributes::TimerFiredEventAttributes(TimerFiredEventAttributes {
|
1234
|
+
started_event_id: timer_started_event_id,
|
1235
|
+
timer_id: "1".to_string(),
|
1236
|
+
}),
|
1237
|
+
);
|
1238
|
+
t.add_full_wf_task();
|
1239
|
+
t.add_workflow_execution_completed();
|
1240
|
+
|
1241
|
+
let mut tasks = VecDeque::from(vec![resp_1]);
|
1242
|
+
// Extend the task list with the now timeout-included version of the task. We add a bunch of
|
1243
|
+
// them because the poll loop will spin while new tasks are available and it is buffering them
|
1244
|
+
tasks.extend(
|
1245
|
+
std::iter::repeat_with(|| {
|
1246
|
+
hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp
|
1247
|
+
})
|
1248
|
+
.take(50),
|
1249
|
+
);
|
1250
|
+
let mut mock = mock_workflow_client();
|
1251
|
+
mock.expect_complete_workflow_task()
|
1252
|
+
.returning(|_| Ok(RespondWorkflowTaskCompletedResponse::default()));
|
1253
|
+
let mut mock = MocksHolder::from_wft_stream(mock, stream::iter(tasks));
|
1254
|
+
// Cache on to avoid being super repetitive
|
1255
|
+
mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
|
1256
|
+
let core = &mock_worker(mock);
|
1257
|
+
|
1258
|
+
// Poll for first WFT
|
1259
|
+
let act1 = core.poll_workflow_activation().await.unwrap();
|
1260
|
+
let poll_fut = async move {
|
1261
|
+
// Now poll again, which will start spinning, and buffer the next WFT with timer fired in it
|
1262
|
+
// - it won't stop spinning until the first task is complete
|
1263
|
+
let t = core.poll_workflow_activation().await.unwrap();
|
1264
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1265
|
+
t.run_id,
|
1266
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1267
|
+
))
|
1268
|
+
.await
|
1269
|
+
.unwrap();
|
1270
|
+
};
|
1271
|
+
let complete_first = async move {
|
1272
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1273
|
+
act1.run_id,
|
1274
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1275
|
+
))
|
1276
|
+
.await
|
1277
|
+
.unwrap();
|
1278
|
+
};
|
1279
|
+
join!(poll_fut, complete_first, async {
|
1280
|
+
// If the shutdown is sent too too fast, we might not have got a chance to even buffer work
|
1281
|
+
tokio::time::sleep(Duration::from_millis(5)).await;
|
1282
|
+
core.shutdown().await;
|
1283
|
+
});
|
1284
|
+
}
|
1285
|
+
|
1286
|
+
#[tokio::test]
|
1287
|
+
async fn fail_wft_then_recover() {
|
1288
|
+
let t = canned_histories::long_sequential_timers(1);
|
1289
|
+
let mut mh = MockPollCfg::from_resp_batches(
|
1290
|
+
"fake_wf_id",
|
1291
|
+
t,
|
1292
|
+
// We need to deliver all of history twice because of eviction
|
1293
|
+
[ResponseType::AllHistory, ResponseType::AllHistory],
|
1294
|
+
mock_workflow_client(),
|
1295
|
+
);
|
1296
|
+
mh.num_expected_fails = 1;
|
1297
|
+
mh.expect_fail_wft_matcher =
|
1298
|
+
Box::new(|_, cause, _| matches!(cause, WorkflowTaskFailedCause::NonDeterministicError));
|
1299
|
+
let mut mock = build_mock_pollers(mh);
|
1300
|
+
mock.worker_cfg(|wc| {
|
1301
|
+
wc.max_cached_workflows = 2;
|
1302
|
+
});
|
1303
|
+
let core = mock_worker(mock);
|
1304
|
+
|
1305
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1306
|
+
// Start an activity instead of a timer, triggering nondeterminism error
|
1307
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1308
|
+
act.run_id.clone(),
|
1309
|
+
vec![ScheduleActivity {
|
1310
|
+
activity_id: "fake_activity".to_string(),
|
1311
|
+
..Default::default()
|
1312
|
+
}
|
1313
|
+
.into()],
|
1314
|
+
))
|
1315
|
+
.await
|
1316
|
+
.unwrap();
|
1317
|
+
// We must handle an eviction now
|
1318
|
+
let evict_act = core.poll_workflow_activation().await.unwrap();
|
1319
|
+
assert_eq!(evict_act.run_id, act.run_id);
|
1320
|
+
assert_matches!(
|
1321
|
+
evict_act.jobs.as_slice(),
|
1322
|
+
[WorkflowActivationJob {
|
1323
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1324
|
+
}]
|
1325
|
+
);
|
1326
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_act.run_id))
|
1327
|
+
.await
|
1328
|
+
.unwrap();
|
1329
|
+
|
1330
|
+
// Workflow starting over, this time issue the right command
|
1331
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1332
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1333
|
+
act.run_id,
|
1334
|
+
vec![start_timer_cmd(1, Duration::from_secs(1))],
|
1335
|
+
))
|
1336
|
+
.await
|
1337
|
+
.unwrap();
|
1338
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1339
|
+
assert_matches!(
|
1340
|
+
act.jobs.as_slice(),
|
1341
|
+
[WorkflowActivationJob {
|
1342
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1343
|
+
},]
|
1344
|
+
);
|
1345
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1346
|
+
act.run_id,
|
1347
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1348
|
+
))
|
1349
|
+
.await
|
1350
|
+
.unwrap();
|
1351
|
+
core.shutdown().await;
|
1352
|
+
}
|
1353
|
+
|
1354
|
+
#[tokio::test]
|
1355
|
+
async fn poll_response_triggers_wf_error() {
|
1356
|
+
let mut t = TestHistoryBuilder::default();
|
1357
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1358
|
+
// Add this nonsense event here to make applying the poll response fail
|
1359
|
+
t.add_external_signal_completed(100);
|
1360
|
+
t.add_full_wf_task();
|
1361
|
+
t.add_workflow_execution_completed();
|
1362
|
+
|
1363
|
+
let mh = MockPollCfg::from_resp_batches(
|
1364
|
+
"fake_wf_id",
|
1365
|
+
t,
|
1366
|
+
[ResponseType::AllHistory],
|
1367
|
+
mock_workflow_client(),
|
1368
|
+
);
|
1369
|
+
let mock = build_mock_pollers(mh);
|
1370
|
+
let core = mock_worker(mock);
|
1371
|
+
// Poll for first WFT, which is immediately an eviction
|
1372
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1373
|
+
assert_matches!(
|
1374
|
+
act.jobs.as_slice(),
|
1375
|
+
[WorkflowActivationJob {
|
1376
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1377
|
+
}]
|
1378
|
+
);
|
1379
|
+
}
|
1380
|
+
|
1381
|
+
// Verifies we can handle multiple wft timeouts in a row if lang is being very slow in responding
|
1382
|
+
#[tokio::test]
|
1383
|
+
async fn lang_slower_than_wft_timeouts() {
|
1384
|
+
let wfid = "fake_wf_id";
|
1385
|
+
let mut t = TestHistoryBuilder::default();
|
1386
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1387
|
+
t.add_workflow_task_scheduled_and_started();
|
1388
|
+
t.add_workflow_task_timed_out();
|
1389
|
+
t.add_full_wf_task();
|
1390
|
+
t.add_workflow_execution_completed();
|
1391
|
+
|
1392
|
+
let mut mock = mock_workflow_client();
|
1393
|
+
mock.expect_complete_workflow_task()
|
1394
|
+
.times(1)
|
1395
|
+
.returning(|_| Err(tonic::Status::not_found("Workflow task not found.")));
|
1396
|
+
mock.expect_complete_workflow_task()
|
1397
|
+
.times(1)
|
1398
|
+
.returning(|_| Ok(Default::default()));
|
1399
|
+
let mut mock = single_hist_mock_sg(wfid, t, [1, 1], mock, true);
|
1400
|
+
let tasksmap = mock.outstanding_task_map.clone().unwrap();
|
1401
|
+
mock.worker_cfg(|wc| {
|
1402
|
+
wc.max_cached_workflows = 2;
|
1403
|
+
});
|
1404
|
+
let core = mock_worker(mock);
|
1405
|
+
|
1406
|
+
// This completion runs into the workflow task not found error
|
1407
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1408
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
|
1409
|
+
.await
|
1410
|
+
.unwrap();
|
1411
|
+
// It will get an eviction
|
1412
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1413
|
+
assert_matches!(
|
1414
|
+
wf_task.jobs.as_slice(),
|
1415
|
+
[WorkflowActivationJob {
|
1416
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1417
|
+
}]
|
1418
|
+
);
|
1419
|
+
// Before we complete, unlock the next task from the mock so that we'll see it get buffered.
|
1420
|
+
tasksmap.release_run(&wf_task.run_id);
|
1421
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
|
1422
|
+
.await
|
1423
|
+
.unwrap();
|
1424
|
+
// The buffered WFT should be applied now
|
1425
|
+
let start_again = core.poll_workflow_activation().await.unwrap();
|
1426
|
+
assert_matches!(
|
1427
|
+
start_again.jobs[0].variant,
|
1428
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_))
|
1429
|
+
);
|
1430
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1431
|
+
start_again.run_id,
|
1432
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1433
|
+
))
|
1434
|
+
.await
|
1435
|
+
.unwrap();
|
1436
|
+
core.shutdown().await;
|
1437
|
+
}
|
1438
|
+
|
1439
|
+
#[tokio::test]
|
1440
|
+
async fn tries_cancel_of_completed_activity() {
|
1441
|
+
let mut t = TestHistoryBuilder::default();
|
1442
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1443
|
+
t.add_full_wf_task();
|
1444
|
+
let scheduled_event_id = t.add_activity_task_scheduled("1");
|
1445
|
+
t.add_we_signaled("sig", vec![]);
|
1446
|
+
let started_event_id = t.add_activity_task_started(scheduled_event_id);
|
1447
|
+
t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
|
1448
|
+
t.add_workflow_task_scheduled_and_started();
|
1449
|
+
|
1450
|
+
let mock = mock_workflow_client();
|
1451
|
+
let mut mock = single_hist_mock_sg("fake_wf_id", t, &[1, 2], mock, true);
|
1452
|
+
mock.worker_cfg(|cfg| cfg.max_cached_workflows = 1);
|
1453
|
+
let core = mock_worker(mock);
|
1454
|
+
|
1455
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1456
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1457
|
+
activation.run_id,
|
1458
|
+
ScheduleActivity {
|
1459
|
+
seq: 1,
|
1460
|
+
activity_id: "1".to_string(),
|
1461
|
+
..Default::default()
|
1462
|
+
}
|
1463
|
+
.into(),
|
1464
|
+
))
|
1465
|
+
.await
|
1466
|
+
.unwrap();
|
1467
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1468
|
+
assert_matches!(
|
1469
|
+
activation.jobs.as_slice(),
|
1470
|
+
[
|
1471
|
+
WorkflowActivationJob {
|
1472
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1473
|
+
},
|
1474
|
+
WorkflowActivationJob {
|
1475
|
+
variant: Some(workflow_activation_job::Variant::ResolveActivity(_)),
|
1476
|
+
}
|
1477
|
+
]
|
1478
|
+
);
|
1479
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1480
|
+
activation.run_id,
|
1481
|
+
vec![
|
1482
|
+
RequestCancelActivity { seq: 1 }.into(),
|
1483
|
+
CompleteWorkflowExecution { result: None }.into(),
|
1484
|
+
],
|
1485
|
+
))
|
1486
|
+
.await
|
1487
|
+
.unwrap();
|
1488
|
+
|
1489
|
+
core.shutdown().await;
|
1490
|
+
}
|
1491
|
+
|
1492
|
+
#[tokio::test]
|
1493
|
+
async fn failing_wft_doesnt_eat_permit_forever() {
|
1494
|
+
let mut t = TestHistoryBuilder::default();
|
1495
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1496
|
+
t.add_workflow_task_scheduled_and_started();
|
1497
|
+
|
1498
|
+
let mock = mock_workflow_client();
|
1499
|
+
let mut mock = MockPollCfg::from_resp_batches("fake_wf_id", t, [1, 1, 1], mock);
|
1500
|
+
mock.num_expected_fails = 1;
|
1501
|
+
let mut mock = build_mock_pollers(mock);
|
1502
|
+
mock.worker_cfg(|cfg| {
|
1503
|
+
cfg.max_cached_workflows = 2;
|
1504
|
+
cfg.max_outstanding_workflow_tasks = 2;
|
1505
|
+
});
|
1506
|
+
let outstanding_mock_tasks = mock.outstanding_task_map.clone();
|
1507
|
+
let worker = mock_worker(mock);
|
1508
|
+
|
1509
|
+
let mut run_id = "".to_string();
|
1510
|
+
// Fail twice, verifying a permit is not eaten. We cannot fail the same run more than twice in a
|
1511
|
+
// row because we purposefully time out rather than spamming.
|
1512
|
+
for _ in 1..=2 {
|
1513
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1514
|
+
// Issue a nonsense completion that will trigger a WFT failure
|
1515
|
+
worker
|
1516
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1517
|
+
activation.run_id,
|
1518
|
+
RequestCancelActivity { seq: 1 }.into(),
|
1519
|
+
))
|
1520
|
+
.await
|
1521
|
+
.unwrap();
|
1522
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1523
|
+
assert_matches!(
|
1524
|
+
activation.jobs.as_slice(),
|
1525
|
+
[WorkflowActivationJob {
|
1526
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1527
|
+
},]
|
1528
|
+
);
|
1529
|
+
run_id = activation.run_id.clone();
|
1530
|
+
worker
|
1531
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
|
1532
|
+
.await
|
1533
|
+
.unwrap();
|
1534
|
+
}
|
1535
|
+
assert_eq!(worker.outstanding_workflow_tasks().await, 0);
|
1536
|
+
// 1 permit is in use because the next task is buffered and has re-used the permit
|
1537
|
+
assert_eq!(worker.available_wft_permits().await, 1);
|
1538
|
+
// We should be "out of work" because the mock service thinks we didn't complete the last task,
|
1539
|
+
// which we didn't, because we don't spam failures. The real server would eventually time out
|
1540
|
+
// the task. Mock doesn't understand that, so the WFT permit is released because eventually a
|
1541
|
+
// new one will be generated. We manually clear the mock's outstanding task list so the next
|
1542
|
+
// poll will work.
|
1543
|
+
outstanding_mock_tasks.unwrap().release_run(&run_id);
|
1544
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1545
|
+
// There should be no change in permits, since this just unbuffered the buffered task
|
1546
|
+
assert_eq!(worker.available_wft_permits().await, 1);
|
1547
|
+
worker
|
1548
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1549
|
+
activation.run_id,
|
1550
|
+
CompleteWorkflowExecution { result: None }.into(),
|
1551
|
+
))
|
1552
|
+
.await
|
1553
|
+
.unwrap();
|
1554
|
+
assert_eq!(worker.available_wft_permits().await, 2);
|
1555
|
+
|
1556
|
+
worker.shutdown().await;
|
1557
|
+
}
|
1558
|
+
|
1559
|
+
#[tokio::test]
|
1560
|
+
async fn cache_miss_will_fetch_history() {
|
1561
|
+
let mut t = TestHistoryBuilder::default();
|
1562
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1563
|
+
t.add_full_wf_task();
|
1564
|
+
t.add_we_signaled("sig", vec![]);
|
1565
|
+
t.add_full_wf_task();
|
1566
|
+
t.add_workflow_execution_completed();
|
1567
|
+
let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
|
1568
|
+
|
1569
|
+
let mut mh = MockPollCfg::from_resp_batches(
|
1570
|
+
"fake_wf_id",
|
1571
|
+
t,
|
1572
|
+
[ResponseType::ToTaskNum(1), ResponseType::OneTask(2)],
|
1573
|
+
mock_workflow_client(),
|
1574
|
+
);
|
1575
|
+
mh.mock_client
|
1576
|
+
.expect_get_workflow_execution_history()
|
1577
|
+
.times(1)
|
1578
|
+
.returning(move |_, _, _| Ok(get_exec_resp.clone()));
|
1579
|
+
let mut mock = build_mock_pollers(mh);
|
1580
|
+
mock.worker_cfg(|cfg| {
|
1581
|
+
cfg.max_cached_workflows = 1;
|
1582
|
+
});
|
1583
|
+
let worker = mock_worker(mock);
|
1584
|
+
|
1585
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1586
|
+
assert_eq!(activation.history_length, 3);
|
1587
|
+
assert_matches!(
|
1588
|
+
activation.jobs.as_slice(),
|
1589
|
+
[WorkflowActivationJob {
|
1590
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
|
1591
|
+
}]
|
1592
|
+
);
|
1593
|
+
// Force an eviction (before complete matters, so that we will be sure the eviction is queued
|
1594
|
+
// up before the next fake WFT is unlocked)
|
1595
|
+
worker.request_wf_eviction(
|
1596
|
+
&activation.run_id,
|
1597
|
+
"whatever",
|
1598
|
+
EvictionReason::LangRequested,
|
1599
|
+
);
|
1600
|
+
worker
|
1601
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(&activation.run_id))
|
1602
|
+
.await
|
1603
|
+
.unwrap();
|
1604
|
+
// Handle the eviction, and the restart
|
1605
|
+
for i in 1..=2 {
|
1606
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1607
|
+
assert_eq!(activation.history_length, 3);
|
1608
|
+
if i == 1 {
|
1609
|
+
assert_matches!(
|
1610
|
+
activation.jobs.as_slice(),
|
1611
|
+
[WorkflowActivationJob {
|
1612
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1613
|
+
}]
|
1614
|
+
);
|
1615
|
+
} else {
|
1616
|
+
assert_matches!(
|
1617
|
+
activation.jobs.as_slice(),
|
1618
|
+
[WorkflowActivationJob {
|
1619
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
|
1620
|
+
}]
|
1621
|
+
);
|
1622
|
+
}
|
1623
|
+
worker
|
1624
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
|
1625
|
+
.await
|
1626
|
+
.unwrap();
|
1627
|
+
}
|
1628
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1629
|
+
assert_eq!(activation.history_length, 7);
|
1630
|
+
assert_matches!(
|
1631
|
+
activation.jobs.as_slice(),
|
1632
|
+
[WorkflowActivationJob {
|
1633
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1634
|
+
}]
|
1635
|
+
);
|
1636
|
+
worker
|
1637
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1638
|
+
activation.run_id,
|
1639
|
+
CompleteWorkflowExecution { result: None }.into(),
|
1640
|
+
))
|
1641
|
+
.await
|
1642
|
+
.unwrap();
|
1643
|
+
assert_eq!(worker.outstanding_workflow_tasks().await, 0);
|
1644
|
+
worker.shutdown().await;
|
1645
|
+
}
|
1646
|
+
|
1647
|
+
/// This test verifies that WFTs which come as replies to completing a WFT are properly delivered
|
1648
|
+
/// via activation polling.
|
1649
|
+
#[tokio::test]
|
1650
|
+
async fn tasks_from_completion_are_delivered() {
|
1651
|
+
let wfid = "fake_wf_id";
|
1652
|
+
let mut t = TestHistoryBuilder::default();
|
1653
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1654
|
+
t.add_full_wf_task();
|
1655
|
+
t.add_we_signaled("sig", vec![]);
|
1656
|
+
t.add_full_wf_task();
|
1657
|
+
t.add_workflow_execution_completed();
|
1658
|
+
|
1659
|
+
let mut mock = mock_workflow_client();
|
1660
|
+
let complete_resp = RespondWorkflowTaskCompletedResponse {
|
1661
|
+
workflow_task: Some(
|
1662
|
+
hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp,
|
1663
|
+
),
|
1664
|
+
activity_tasks: vec![],
|
1665
|
+
};
|
1666
|
+
mock.expect_complete_workflow_task()
|
1667
|
+
.times(1)
|
1668
|
+
.returning(move |_| Ok(complete_resp.clone()));
|
1669
|
+
mock.expect_complete_workflow_task()
|
1670
|
+
.times(1)
|
1671
|
+
.returning(|_| Ok(Default::default()));
|
1672
|
+
let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
|
1673
|
+
mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
|
1674
|
+
let core = mock_worker(mock);
|
1675
|
+
|
1676
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1677
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
|
1678
|
+
.await
|
1679
|
+
.unwrap();
|
1680
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1681
|
+
assert_matches!(
|
1682
|
+
wf_task.jobs.as_slice(),
|
1683
|
+
[WorkflowActivationJob {
|
1684
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1685
|
+
},]
|
1686
|
+
);
|
1687
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1688
|
+
wf_task.run_id,
|
1689
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1690
|
+
))
|
1691
|
+
.await
|
1692
|
+
.unwrap();
|
1693
|
+
core.shutdown().await;
|
1694
|
+
}
|
1695
|
+
|
1696
|
+
#[tokio::test]
|
1697
|
+
async fn poll_faster_than_complete_wont_overflow_cache() {
|
1698
|
+
// Make workflow tasks for 5 different runs
|
1699
|
+
let tasks: Vec<_> = (1..=5)
|
1700
|
+
.map(|i| FakeWfResponses {
|
1701
|
+
wf_id: format!("wf-{}", i),
|
1702
|
+
hist: canned_histories::single_timer("1"),
|
1703
|
+
response_batches: vec![ResponseType::ToTaskNum(1)],
|
1704
|
+
})
|
1705
|
+
.collect();
|
1706
|
+
let mut mock_client = mock_workflow_client();
|
1707
|
+
mock_client
|
1708
|
+
.expect_complete_workflow_task()
|
1709
|
+
.times(3)
|
1710
|
+
.returning(|_| Ok(Default::default()));
|
1711
|
+
let mut mock_cfg = MockPollCfg::new(tasks, true, 0);
|
1712
|
+
mock_cfg.mock_client = mock_client;
|
1713
|
+
let mut mock = build_mock_pollers(mock_cfg);
|
1714
|
+
mock.worker_cfg(|wc| {
|
1715
|
+
wc.max_cached_workflows = 3;
|
1716
|
+
wc.max_outstanding_workflow_tasks = 3;
|
1717
|
+
});
|
1718
|
+
let core = mock_worker(mock);
|
1719
|
+
// Poll 4 times, completing once, such that max tasks are never exceeded
|
1720
|
+
let p1 = core.poll_workflow_activation().await.unwrap();
|
1721
|
+
let p2 = core.poll_workflow_activation().await.unwrap();
|
1722
|
+
let p3 = core.poll_workflow_activation().await.unwrap();
|
1723
|
+
for (i, p_res) in [&p1, &p2, &p3].into_iter().enumerate() {
|
1724
|
+
assert_matches!(
|
1725
|
+
&p_res.jobs[0].variant,
|
1726
|
+
Some(workflow_activation_job::Variant::StartWorkflow(sw))
|
1727
|
+
if sw.workflow_id == format!("wf-{}", i + 1)
|
1728
|
+
);
|
1729
|
+
}
|
1730
|
+
// Complete first task to free a wft slot. Cache size is at 3
|
1731
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1732
|
+
p1.run_id,
|
1733
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1734
|
+
))
|
1735
|
+
.await
|
1736
|
+
.unwrap();
|
1737
|
+
// Now we're at cache limit. We will poll for a task, discover it is for a new run, issue
|
1738
|
+
// an eviction, and buffer the new run task. However, the run we're trying to evict has pending
|
1739
|
+
// activations! Thus, we must complete them first before this poll will unblock, and then it
|
1740
|
+
// will unblock with the eviciton.
|
1741
|
+
let p4 = core.poll_workflow_activation();
|
1742
|
+
// Make sure the task gets buffered before we start the complete, so the LRU list is in the
|
1743
|
+
// expected order and what we expect to evict will be evicted.
|
1744
|
+
advance_fut!(p4);
|
1745
|
+
let p4 = async {
|
1746
|
+
let p4 = p4.await.unwrap();
|
1747
|
+
assert_matches!(
|
1748
|
+
&p4.jobs.as_slice(),
|
1749
|
+
[WorkflowActivationJob {
|
1750
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1751
|
+
}]
|
1752
|
+
);
|
1753
|
+
p4
|
1754
|
+
};
|
1755
|
+
let p2_pending_completer = async {
|
1756
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1757
|
+
p2.run_id,
|
1758
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1759
|
+
))
|
1760
|
+
.await
|
1761
|
+
.unwrap();
|
1762
|
+
};
|
1763
|
+
let (p4, _) = join!(p4, p2_pending_completer);
|
1764
|
+
assert_eq!(core.cached_workflows().await, 3);
|
1765
|
+
|
1766
|
+
// This poll should also block until the eviction is actually completed
|
1767
|
+
let blocking_poll = async {
|
1768
|
+
let res = core.poll_workflow_activation().await.unwrap();
|
1769
|
+
assert_matches!(
|
1770
|
+
&res.jobs[0].variant,
|
1771
|
+
Some(workflow_activation_job::Variant::StartWorkflow(sw))
|
1772
|
+
if sw.workflow_id == format!("wf-{}", 4)
|
1773
|
+
);
|
1774
|
+
res
|
1775
|
+
};
|
1776
|
+
let complete_evict = async {
|
1777
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(p4.run_id))
|
1778
|
+
.await
|
1779
|
+
.unwrap();
|
1780
|
+
};
|
1781
|
+
|
1782
|
+
let (_p5, _) = join!(blocking_poll, complete_evict);
|
1783
|
+
assert_eq!(core.cached_workflows().await, 3);
|
1784
|
+
// The next poll will get an buffer a task for a new run, and generate an eviction for p3 but
|
1785
|
+
// that eviction cannot be obtained until we complete the existing outstanding task.
|
1786
|
+
let p6 = async {
|
1787
|
+
let p6 = core.poll_workflow_activation().await.unwrap();
|
1788
|
+
assert_matches!(
|
1789
|
+
p6.jobs.as_slice(),
|
1790
|
+
[WorkflowActivationJob {
|
1791
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1792
|
+
}]
|
1793
|
+
);
|
1794
|
+
p6
|
1795
|
+
};
|
1796
|
+
let completer = async {
|
1797
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1798
|
+
p3.run_id,
|
1799
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1800
|
+
))
|
1801
|
+
.await
|
1802
|
+
.unwrap();
|
1803
|
+
};
|
1804
|
+
let (p6, _) = join!(p6, completer);
|
1805
|
+
let complete_evict = async {
|
1806
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(p6.run_id))
|
1807
|
+
.await
|
1808
|
+
.unwrap();
|
1809
|
+
};
|
1810
|
+
let blocking_poll = async {
|
1811
|
+
// This poll will also block until the last eviction goes through, and when it does it'll
|
1812
|
+
// produce the final start workflow task
|
1813
|
+
let res = core.poll_workflow_activation().await.unwrap();
|
1814
|
+
assert_matches!(
|
1815
|
+
&res.jobs[0].variant,
|
1816
|
+
Some(workflow_activation_job::Variant::StartWorkflow(sw))
|
1817
|
+
if sw.workflow_id == "wf-5"
|
1818
|
+
);
|
1819
|
+
};
|
1820
|
+
|
1821
|
+
join!(blocking_poll, complete_evict);
|
1822
|
+
// p5 outstanding and final poll outstanding -- hence one permit available
|
1823
|
+
assert_eq!(core.available_wft_permits().await, 1);
|
1824
|
+
assert_eq!(core.cached_workflows().await, 3);
|
1825
|
+
}
|
1826
|
+
|
1827
|
+
#[tokio::test]
|
1828
|
+
async fn eviction_waits_until_replay_finished() {
|
1829
|
+
let wfid = "fake_wf_id";
|
1830
|
+
let t = canned_histories::long_sequential_timers(3);
|
1831
|
+
let mock = mock_workflow_client();
|
1832
|
+
let mock = single_hist_mock_sg(wfid, t, &[3], mock, true);
|
1833
|
+
let core = mock_worker(mock);
|
1834
|
+
|
1835
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1836
|
+
assert_eq!(activation.history_length, 3);
|
1837
|
+
// Immediately request eviction after getting start workflow
|
1838
|
+
core.request_workflow_eviction(&activation.run_id);
|
1839
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1840
|
+
activation.run_id,
|
1841
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1842
|
+
))
|
1843
|
+
.await
|
1844
|
+
.unwrap();
|
1845
|
+
let t1_fired = core.poll_workflow_activation().await.unwrap();
|
1846
|
+
assert_eq!(t1_fired.history_length, 8);
|
1847
|
+
assert_matches!(
|
1848
|
+
t1_fired.jobs.as_slice(),
|
1849
|
+
[WorkflowActivationJob {
|
1850
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1851
|
+
}]
|
1852
|
+
);
|
1853
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1854
|
+
t1_fired.run_id,
|
1855
|
+
start_timer_cmd(2, Duration::from_secs(1)),
|
1856
|
+
))
|
1857
|
+
.await
|
1858
|
+
.unwrap();
|
1859
|
+
let t2_fired = core.poll_workflow_activation().await.unwrap();
|
1860
|
+
assert_eq!(t2_fired.history_length, 13);
|
1861
|
+
assert_matches!(
|
1862
|
+
t2_fired.jobs.as_slice(),
|
1863
|
+
[WorkflowActivationJob {
|
1864
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1865
|
+
}]
|
1866
|
+
);
|
1867
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1868
|
+
t2_fired.run_id,
|
1869
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1870
|
+
))
|
1871
|
+
.await
|
1872
|
+
.unwrap();
|
1873
|
+
|
1874
|
+
core.shutdown().await;
|
1875
|
+
}
|
1876
|
+
|
1877
|
+
#[tokio::test]
|
1878
|
+
async fn autocompletes_wft_no_work() {
|
1879
|
+
let wfid = "fake_wf_id";
|
1880
|
+
let activity_id = "1";
|
1881
|
+
|
1882
|
+
let mut t = TestHistoryBuilder::default();
|
1883
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1884
|
+
t.add_full_wf_task();
|
1885
|
+
let scheduled_event_id = t.add_activity_task_scheduled(activity_id);
|
1886
|
+
t.add_full_wf_task();
|
1887
|
+
t.add_we_signaled("sig1", vec![]);
|
1888
|
+
t.add_full_wf_task();
|
1889
|
+
let started_event_id = t.add_activity_task_started(scheduled_event_id);
|
1890
|
+
t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
|
1891
|
+
t.add_full_wf_task();
|
1892
|
+
let mock = mock_workflow_client();
|
1893
|
+
let mut mock = single_hist_mock_sg(wfid, t, &[1, 2, 3, 4], mock, true);
|
1894
|
+
mock.worker_cfg(|w| w.max_cached_workflows = 1);
|
1895
|
+
let core = mock_worker(mock);
|
1896
|
+
|
1897
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1898
|
+
assert_matches!(
|
1899
|
+
act.jobs.as_slice(),
|
1900
|
+
[WorkflowActivationJob {
|
1901
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
|
1902
|
+
}]
|
1903
|
+
);
|
1904
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1905
|
+
act.run_id,
|
1906
|
+
ScheduleActivity {
|
1907
|
+
seq: 1,
|
1908
|
+
activity_id: activity_id.to_string(),
|
1909
|
+
cancellation_type: ActivityCancellationType::Abandon as i32,
|
1910
|
+
..Default::default()
|
1911
|
+
}
|
1912
|
+
.into(),
|
1913
|
+
))
|
1914
|
+
.await
|
1915
|
+
.unwrap();
|
1916
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1917
|
+
assert_matches!(
|
1918
|
+
act.jobs.as_slice(),
|
1919
|
+
[WorkflowActivationJob {
|
1920
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1921
|
+
}]
|
1922
|
+
);
|
1923
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1924
|
+
act.run_id,
|
1925
|
+
RequestCancelActivity { seq: 1 }.into(),
|
1926
|
+
))
|
1927
|
+
.await
|
1928
|
+
.unwrap();
|
1929
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1930
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
|
1931
|
+
.await
|
1932
|
+
.unwrap();
|
1933
|
+
// The last task will autocomplete, and thus this will return shutdown since there is no more
|
1934
|
+
// work
|
1935
|
+
assert_matches!(
|
1936
|
+
core.poll_workflow_activation().await.unwrap_err(),
|
1937
|
+
PollWfError::ShutDown
|
1938
|
+
);
|
1939
|
+
|
1940
|
+
core.shutdown().await;
|
1941
|
+
}
|
1942
|
+
|
1943
|
+
#[tokio::test]
|
1944
|
+
async fn no_race_acquiring_permits() {
|
1945
|
+
let wfid = "fake_wf_id";
|
1946
|
+
let mut mock_client = mock_manual_workflow_client();
|
1947
|
+
// We need to allow two polls to happen by triggering two processing events in the workflow
|
1948
|
+
// stream, but then delivering the actual tasks after that
|
1949
|
+
let task_barr: &'static Barrier = Box::leak(Box::new(Barrier::new(2)));
|
1950
|
+
mock_client
|
1951
|
+
.expect_poll_workflow_task()
|
1952
|
+
.returning(move |_, _| {
|
1953
|
+
let t = canned_histories::single_timer("1");
|
1954
|
+
let poll_resp =
|
1955
|
+
hist_to_poll_resp(&t, wfid.to_owned(), 2.into(), TEST_Q.to_string()).resp;
|
1956
|
+
async move {
|
1957
|
+
task_barr.wait().await;
|
1958
|
+
Ok(poll_resp.clone())
|
1959
|
+
}
|
1960
|
+
.boxed()
|
1961
|
+
});
|
1962
|
+
mock_client
|
1963
|
+
.expect_complete_workflow_task()
|
1964
|
+
.returning(|_| async move { Ok(Default::default()) }.boxed());
|
1965
|
+
|
1966
|
+
let worker = Worker::new_test(
|
1967
|
+
test_worker_cfg()
|
1968
|
+
.max_outstanding_workflow_tasks(1_usize)
|
1969
|
+
.max_cached_workflows(10_usize)
|
1970
|
+
.build()
|
1971
|
+
.unwrap(),
|
1972
|
+
mock_client,
|
1973
|
+
);
|
1974
|
+
|
1975
|
+
// Two polls in a row, both of which will get stuck on the barrier and are only allowed to
|
1976
|
+
// proceed after a call which will cause the workflow stream to process an event. Without the
|
1977
|
+
// fix, this would've meant the stream though it was OK to poll twice, but once the tasks
|
1978
|
+
// are received, it would find there was only one permit.
|
1979
|
+
let poll_1_f = async {
|
1980
|
+
let r = worker.poll_workflow_activation().await.unwrap();
|
1981
|
+
worker
|
1982
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1983
|
+
r.run_id,
|
1984
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1985
|
+
))
|
1986
|
+
.await
|
1987
|
+
.unwrap();
|
1988
|
+
};
|
1989
|
+
let poll_2_f = async {
|
1990
|
+
let r = worker.poll_workflow_activation().await.unwrap();
|
1991
|
+
worker
|
1992
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1993
|
+
r.run_id,
|
1994
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1995
|
+
))
|
1996
|
+
.await
|
1997
|
+
.unwrap();
|
1998
|
+
};
|
1999
|
+
let other_f = async {
|
2000
|
+
worker.cached_workflows().await;
|
2001
|
+
task_barr.wait().await;
|
2002
|
+
worker.cached_workflows().await;
|
2003
|
+
task_barr.wait().await;
|
2004
|
+
};
|
2005
|
+
join!(poll_1_f, poll_2_f, other_f);
|
2006
|
+
}
|
2007
|
+
|
2008
|
+
#[tokio::test]
|
2009
|
+
async fn continue_as_new_preserves_some_values() {
|
2010
|
+
let wfid = "fake_wf_id";
|
2011
|
+
let memo = HashMap::<String, Payload>::from([("enchi".to_string(), b"cat".into())]).into();
|
2012
|
+
let search = HashMap::<String, Payload>::from([("noisy".to_string(), b"kitty".into())]).into();
|
2013
|
+
let retry_policy = RetryPolicy {
|
2014
|
+
backoff_coefficient: 13.37,
|
2015
|
+
..Default::default()
|
2016
|
+
};
|
2017
|
+
let mut wes_attrs = default_wes_attribs();
|
2018
|
+
wes_attrs.memo = Some(memo);
|
2019
|
+
wes_attrs.search_attributes = Some(search);
|
2020
|
+
wes_attrs.retry_policy = Some(retry_policy);
|
2021
|
+
let mut mock_client = mock_workflow_client();
|
2022
|
+
let hist = {
|
2023
|
+
let mut t = TestHistoryBuilder::default();
|
2024
|
+
t.add(
|
2025
|
+
EventType::WorkflowExecutionStarted,
|
2026
|
+
wes_attrs.clone().into(),
|
2027
|
+
);
|
2028
|
+
t.add_full_wf_task();
|
2029
|
+
t
|
2030
|
+
};
|
2031
|
+
mock_client
|
2032
|
+
.expect_poll_workflow_task()
|
2033
|
+
.returning(move |_, _| {
|
2034
|
+
Ok(hist_to_poll_resp(
|
2035
|
+
&hist,
|
2036
|
+
wfid.to_owned(),
|
2037
|
+
ResponseType::AllHistory,
|
2038
|
+
TEST_Q.to_string(),
|
2039
|
+
)
|
2040
|
+
.resp)
|
2041
|
+
});
|
2042
|
+
mock_client
|
2043
|
+
.expect_complete_workflow_task()
|
2044
|
+
.returning(move |mut c| {
|
2045
|
+
let can_cmd = c.commands.pop().unwrap().attributes.unwrap();
|
2046
|
+
if let Attributes::ContinueAsNewWorkflowExecutionCommandAttributes(a) = can_cmd {
|
2047
|
+
assert_eq!(a.workflow_type.unwrap().name, "meow");
|
2048
|
+
assert_eq!(a.memo, wes_attrs.memo);
|
2049
|
+
assert_eq!(a.search_attributes, wes_attrs.search_attributes);
|
2050
|
+
assert_eq!(a.retry_policy, wes_attrs.retry_policy);
|
2051
|
+
} else {
|
2052
|
+
panic!("Wrong attributes type");
|
2053
|
+
}
|
2054
|
+
Ok(Default::default())
|
2055
|
+
});
|
2056
|
+
|
2057
|
+
let worker = Worker::new_test(test_worker_cfg().build().unwrap(), mock_client);
|
2058
|
+
let r = worker.poll_workflow_activation().await.unwrap();
|
2059
|
+
worker
|
2060
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
2061
|
+
r.run_id,
|
2062
|
+
ContinueAsNewWorkflowExecution {
|
2063
|
+
workflow_type: "meow".to_string(),
|
2064
|
+
..Default::default()
|
2065
|
+
}
|
2066
|
+
.into(),
|
2067
|
+
))
|
2068
|
+
.await
|
2069
|
+
.unwrap();
|
2070
|
+
}
|