temporalio 0.0.0 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +301 -0
- data/bridge/Cargo.lock +2888 -0
- data/bridge/Cargo.toml +27 -0
- data/bridge/sdk-core/ARCHITECTURE.md +76 -0
- data/bridge/sdk-core/Cargo.lock +2606 -0
- data/bridge/sdk-core/Cargo.toml +2 -0
- data/bridge/sdk-core/LICENSE.txt +23 -0
- data/bridge/sdk-core/README.md +104 -0
- data/bridge/sdk-core/arch_docs/diagrams/README.md +10 -0
- data/bridge/sdk-core/arch_docs/diagrams/sticky_queues.puml +40 -0
- data/bridge/sdk-core/arch_docs/diagrams/workflow_internals.svg +1 -0
- data/bridge/sdk-core/arch_docs/sticky_queues.md +51 -0
- data/bridge/sdk-core/client/Cargo.toml +40 -0
- data/bridge/sdk-core/client/LICENSE.txt +23 -0
- data/bridge/sdk-core/client/src/lib.rs +1286 -0
- data/bridge/sdk-core/client/src/metrics.rs +165 -0
- data/bridge/sdk-core/client/src/raw.rs +932 -0
- data/bridge/sdk-core/client/src/retry.rs +751 -0
- data/bridge/sdk-core/client/src/workflow_handle/mod.rs +185 -0
- data/bridge/sdk-core/core/Cargo.toml +116 -0
- data/bridge/sdk-core/core/LICENSE.txt +23 -0
- data/bridge/sdk-core/core/benches/workflow_replay.rs +76 -0
- data/bridge/sdk-core/core/src/abstractions.rs +166 -0
- data/bridge/sdk-core/core/src/core_tests/activity_tasks.rs +1014 -0
- data/bridge/sdk-core/core/src/core_tests/child_workflows.rs +221 -0
- data/bridge/sdk-core/core/src/core_tests/determinism.rs +107 -0
- data/bridge/sdk-core/core/src/core_tests/local_activities.rs +925 -0
- data/bridge/sdk-core/core/src/core_tests/mod.rs +100 -0
- data/bridge/sdk-core/core/src/core_tests/queries.rs +894 -0
- data/bridge/sdk-core/core/src/core_tests/replay_flag.rs +65 -0
- data/bridge/sdk-core/core/src/core_tests/workers.rs +259 -0
- data/bridge/sdk-core/core/src/core_tests/workflow_cancels.rs +124 -0
- data/bridge/sdk-core/core/src/core_tests/workflow_tasks.rs +2090 -0
- data/bridge/sdk-core/core/src/ephemeral_server/mod.rs +515 -0
- data/bridge/sdk-core/core/src/lib.rs +282 -0
- data/bridge/sdk-core/core/src/pollers/mod.rs +54 -0
- data/bridge/sdk-core/core/src/pollers/poll_buffer.rs +297 -0
- data/bridge/sdk-core/core/src/protosext/mod.rs +428 -0
- data/bridge/sdk-core/core/src/replay/mod.rs +215 -0
- data/bridge/sdk-core/core/src/retry_logic.rs +202 -0
- data/bridge/sdk-core/core/src/telemetry/log_export.rs +190 -0
- data/bridge/sdk-core/core/src/telemetry/metrics.rs +428 -0
- data/bridge/sdk-core/core/src/telemetry/mod.rs +407 -0
- data/bridge/sdk-core/core/src/telemetry/prometheus_server.rs +78 -0
- data/bridge/sdk-core/core/src/test_help/mod.rs +889 -0
- data/bridge/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +580 -0
- data/bridge/sdk-core/core/src/worker/activities/local_activities.rs +1048 -0
- data/bridge/sdk-core/core/src/worker/activities.rs +481 -0
- data/bridge/sdk-core/core/src/worker/client/mocks.rs +87 -0
- data/bridge/sdk-core/core/src/worker/client.rs +373 -0
- data/bridge/sdk-core/core/src/worker/mod.rs +570 -0
- data/bridge/sdk-core/core/src/worker/workflow/bridge.rs +37 -0
- data/bridge/sdk-core/core/src/worker/workflow/driven_workflow.rs +101 -0
- data/bridge/sdk-core/core/src/worker/workflow/history_update.rs +532 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs +907 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_external_state_machine.rs +294 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/cancel_workflow_state_machine.rs +167 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +858 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/complete_workflow_state_machine.rs +136 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs +157 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/fail_workflow_state_machine.rs +129 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs +1450 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/mod.rs +316 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/modify_workflow_properties_state_machine.rs +178 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/patch_state_machine.rs +708 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/signal_external_state_machine.rs +439 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/timer_state_machine.rs +435 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs +175 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs +242 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines/local_acts.rs +96 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +1200 -0
- data/bridge/sdk-core/core/src/worker/workflow/machines/workflow_task_state_machine.rs +272 -0
- data/bridge/sdk-core/core/src/worker/workflow/managed_run/managed_wf_test.rs +198 -0
- data/bridge/sdk-core/core/src/worker/workflow/managed_run.rs +655 -0
- data/bridge/sdk-core/core/src/worker/workflow/mod.rs +1200 -0
- data/bridge/sdk-core/core/src/worker/workflow/run_cache.rs +145 -0
- data/bridge/sdk-core/core/src/worker/workflow/wft_poller.rs +88 -0
- data/bridge/sdk-core/core/src/worker/workflow/workflow_stream.rs +985 -0
- data/bridge/sdk-core/core-api/Cargo.toml +32 -0
- data/bridge/sdk-core/core-api/LICENSE.txt +23 -0
- data/bridge/sdk-core/core-api/src/errors.rs +95 -0
- data/bridge/sdk-core/core-api/src/lib.rs +109 -0
- data/bridge/sdk-core/core-api/src/telemetry.rs +147 -0
- data/bridge/sdk-core/core-api/src/worker.rs +148 -0
- data/bridge/sdk-core/etc/deps.svg +162 -0
- data/bridge/sdk-core/etc/dynamic-config.yaml +2 -0
- data/bridge/sdk-core/etc/otel-collector-config.yaml +36 -0
- data/bridge/sdk-core/etc/prometheus.yaml +6 -0
- data/bridge/sdk-core/etc/regen-depgraph.sh +5 -0
- data/bridge/sdk-core/fsm/Cargo.toml +18 -0
- data/bridge/sdk-core/fsm/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/README.md +3 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +27 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +647 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/progress.rs +8 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.rs +18 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.stderr +12 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dynamic_dest_pass.rs +41 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.rs +14 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/forgot_name_fail.stderr +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_arg_pass.rs +32 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/handler_pass.rs +31 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/medium_complex_pass.rs +46 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.rs +29 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +12 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/simple_pass.rs +32 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.rs +18 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/struct_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.rs +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_more_item_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.rs +11 -0
- data/bridge/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/tuple_zero_item_event_variant_fail.stderr +5 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/Cargo.toml +14 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/LICENSE.txt +23 -0
- data/bridge/sdk-core/fsm/rustfsm_trait/src/lib.rs +249 -0
- data/bridge/sdk-core/fsm/src/lib.rs +2 -0
- data/bridge/sdk-core/histories/evict_while_la_running_no_interference-23_history.bin +0 -0
- data/bridge/sdk-core/histories/evict_while_la_running_no_interference-85_history.bin +0 -0
- data/bridge/sdk-core/histories/fail_wf_task.bin +0 -0
- data/bridge/sdk-core/histories/timer_workflow_history.bin +0 -0
- data/bridge/sdk-core/integ-with-otel.sh +7 -0
- data/bridge/sdk-core/protos/api_upstream/README.md +9 -0
- data/bridge/sdk-core/protos/api_upstream/api-linter.yaml +40 -0
- data/bridge/sdk-core/protos/api_upstream/buf.yaml +9 -0
- data/bridge/sdk-core/protos/api_upstream/build/go.mod +7 -0
- data/bridge/sdk-core/protos/api_upstream/build/go.sum +5 -0
- data/bridge/sdk-core/protos/api_upstream/build/tools.go +29 -0
- data/bridge/sdk-core/protos/api_upstream/dependencies/gogoproto/gogo.proto +141 -0
- data/bridge/sdk-core/protos/api_upstream/go.mod +6 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/batch/v1/message.proto +89 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto +260 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/common/v1/message.proto +112 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +47 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/command_type.proto +57 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/common.proto +56 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/event_type.proto +170 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto +118 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/interaction_type.proto +39 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/namespace.proto +51 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/query.proto +50 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/reset.proto +41 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/schedule.proto +60 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/task_queue.proto +59 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/update.proto +40 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/enums/v1/workflow.proto +122 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/errordetails/v1/message.proto +108 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/failure/v1/message.proto +114 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/filter/v1/message.proto +56 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto +758 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/interaction/v1/message.proto +87 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/namespace/v1/message.proto +97 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +121 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +80 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/query/v1/message.proto +61 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/replication/v1/message.proto +55 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/schedule/v1/message.proto +379 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +108 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/version/v1/message.proto +59 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflow/v1/message.proto +146 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +1168 -0
- data/bridge/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +415 -0
- data/bridge/sdk-core/protos/grpc/health/v1/health.proto +63 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_result/activity_result.proto +78 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/activity_task/activity_task.proto +79 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/child_workflow/child_workflow.proto +77 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/common/common.proto +15 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/core_interface.proto +30 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/external_data/external_data.proto +30 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +263 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +304 -0
- data/bridge/sdk-core/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto +29 -0
- data/bridge/sdk-core/protos/testsrv_upstream/api-linter.yaml +38 -0
- data/bridge/sdk-core/protos/testsrv_upstream/buf.yaml +13 -0
- data/bridge/sdk-core/protos/testsrv_upstream/dependencies/gogoproto/gogo.proto +141 -0
- data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/request_response.proto +63 -0
- data/bridge/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/service.proto +90 -0
- data/bridge/sdk-core/rustfmt.toml +1 -0
- data/bridge/sdk-core/sdk/Cargo.toml +47 -0
- data/bridge/sdk-core/sdk/LICENSE.txt +23 -0
- data/bridge/sdk-core/sdk/src/activity_context.rs +230 -0
- data/bridge/sdk-core/sdk/src/app_data.rs +37 -0
- data/bridge/sdk-core/sdk/src/interceptors.rs +50 -0
- data/bridge/sdk-core/sdk/src/lib.rs +794 -0
- data/bridge/sdk-core/sdk/src/payload_converter.rs +11 -0
- data/bridge/sdk-core/sdk/src/workflow_context/options.rs +295 -0
- data/bridge/sdk-core/sdk/src/workflow_context.rs +694 -0
- data/bridge/sdk-core/sdk/src/workflow_future.rs +499 -0
- data/bridge/sdk-core/sdk-core-protos/Cargo.toml +30 -0
- data/bridge/sdk-core/sdk-core-protos/LICENSE.txt +23 -0
- data/bridge/sdk-core/sdk-core-protos/build.rs +107 -0
- data/bridge/sdk-core/sdk-core-protos/src/constants.rs +7 -0
- data/bridge/sdk-core/sdk-core-protos/src/history_builder.rs +544 -0
- data/bridge/sdk-core/sdk-core-protos/src/history_info.rs +230 -0
- data/bridge/sdk-core/sdk-core-protos/src/lib.rs +1970 -0
- data/bridge/sdk-core/sdk-core-protos/src/task_token.rs +38 -0
- data/bridge/sdk-core/sdk-core-protos/src/utilities.rs +14 -0
- data/bridge/sdk-core/test-utils/Cargo.toml +36 -0
- data/bridge/sdk-core/test-utils/src/canned_histories.rs +1579 -0
- data/bridge/sdk-core/test-utils/src/histfetch.rs +28 -0
- data/bridge/sdk-core/test-utils/src/lib.rs +650 -0
- data/bridge/sdk-core/tests/integ_tests/client_tests.rs +36 -0
- data/bridge/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +128 -0
- data/bridge/sdk-core/tests/integ_tests/heartbeat_tests.rs +221 -0
- data/bridge/sdk-core/tests/integ_tests/metrics_tests.rs +37 -0
- data/bridge/sdk-core/tests/integ_tests/polling_tests.rs +133 -0
- data/bridge/sdk-core/tests/integ_tests/queries_tests.rs +437 -0
- data/bridge/sdk-core/tests/integ_tests/visibility_tests.rs +93 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/activities.rs +878 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/appdata_propagation.rs +61 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_external.rs +59 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +58 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +50 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs +60 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +54 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +788 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/modify_wf_properties.rs +53 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/patches.rs +113 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/replay.rs +223 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/resets.rs +93 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/signals.rs +167 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +99 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/timers.rs +131 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +75 -0
- data/bridge/sdk-core/tests/integ_tests/workflow_tests.rs +597 -0
- data/bridge/sdk-core/tests/load_tests.rs +191 -0
- data/bridge/sdk-core/tests/main.rs +113 -0
- data/bridge/sdk-core/tests/runner.rs +93 -0
- data/bridge/src/connection.rs +186 -0
- data/bridge/src/lib.rs +239 -0
- data/bridge/src/runtime.rs +54 -0
- data/bridge/src/worker.rs +124 -0
- data/ext/Rakefile +9 -0
- data/lib/bridge.so +0 -0
- data/lib/gen/dependencies/gogoproto/gogo_pb.rb +14 -0
- data/lib/gen/temporal/api/batch/v1/message_pb.rb +50 -0
- data/lib/gen/temporal/api/command/v1/message_pb.rb +174 -0
- data/lib/gen/temporal/api/common/v1/message_pb.rb +69 -0
- data/lib/gen/temporal/api/enums/v1/batch_operation_pb.rb +33 -0
- data/lib/gen/temporal/api/enums/v1/command_type_pb.rb +39 -0
- data/lib/gen/temporal/api/enums/v1/common_pb.rb +42 -0
- data/lib/gen/temporal/api/enums/v1/event_type_pb.rb +68 -0
- data/lib/gen/temporal/api/enums/v1/failed_cause_pb.rb +77 -0
- data/lib/gen/temporal/api/enums/v1/interaction_type_pb.rb +25 -0
- data/lib/gen/temporal/api/enums/v1/namespace_pb.rb +37 -0
- data/lib/gen/temporal/api/enums/v1/query_pb.rb +31 -0
- data/lib/gen/temporal/api/enums/v1/reset_pb.rb +24 -0
- data/lib/gen/temporal/api/enums/v1/schedule_pb.rb +28 -0
- data/lib/gen/temporal/api/enums/v1/task_queue_pb.rb +30 -0
- data/lib/gen/temporal/api/enums/v1/update_pb.rb +23 -0
- data/lib/gen/temporal/api/enums/v1/workflow_pb.rb +89 -0
- data/lib/gen/temporal/api/errordetails/v1/message_pb.rb +84 -0
- data/lib/gen/temporal/api/failure/v1/message_pb.rb +83 -0
- data/lib/gen/temporal/api/filter/v1/message_pb.rb +40 -0
- data/lib/gen/temporal/api/history/v1/message_pb.rb +490 -0
- data/lib/gen/temporal/api/interaction/v1/message_pb.rb +49 -0
- data/lib/gen/temporal/api/namespace/v1/message_pb.rb +63 -0
- data/lib/gen/temporal/api/operatorservice/v1/request_response_pb.rb +85 -0
- data/lib/gen/temporal/api/operatorservice/v1/service_pb.rb +20 -0
- data/lib/gen/temporal/api/query/v1/message_pb.rb +38 -0
- data/lib/gen/temporal/api/replication/v1/message_pb.rb +37 -0
- data/lib/gen/temporal/api/schedule/v1/message_pb.rb +149 -0
- data/lib/gen/temporal/api/taskqueue/v1/message_pb.rb +73 -0
- data/lib/gen/temporal/api/version/v1/message_pb.rb +41 -0
- data/lib/gen/temporal/api/workflow/v1/message_pb.rb +111 -0
- data/lib/gen/temporal/api/workflowservice/v1/request_response_pb.rb +788 -0
- data/lib/gen/temporal/api/workflowservice/v1/service_pb.rb +20 -0
- data/lib/gen/temporal/sdk/core/activity_result/activity_result_pb.rb +58 -0
- data/lib/gen/temporal/sdk/core/activity_task/activity_task_pb.rb +57 -0
- data/lib/gen/temporal/sdk/core/bridge/bridge_pb.rb +222 -0
- data/lib/gen/temporal/sdk/core/child_workflow/child_workflow_pb.rb +57 -0
- data/lib/gen/temporal/sdk/core/common/common_pb.rb +22 -0
- data/lib/gen/temporal/sdk/core/core_interface_pb.rb +34 -0
- data/lib/gen/temporal/sdk/core/external_data/external_data_pb.rb +27 -0
- data/lib/gen/temporal/sdk/core/workflow_activation/workflow_activation_pb.rb +165 -0
- data/lib/gen/temporal/sdk/core/workflow_commands/workflow_commands_pb.rb +196 -0
- data/lib/gen/temporal/sdk/core/workflow_completion/workflow_completion_pb.rb +34 -0
- data/lib/temporalio/activity/context.rb +97 -0
- data/lib/temporalio/activity/info.rb +67 -0
- data/lib/temporalio/activity.rb +85 -0
- data/lib/temporalio/bridge/error.rb +8 -0
- data/lib/temporalio/bridge.rb +14 -0
- data/lib/temporalio/client/implementation.rb +340 -0
- data/lib/temporalio/client/workflow_handle.rb +243 -0
- data/lib/temporalio/client.rb +131 -0
- data/lib/temporalio/connection.rb +751 -0
- data/lib/temporalio/data_converter.rb +191 -0
- data/lib/temporalio/error/failure.rb +194 -0
- data/lib/temporalio/error/workflow_failure.rb +19 -0
- data/lib/temporalio/errors.rb +40 -0
- data/lib/temporalio/failure_converter/base.rb +26 -0
- data/lib/temporalio/failure_converter/basic.rb +319 -0
- data/lib/temporalio/failure_converter.rb +7 -0
- data/lib/temporalio/interceptor/chain.rb +28 -0
- data/lib/temporalio/interceptor/client.rb +123 -0
- data/lib/temporalio/payload_codec/base.rb +32 -0
- data/lib/temporalio/payload_converter/base.rb +24 -0
- data/lib/temporalio/payload_converter/bytes.rb +27 -0
- data/lib/temporalio/payload_converter/composite.rb +49 -0
- data/lib/temporalio/payload_converter/encoding_base.rb +35 -0
- data/lib/temporalio/payload_converter/json.rb +26 -0
- data/lib/temporalio/payload_converter/nil.rb +26 -0
- data/lib/temporalio/payload_converter.rb +14 -0
- data/lib/temporalio/retry_policy.rb +82 -0
- data/lib/temporalio/retry_state.rb +35 -0
- data/lib/temporalio/runtime.rb +25 -0
- data/lib/temporalio/timeout_type.rb +29 -0
- data/lib/temporalio/version.rb +3 -0
- data/lib/temporalio/worker/activity_runner.rb +92 -0
- data/lib/temporalio/worker/activity_worker.rb +138 -0
- data/lib/temporalio/worker/reactor.rb +46 -0
- data/lib/temporalio/worker/runner.rb +63 -0
- data/lib/temporalio/worker/sync_worker.rb +88 -0
- data/lib/temporalio/worker/thread_pool_executor.rb +51 -0
- data/lib/temporalio/worker.rb +198 -0
- data/lib/temporalio/workflow/execution_info.rb +54 -0
- data/lib/temporalio/workflow/execution_status.rb +36 -0
- data/lib/temporalio/workflow/id_reuse_policy.rb +36 -0
- data/lib/temporalio/workflow/query_reject_condition.rb +33 -0
- data/lib/temporalio.rb +12 -1
- data/lib/thermite_patch.rb +23 -0
- data/temporalio.gemspec +45 -0
- metadata +566 -9
- data/lib/temporal/version.rb +0 -3
- data/lib/temporal.rb +0 -4
- data/temporal.gemspec +0 -20
@@ -0,0 +1,2090 @@
|
|
1
|
+
use crate::{
|
2
|
+
advance_fut, job_assert,
|
3
|
+
replay::TestHistoryBuilder,
|
4
|
+
test_help::{
|
5
|
+
build_fake_worker, build_mock_pollers, build_multihist_mock_sg, canned_histories,
|
6
|
+
gen_assert_and_fail, gen_assert_and_reply, hist_to_poll_resp, mock_sdk, mock_sdk_cfg,
|
7
|
+
mock_worker, poll_and_reply, poll_and_reply_clears_outstanding_evicts, single_hist_mock_sg,
|
8
|
+
test_worker_cfg, FakeWfResponses, MockPollCfg, MocksHolder, ResponseType,
|
9
|
+
WorkflowCachingPolicy::{self, AfterEveryReply, NonSticky},
|
10
|
+
},
|
11
|
+
worker::client::mocks::{mock_manual_workflow_client, mock_workflow_client},
|
12
|
+
Worker,
|
13
|
+
};
|
14
|
+
use futures::{stream, FutureExt};
|
15
|
+
use rstest::{fixture, rstest};
|
16
|
+
use std::{
|
17
|
+
collections::{HashMap, VecDeque},
|
18
|
+
sync::{
|
19
|
+
atomic::{AtomicU64, Ordering},
|
20
|
+
Arc,
|
21
|
+
},
|
22
|
+
time::Duration,
|
23
|
+
};
|
24
|
+
use temporal_sdk::{ActivityOptions, CancellableFuture, WfContext};
|
25
|
+
use temporal_sdk_core_api::{errors::PollWfError, Worker as WorkerTrait};
|
26
|
+
use temporal_sdk_core_protos::{
|
27
|
+
coresdk::{
|
28
|
+
activity_result::{self as ar, activity_resolution, ActivityResolution},
|
29
|
+
workflow_activation::{
|
30
|
+
remove_from_cache::EvictionReason, workflow_activation_job, FireTimer, ResolveActivity,
|
31
|
+
StartWorkflow, UpdateRandomSeed, WorkflowActivationJob,
|
32
|
+
},
|
33
|
+
workflow_commands::{
|
34
|
+
ActivityCancellationType, CancelTimer, CompleteWorkflowExecution,
|
35
|
+
ContinueAsNewWorkflowExecution, FailWorkflowExecution, RequestCancelActivity,
|
36
|
+
ScheduleActivity,
|
37
|
+
},
|
38
|
+
workflow_completion::WorkflowActivationCompletion,
|
39
|
+
},
|
40
|
+
default_wes_attribs,
|
41
|
+
temporal::api::{
|
42
|
+
command::v1::command::Attributes,
|
43
|
+
common::v1::{Payload, RetryPolicy},
|
44
|
+
enums::v1::{EventType, WorkflowTaskFailedCause},
|
45
|
+
failure::v1::Failure,
|
46
|
+
history::v1::{history_event, TimerFiredEventAttributes},
|
47
|
+
workflowservice::v1::{
|
48
|
+
GetWorkflowExecutionHistoryResponse, RespondWorkflowTaskCompletedResponse,
|
49
|
+
},
|
50
|
+
},
|
51
|
+
DEFAULT_WORKFLOW_TYPE,
|
52
|
+
};
|
53
|
+
use temporal_sdk_core_test_utils::{fanout_tasks, start_timer_cmd};
|
54
|
+
use tokio::{
|
55
|
+
join,
|
56
|
+
sync::{Barrier, Semaphore},
|
57
|
+
};
|
58
|
+
|
59
|
+
#[fixture(hist_batches = &[])]
|
60
|
+
fn single_timer_setup(hist_batches: &'static [usize]) -> Worker {
|
61
|
+
let wfid = "fake_wf_id";
|
62
|
+
|
63
|
+
let t = canned_histories::single_timer("1");
|
64
|
+
build_fake_worker(wfid, t, hist_batches)
|
65
|
+
}
|
66
|
+
|
67
|
+
#[fixture(hist_batches = &[])]
|
68
|
+
fn single_activity_setup(hist_batches: &'static [usize]) -> Worker {
|
69
|
+
let wfid = "fake_wf_id";
|
70
|
+
|
71
|
+
let t = canned_histories::single_activity("fake_activity");
|
72
|
+
build_fake_worker(wfid, t, hist_batches)
|
73
|
+
}
|
74
|
+
|
75
|
+
#[fixture(hist_batches = &[])]
|
76
|
+
fn single_activity_failure_setup(hist_batches: &'static [usize]) -> Worker {
|
77
|
+
let wfid = "fake_wf_id";
|
78
|
+
|
79
|
+
let t = canned_histories::single_failed_activity("fake_activity");
|
80
|
+
build_fake_worker(wfid, t, hist_batches)
|
81
|
+
}
|
82
|
+
|
83
|
+
#[rstest]
|
84
|
+
#[case::incremental(single_timer_setup(&[1, 2]), NonSticky)]
|
85
|
+
#[case::replay(single_timer_setup(&[2]), NonSticky)]
|
86
|
+
#[case::incremental_evict(single_timer_setup(&[1, 2]), AfterEveryReply)]
|
87
|
+
#[case::replay_evict(single_timer_setup(&[2]), AfterEveryReply)]
|
88
|
+
#[tokio::test]
|
89
|
+
async fn single_timer(#[case] worker: Worker, #[case] evict: WorkflowCachingPolicy) {
|
90
|
+
poll_and_reply(
|
91
|
+
&worker,
|
92
|
+
evict,
|
93
|
+
&[
|
94
|
+
gen_assert_and_reply(
|
95
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
96
|
+
vec![start_timer_cmd(1, Duration::from_secs(1))],
|
97
|
+
),
|
98
|
+
gen_assert_and_reply(
|
99
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
100
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
101
|
+
),
|
102
|
+
],
|
103
|
+
)
|
104
|
+
.await;
|
105
|
+
}
|
106
|
+
|
107
|
+
#[rstest(worker,
|
108
|
+
case::incremental(single_activity_setup(&[1, 2])),
|
109
|
+
case::incremental_activity_failure(single_activity_failure_setup(&[1, 2])),
|
110
|
+
case::replay(single_activity_setup(&[2])),
|
111
|
+
case::replay_activity_failure(single_activity_failure_setup(&[2]))
|
112
|
+
)]
|
113
|
+
#[tokio::test]
|
114
|
+
async fn single_activity_completion(worker: Worker) {
|
115
|
+
poll_and_reply(
|
116
|
+
&worker,
|
117
|
+
NonSticky,
|
118
|
+
&[
|
119
|
+
gen_assert_and_reply(
|
120
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
121
|
+
vec![ScheduleActivity {
|
122
|
+
activity_id: "fake_activity".to_string(),
|
123
|
+
..Default::default()
|
124
|
+
}
|
125
|
+
.into()],
|
126
|
+
),
|
127
|
+
gen_assert_and_reply(
|
128
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(_)),
|
129
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
130
|
+
),
|
131
|
+
],
|
132
|
+
)
|
133
|
+
.await;
|
134
|
+
}
|
135
|
+
|
136
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
137
|
+
#[tokio::test]
|
138
|
+
async fn parallel_timer_test_across_wf_bridge(hist_batches: &'static [usize]) {
|
139
|
+
let wfid = "fake_wf_id";
|
140
|
+
let timer_1_id = 1;
|
141
|
+
let timer_2_id = 2;
|
142
|
+
|
143
|
+
let t = canned_histories::parallel_timer(
|
144
|
+
timer_1_id.to_string().as_str(),
|
145
|
+
timer_2_id.to_string().as_str(),
|
146
|
+
);
|
147
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
148
|
+
|
149
|
+
poll_and_reply(
|
150
|
+
&core,
|
151
|
+
NonSticky,
|
152
|
+
&[
|
153
|
+
gen_assert_and_reply(
|
154
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
155
|
+
vec![
|
156
|
+
start_timer_cmd(timer_1_id, Duration::from_secs(1)),
|
157
|
+
start_timer_cmd(timer_2_id, Duration::from_secs(1)),
|
158
|
+
],
|
159
|
+
),
|
160
|
+
gen_assert_and_reply(
|
161
|
+
&|res| {
|
162
|
+
assert_matches!(
|
163
|
+
res.jobs.as_slice(),
|
164
|
+
[
|
165
|
+
WorkflowActivationJob {
|
166
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(
|
167
|
+
FireTimer { seq: t1_id }
|
168
|
+
)),
|
169
|
+
},
|
170
|
+
WorkflowActivationJob {
|
171
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(
|
172
|
+
FireTimer { seq: t2_id }
|
173
|
+
)),
|
174
|
+
}
|
175
|
+
] => {
|
176
|
+
assert_eq!(t1_id, &timer_1_id);
|
177
|
+
assert_eq!(t2_id, &timer_2_id);
|
178
|
+
}
|
179
|
+
);
|
180
|
+
},
|
181
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
182
|
+
),
|
183
|
+
],
|
184
|
+
)
|
185
|
+
.await;
|
186
|
+
}
|
187
|
+
|
188
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
189
|
+
#[tokio::test]
|
190
|
+
async fn timer_cancel(hist_batches: &'static [usize]) {
|
191
|
+
let wfid = "fake_wf_id";
|
192
|
+
let timer_id = 1;
|
193
|
+
let cancel_timer_id = 2;
|
194
|
+
|
195
|
+
let t = canned_histories::cancel_timer(
|
196
|
+
timer_id.to_string().as_str(),
|
197
|
+
cancel_timer_id.to_string().as_str(),
|
198
|
+
);
|
199
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
200
|
+
|
201
|
+
poll_and_reply(
|
202
|
+
&core,
|
203
|
+
NonSticky,
|
204
|
+
&[
|
205
|
+
gen_assert_and_reply(
|
206
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
207
|
+
vec![
|
208
|
+
start_timer_cmd(cancel_timer_id, Duration::from_secs(1)),
|
209
|
+
start_timer_cmd(timer_id, Duration::from_secs(1)),
|
210
|
+
],
|
211
|
+
),
|
212
|
+
gen_assert_and_reply(
|
213
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
214
|
+
vec![
|
215
|
+
CancelTimer {
|
216
|
+
seq: cancel_timer_id,
|
217
|
+
}
|
218
|
+
.into(),
|
219
|
+
CompleteWorkflowExecution { result: None }.into(),
|
220
|
+
],
|
221
|
+
),
|
222
|
+
],
|
223
|
+
)
|
224
|
+
.await;
|
225
|
+
}
|
226
|
+
|
227
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
228
|
+
#[tokio::test]
|
229
|
+
async fn scheduled_activity_cancellation_try_cancel(hist_batches: &'static [usize]) {
|
230
|
+
let wfid = "fake_wf_id";
|
231
|
+
let activity_seq = 1;
|
232
|
+
let activity_id = "fake_activity";
|
233
|
+
let signal_id = "signal";
|
234
|
+
|
235
|
+
let t = canned_histories::cancel_scheduled_activity(activity_id, signal_id);
|
236
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
237
|
+
|
238
|
+
poll_and_reply(
|
239
|
+
&core,
|
240
|
+
NonSticky,
|
241
|
+
&[
|
242
|
+
gen_assert_and_reply(
|
243
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
244
|
+
vec![ScheduleActivity {
|
245
|
+
seq: activity_seq,
|
246
|
+
activity_id: activity_id.to_string(),
|
247
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
248
|
+
..Default::default()
|
249
|
+
}
|
250
|
+
.into()],
|
251
|
+
),
|
252
|
+
gen_assert_and_reply(
|
253
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
254
|
+
vec![RequestCancelActivity { seq: activity_seq }.into()],
|
255
|
+
),
|
256
|
+
// Activity is getting resolved right away as we are in the TryCancel mode.
|
257
|
+
gen_assert_and_reply(
|
258
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(_)),
|
259
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
260
|
+
),
|
261
|
+
],
|
262
|
+
)
|
263
|
+
.await;
|
264
|
+
}
|
265
|
+
|
266
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
267
|
+
#[tokio::test]
|
268
|
+
async fn scheduled_activity_timeout(hist_batches: &'static [usize]) {
|
269
|
+
let wfid = "fake_wf_id";
|
270
|
+
let activity_seq = 1;
|
271
|
+
let activity_id = "fake_activity";
|
272
|
+
|
273
|
+
let t = canned_histories::scheduled_activity_timeout(activity_id);
|
274
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
275
|
+
poll_and_reply(
|
276
|
+
&core,
|
277
|
+
NonSticky,
|
278
|
+
&[
|
279
|
+
gen_assert_and_reply(
|
280
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
281
|
+
vec![ScheduleActivity {
|
282
|
+
seq: activity_seq,
|
283
|
+
activity_id: activity_id.to_string(),
|
284
|
+
..Default::default()
|
285
|
+
}
|
286
|
+
.into()],
|
287
|
+
),
|
288
|
+
// Activity is getting resolved right away as it has been timed out.
|
289
|
+
gen_assert_and_reply(
|
290
|
+
&|res| {
|
291
|
+
assert_matches!(
|
292
|
+
res.jobs.as_slice(),
|
293
|
+
[
|
294
|
+
WorkflowActivationJob {
|
295
|
+
variant: Some(workflow_activation_job::Variant::ResolveActivity(
|
296
|
+
ResolveActivity {
|
297
|
+
seq,
|
298
|
+
result: Some(ActivityResolution {
|
299
|
+
status: Some(activity_resolution::Status::Failed(ar::Failure {
|
300
|
+
failure: Some(failure)
|
301
|
+
})),
|
302
|
+
})
|
303
|
+
}
|
304
|
+
)),
|
305
|
+
}
|
306
|
+
] => {
|
307
|
+
assert_eq!(failure.message, "Activity task timed out".to_string());
|
308
|
+
assert_eq!(*seq, activity_seq);
|
309
|
+
}
|
310
|
+
);
|
311
|
+
},
|
312
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
313
|
+
),
|
314
|
+
],
|
315
|
+
)
|
316
|
+
.await;
|
317
|
+
}
|
318
|
+
|
319
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
320
|
+
#[tokio::test]
|
321
|
+
async fn started_activity_timeout(hist_batches: &'static [usize]) {
|
322
|
+
let wfid = "fake_wf_id";
|
323
|
+
let activity_seq = 1;
|
324
|
+
|
325
|
+
let t = canned_histories::started_activity_timeout(activity_seq.to_string().as_str());
|
326
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
327
|
+
|
328
|
+
poll_and_reply(
|
329
|
+
&core,
|
330
|
+
NonSticky,
|
331
|
+
&[
|
332
|
+
gen_assert_and_reply(
|
333
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
334
|
+
vec![ScheduleActivity {
|
335
|
+
seq: activity_seq,
|
336
|
+
activity_id: activity_seq.to_string(),
|
337
|
+
..Default::default()
|
338
|
+
}
|
339
|
+
.into()],
|
340
|
+
),
|
341
|
+
// Activity is getting resolved right away as it has been timed out.
|
342
|
+
gen_assert_and_reply(
|
343
|
+
&|res| {
|
344
|
+
assert_matches!(
|
345
|
+
res.jobs.as_slice(),
|
346
|
+
[
|
347
|
+
WorkflowActivationJob {
|
348
|
+
variant: Some(workflow_activation_job::Variant::ResolveActivity(
|
349
|
+
ResolveActivity {
|
350
|
+
seq,
|
351
|
+
result: Some(ActivityResolution {
|
352
|
+
status: Some(activity_resolution::Status::Failed(ar::Failure {
|
353
|
+
failure: Some(failure)
|
354
|
+
})),
|
355
|
+
})
|
356
|
+
}
|
357
|
+
)),
|
358
|
+
}
|
359
|
+
] => {
|
360
|
+
assert_eq!(failure.message, "Activity task timed out".to_string());
|
361
|
+
assert_eq!(*seq, activity_seq);
|
362
|
+
}
|
363
|
+
);
|
364
|
+
},
|
365
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
366
|
+
),
|
367
|
+
],
|
368
|
+
)
|
369
|
+
.await;
|
370
|
+
}
|
371
|
+
|
372
|
+
#[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
|
373
|
+
#[tokio::test]
|
374
|
+
async fn cancelled_activity_timeout(hist_batches: &'static [usize]) {
|
375
|
+
let wfid = "fake_wf_id";
|
376
|
+
let activity_seq = 0;
|
377
|
+
let activity_id = "fake_activity";
|
378
|
+
let signal_id = "signal";
|
379
|
+
|
380
|
+
let t = canned_histories::scheduled_cancelled_activity_timeout(activity_id, signal_id);
|
381
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
382
|
+
|
383
|
+
poll_and_reply(
|
384
|
+
&core,
|
385
|
+
NonSticky,
|
386
|
+
&[
|
387
|
+
gen_assert_and_reply(
|
388
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
389
|
+
vec![ScheduleActivity {
|
390
|
+
seq: activity_seq,
|
391
|
+
activity_id: activity_id.to_string(),
|
392
|
+
..Default::default()
|
393
|
+
}
|
394
|
+
.into()],
|
395
|
+
),
|
396
|
+
gen_assert_and_reply(
|
397
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
398
|
+
vec![RequestCancelActivity { seq: activity_seq }.into()],
|
399
|
+
),
|
400
|
+
// Activity is resolved right away as it has timed out.
|
401
|
+
gen_assert_and_reply(
|
402
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
403
|
+
ResolveActivity {
|
404
|
+
seq: _,
|
405
|
+
result: Some(ActivityResolution {
|
406
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
407
|
+
})
|
408
|
+
}
|
409
|
+
)),
|
410
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
411
|
+
),
|
412
|
+
],
|
413
|
+
)
|
414
|
+
.await;
|
415
|
+
}
|
416
|
+
|
417
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
418
|
+
#[tokio::test]
|
419
|
+
async fn scheduled_activity_cancellation_abandon(hist_batches: &'static [usize]) {
|
420
|
+
let wfid = "fake_wf_id";
|
421
|
+
let activity_id = 1;
|
422
|
+
let signal_id = "signal";
|
423
|
+
|
424
|
+
let t = canned_histories::cancel_scheduled_activity_abandon(
|
425
|
+
activity_id.to_string().as_str(),
|
426
|
+
signal_id,
|
427
|
+
);
|
428
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
429
|
+
|
430
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::Abandon).await;
|
431
|
+
}
|
432
|
+
|
433
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
434
|
+
#[tokio::test]
|
435
|
+
async fn started_activity_cancellation_abandon(hist_batches: &'static [usize]) {
|
436
|
+
let wfid = "fake_wf_id";
|
437
|
+
let activity_id = 1;
|
438
|
+
let signal_id = "signal";
|
439
|
+
|
440
|
+
let t = canned_histories::cancel_started_activity_abandon(
|
441
|
+
activity_id.to_string().as_str(),
|
442
|
+
signal_id,
|
443
|
+
);
|
444
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
445
|
+
|
446
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::Abandon).await;
|
447
|
+
}
|
448
|
+
|
449
|
+
#[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
|
450
|
+
#[tokio::test]
|
451
|
+
async fn abandoned_activities_ignore_start_and_complete(hist_batches: &'static [usize]) {
|
452
|
+
let wfid = "fake_wf_id";
|
453
|
+
let wf_type = DEFAULT_WORKFLOW_TYPE;
|
454
|
+
let activity_id = "1";
|
455
|
+
|
456
|
+
let mut t = TestHistoryBuilder::default();
|
457
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
458
|
+
t.add_full_wf_task();
|
459
|
+
let act_scheduled_event_id = t.add_activity_task_scheduled(activity_id);
|
460
|
+
let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
|
461
|
+
t.add_timer_fired(timer_started_event_id, "1".to_string());
|
462
|
+
t.add_full_wf_task();
|
463
|
+
let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
|
464
|
+
let act_started_event_id = t.add_activity_task_started(act_scheduled_event_id);
|
465
|
+
t.add_activity_task_completed(
|
466
|
+
act_scheduled_event_id,
|
467
|
+
act_started_event_id,
|
468
|
+
Default::default(),
|
469
|
+
);
|
470
|
+
t.add_full_wf_task();
|
471
|
+
t.add_timer_fired(timer_started_event_id, "2".to_string());
|
472
|
+
t.add_full_wf_task();
|
473
|
+
t.add_workflow_execution_completed();
|
474
|
+
let mock = mock_workflow_client();
|
475
|
+
let mut worker = mock_sdk(MockPollCfg::from_resp_batches(wfid, t, hist_batches, mock));
|
476
|
+
|
477
|
+
worker.register_wf(wf_type.to_owned(), |ctx: WfContext| async move {
|
478
|
+
let act_fut = ctx.activity(ActivityOptions {
|
479
|
+
activity_type: "echo_activity".to_string(),
|
480
|
+
start_to_close_timeout: Some(Duration::from_secs(5)),
|
481
|
+
cancellation_type: ActivityCancellationType::Abandon,
|
482
|
+
..Default::default()
|
483
|
+
});
|
484
|
+
ctx.timer(Duration::from_secs(1)).await;
|
485
|
+
act_fut.cancel(&ctx);
|
486
|
+
ctx.timer(Duration::from_secs(3)).await;
|
487
|
+
act_fut.await;
|
488
|
+
Ok(().into())
|
489
|
+
});
|
490
|
+
worker
|
491
|
+
.submit_wf(wfid, wf_type, vec![], Default::default())
|
492
|
+
.await
|
493
|
+
.unwrap();
|
494
|
+
worker.run_until_done().await.unwrap();
|
495
|
+
}
|
496
|
+
|
497
|
+
#[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
|
498
|
+
#[tokio::test]
|
499
|
+
async fn scheduled_activity_cancellation_try_cancel_task_canceled(hist_batches: &'static [usize]) {
|
500
|
+
let wfid = "fake_wf_id";
|
501
|
+
let activity_id = 1;
|
502
|
+
let signal_id = "signal";
|
503
|
+
|
504
|
+
let t = canned_histories::cancel_scheduled_activity_with_activity_task_cancel(
|
505
|
+
activity_id.to_string().as_str(),
|
506
|
+
signal_id,
|
507
|
+
);
|
508
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
509
|
+
|
510
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::TryCancel).await;
|
511
|
+
}
|
512
|
+
|
513
|
+
#[rstest(hist_batches, case::incremental(&[1, 3]), case::replay(&[3]))]
|
514
|
+
#[tokio::test]
|
515
|
+
async fn started_activity_cancellation_try_cancel_task_canceled(hist_batches: &'static [usize]) {
|
516
|
+
let wfid = "fake_wf_id";
|
517
|
+
let activity_id = 1;
|
518
|
+
let signal_id = "signal";
|
519
|
+
|
520
|
+
let t = canned_histories::cancel_started_activity_with_activity_task_cancel(
|
521
|
+
activity_id.to_string().as_str(),
|
522
|
+
signal_id,
|
523
|
+
);
|
524
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
525
|
+
|
526
|
+
verify_activity_cancellation(&core, activity_id, ActivityCancellationType::TryCancel).await;
|
527
|
+
}
|
528
|
+
|
529
|
+
/// Verification for try cancel & abandon histories
|
530
|
+
async fn verify_activity_cancellation(
|
531
|
+
worker: &Worker,
|
532
|
+
activity_seq: u32,
|
533
|
+
cancel_type: ActivityCancellationType,
|
534
|
+
) {
|
535
|
+
poll_and_reply(
|
536
|
+
worker,
|
537
|
+
NonSticky,
|
538
|
+
&[
|
539
|
+
gen_assert_and_reply(
|
540
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
541
|
+
vec![ScheduleActivity {
|
542
|
+
seq: activity_seq,
|
543
|
+
activity_id: activity_seq.to_string(),
|
544
|
+
cancellation_type: cancel_type as i32,
|
545
|
+
..Default::default()
|
546
|
+
}
|
547
|
+
.into()],
|
548
|
+
),
|
549
|
+
gen_assert_and_reply(
|
550
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
551
|
+
vec![RequestCancelActivity { seq: activity_seq }.into()],
|
552
|
+
),
|
553
|
+
// Activity should be resolved right away
|
554
|
+
gen_assert_and_reply(
|
555
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
556
|
+
ResolveActivity {
|
557
|
+
seq: _,
|
558
|
+
result: Some(ActivityResolution {
|
559
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
560
|
+
})
|
561
|
+
}
|
562
|
+
)),
|
563
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
564
|
+
),
|
565
|
+
],
|
566
|
+
)
|
567
|
+
.await;
|
568
|
+
}
|
569
|
+
|
570
|
+
#[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
|
571
|
+
#[tokio::test]
|
572
|
+
async fn scheduled_activity_cancellation_wait_for_cancellation(hist_batches: &'static [usize]) {
|
573
|
+
let wfid = "fake_wf_id";
|
574
|
+
let activity_id = 1;
|
575
|
+
let signal_id = "signal";
|
576
|
+
|
577
|
+
let t = canned_histories::cancel_scheduled_activity_with_signal_and_activity_task_cancel(
|
578
|
+
activity_id.to_string().as_str(),
|
579
|
+
signal_id,
|
580
|
+
);
|
581
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
582
|
+
|
583
|
+
verify_activity_cancellation_wait_for_cancellation(activity_id, &core).await;
|
584
|
+
}
|
585
|
+
|
586
|
+
#[rstest(hist_batches, case::incremental(&[1, 2, 3, 4]), case::replay(&[4]))]
|
587
|
+
#[tokio::test]
|
588
|
+
async fn started_activity_cancellation_wait_for_cancellation(hist_batches: &'static [usize]) {
|
589
|
+
let wfid = "fake_wf_id";
|
590
|
+
let activity_id = 1;
|
591
|
+
let signal_id = "signal";
|
592
|
+
|
593
|
+
let t = canned_histories::cancel_started_activity_with_signal_and_activity_task_cancel(
|
594
|
+
activity_id.to_string().as_str(),
|
595
|
+
signal_id,
|
596
|
+
);
|
597
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
598
|
+
|
599
|
+
verify_activity_cancellation_wait_for_cancellation(activity_id, &core).await;
|
600
|
+
}
|
601
|
+
|
602
|
+
async fn verify_activity_cancellation_wait_for_cancellation(activity_id: u32, worker: &Worker) {
|
603
|
+
poll_and_reply(
|
604
|
+
worker,
|
605
|
+
NonSticky,
|
606
|
+
&[
|
607
|
+
gen_assert_and_reply(
|
608
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
609
|
+
vec![ScheduleActivity {
|
610
|
+
seq: activity_id,
|
611
|
+
activity_id: activity_id.to_string(),
|
612
|
+
cancellation_type: ActivityCancellationType::WaitCancellationCompleted as i32,
|
613
|
+
..Default::default()
|
614
|
+
}
|
615
|
+
.into()],
|
616
|
+
),
|
617
|
+
gen_assert_and_reply(
|
618
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
619
|
+
vec![RequestCancelActivity { seq: activity_id }.into()],
|
620
|
+
),
|
621
|
+
// Making sure that activity is not resolved until it's cancelled.
|
622
|
+
gen_assert_and_reply(
|
623
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
624
|
+
vec![],
|
625
|
+
),
|
626
|
+
// Now ActivityTaskCanceled has been processed and activity can be resolved.
|
627
|
+
gen_assert_and_reply(
|
628
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
629
|
+
ResolveActivity {
|
630
|
+
seq: _,
|
631
|
+
result: Some(ActivityResolution {
|
632
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
633
|
+
})
|
634
|
+
}
|
635
|
+
)),
|
636
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
637
|
+
),
|
638
|
+
],
|
639
|
+
)
|
640
|
+
.await;
|
641
|
+
}
|
642
|
+
|
643
|
+
#[tokio::test]
|
644
|
+
async fn workflow_update_random_seed_on_workflow_reset() {
|
645
|
+
let wfid = "fake_wf_id";
|
646
|
+
let new_run_id = "86E39A5F-AE31-4626-BDFE-398EE072D156";
|
647
|
+
let timer_1_id = 1;
|
648
|
+
let randomness_seed_from_start = AtomicU64::new(0);
|
649
|
+
|
650
|
+
let t = canned_histories::workflow_fails_with_reset_after_timer(
|
651
|
+
timer_1_id.to_string().as_str(),
|
652
|
+
new_run_id,
|
653
|
+
);
|
654
|
+
let core = build_fake_worker(wfid, t, [2]);
|
655
|
+
|
656
|
+
poll_and_reply(
|
657
|
+
&core,
|
658
|
+
NonSticky,
|
659
|
+
&[
|
660
|
+
gen_assert_and_reply(
|
661
|
+
&|res| {
|
662
|
+
assert_matches!(
|
663
|
+
res.jobs.as_slice(),
|
664
|
+
[WorkflowActivationJob {
|
665
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(
|
666
|
+
StartWorkflow{randomness_seed, ..}
|
667
|
+
)),
|
668
|
+
}] => {
|
669
|
+
randomness_seed_from_start.store(*randomness_seed, Ordering::SeqCst);
|
670
|
+
}
|
671
|
+
);
|
672
|
+
},
|
673
|
+
vec![start_timer_cmd(timer_1_id, Duration::from_secs(1))],
|
674
|
+
),
|
675
|
+
gen_assert_and_reply(
|
676
|
+
&|res| {
|
677
|
+
assert_matches!(
|
678
|
+
res.jobs.as_slice(),
|
679
|
+
[WorkflowActivationJob {
|
680
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_),),
|
681
|
+
},
|
682
|
+
WorkflowActivationJob {
|
683
|
+
variant: Some(workflow_activation_job::Variant::UpdateRandomSeed(
|
684
|
+
UpdateRandomSeed{randomness_seed})),
|
685
|
+
}] => {
|
686
|
+
assert_ne!(randomness_seed_from_start.load(Ordering::SeqCst),
|
687
|
+
*randomness_seed);
|
688
|
+
}
|
689
|
+
);
|
690
|
+
},
|
691
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
692
|
+
),
|
693
|
+
],
|
694
|
+
)
|
695
|
+
.await;
|
696
|
+
}
|
697
|
+
|
698
|
+
#[tokio::test]
|
699
|
+
async fn cancel_timer_before_sent_wf_bridge() {
|
700
|
+
let wfid = "fake_wf_id";
|
701
|
+
let cancel_timer_id = 1;
|
702
|
+
|
703
|
+
let mut t = TestHistoryBuilder::default();
|
704
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
705
|
+
t.add_full_wf_task();
|
706
|
+
t.add_workflow_execution_completed();
|
707
|
+
|
708
|
+
let core = build_fake_worker(wfid, t, [1]);
|
709
|
+
|
710
|
+
poll_and_reply(
|
711
|
+
&core,
|
712
|
+
NonSticky,
|
713
|
+
&[gen_assert_and_reply(
|
714
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
715
|
+
vec![
|
716
|
+
start_timer_cmd(cancel_timer_id, Duration::from_secs(1)),
|
717
|
+
CancelTimer {
|
718
|
+
seq: cancel_timer_id,
|
719
|
+
}
|
720
|
+
.into(),
|
721
|
+
CompleteWorkflowExecution { result: None }.into(),
|
722
|
+
],
|
723
|
+
)],
|
724
|
+
)
|
725
|
+
.await;
|
726
|
+
}
|
727
|
+
|
728
|
+
#[rstest]
|
729
|
+
#[case::no_evict_inc(&[1, 2, 2], NonSticky)]
|
730
|
+
#[case::no_evict(&[2, 2], NonSticky)]
|
731
|
+
#[tokio::test]
|
732
|
+
async fn complete_activation_with_failure(
|
733
|
+
#[case] batches: &'static [usize],
|
734
|
+
#[case] evict: WorkflowCachingPolicy,
|
735
|
+
) {
|
736
|
+
let wfid = "fake_wf_id";
|
737
|
+
let timer_id = 1;
|
738
|
+
|
739
|
+
let hist =
|
740
|
+
canned_histories::workflow_fails_with_failure_after_timer(timer_id.to_string().as_str());
|
741
|
+
let mock_sg = build_multihist_mock_sg(
|
742
|
+
vec![FakeWfResponses {
|
743
|
+
wf_id: wfid.to_string(),
|
744
|
+
hist,
|
745
|
+
response_batches: batches.iter().map(Into::into).collect(),
|
746
|
+
}],
|
747
|
+
true,
|
748
|
+
1,
|
749
|
+
);
|
750
|
+
let core = mock_worker(mock_sg);
|
751
|
+
|
752
|
+
poll_and_reply(
|
753
|
+
&core,
|
754
|
+
evict,
|
755
|
+
&[
|
756
|
+
gen_assert_and_reply(
|
757
|
+
&|_| {},
|
758
|
+
vec![start_timer_cmd(timer_id, Duration::from_secs(1))],
|
759
|
+
),
|
760
|
+
gen_assert_and_fail(&|_| {}),
|
761
|
+
gen_assert_and_reply(
|
762
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
763
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
764
|
+
),
|
765
|
+
],
|
766
|
+
)
|
767
|
+
.await;
|
768
|
+
core.shutdown().await;
|
769
|
+
}
|
770
|
+
|
771
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
772
|
+
#[tokio::test]
|
773
|
+
async fn simple_timer_fail_wf_execution(hist_batches: &'static [usize]) {
|
774
|
+
let wfid = "fake_wf_id";
|
775
|
+
let timer_id = 1;
|
776
|
+
|
777
|
+
let t = canned_histories::single_timer(timer_id.to_string().as_str());
|
778
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
779
|
+
|
780
|
+
poll_and_reply(
|
781
|
+
&core,
|
782
|
+
NonSticky,
|
783
|
+
&[
|
784
|
+
gen_assert_and_reply(
|
785
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
786
|
+
vec![start_timer_cmd(timer_id, Duration::from_secs(1))],
|
787
|
+
),
|
788
|
+
gen_assert_and_reply(
|
789
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
790
|
+
vec![FailWorkflowExecution {
|
791
|
+
failure: Some(Failure {
|
792
|
+
message: "I'm ded".to_string(),
|
793
|
+
..Default::default()
|
794
|
+
}),
|
795
|
+
}
|
796
|
+
.into()],
|
797
|
+
),
|
798
|
+
],
|
799
|
+
)
|
800
|
+
.await;
|
801
|
+
}
|
802
|
+
|
803
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
804
|
+
#[tokio::test]
|
805
|
+
async fn two_signals(hist_batches: &'static [usize]) {
|
806
|
+
let wfid = "fake_wf_id";
|
807
|
+
|
808
|
+
let t = canned_histories::two_signals("sig1", "sig2");
|
809
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
810
|
+
|
811
|
+
poll_and_reply(
|
812
|
+
&core,
|
813
|
+
NonSticky,
|
814
|
+
&[
|
815
|
+
gen_assert_and_reply(
|
816
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
817
|
+
// Task is completed with no commands
|
818
|
+
vec![],
|
819
|
+
),
|
820
|
+
gen_assert_and_reply(
|
821
|
+
&job_assert!(
|
822
|
+
workflow_activation_job::Variant::SignalWorkflow(_),
|
823
|
+
workflow_activation_job::Variant::SignalWorkflow(_)
|
824
|
+
),
|
825
|
+
vec![],
|
826
|
+
),
|
827
|
+
],
|
828
|
+
)
|
829
|
+
.await;
|
830
|
+
}
|
831
|
+
|
832
|
+
#[tokio::test]
|
833
|
+
async fn workflow_failures_only_reported_once() {
|
834
|
+
let wfid = "fake_wf_id";
|
835
|
+
let timer_1 = 1;
|
836
|
+
let timer_2 = 2;
|
837
|
+
|
838
|
+
let hist = canned_histories::workflow_fails_with_failure_two_different_points(
|
839
|
+
timer_1.to_string().as_str(),
|
840
|
+
timer_2.to_string().as_str(),
|
841
|
+
);
|
842
|
+
let response_batches = vec![
|
843
|
+
1, 2, // Start then first good reply
|
844
|
+
2, 2, 2, // Poll for every failure
|
845
|
+
// Poll again after evicting after second good reply, then two more fails
|
846
|
+
3, 3, 3,
|
847
|
+
];
|
848
|
+
let mocks = build_multihist_mock_sg(
|
849
|
+
vec![FakeWfResponses {
|
850
|
+
wf_id: wfid.to_string(),
|
851
|
+
hist,
|
852
|
+
response_batches: response_batches.into_iter().map(Into::into).collect(),
|
853
|
+
}],
|
854
|
+
true,
|
855
|
+
// We should only call the server to say we failed twice (once after each success)
|
856
|
+
2,
|
857
|
+
);
|
858
|
+
let omap = mocks.outstanding_task_map.clone();
|
859
|
+
let core = mock_worker(mocks);
|
860
|
+
|
861
|
+
poll_and_reply_clears_outstanding_evicts(
|
862
|
+
&core,
|
863
|
+
omap,
|
864
|
+
NonSticky,
|
865
|
+
&[
|
866
|
+
gen_assert_and_reply(
|
867
|
+
&|_| {},
|
868
|
+
vec![start_timer_cmd(timer_1, Duration::from_secs(1))],
|
869
|
+
),
|
870
|
+
// Fail a few times in a row (only one of which should be reported)
|
871
|
+
gen_assert_and_fail(&|_| {}),
|
872
|
+
gen_assert_and_fail(&|_| {}),
|
873
|
+
gen_assert_and_fail(&|_| {}),
|
874
|
+
gen_assert_and_reply(
|
875
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
876
|
+
vec![start_timer_cmd(timer_2, Duration::from_secs(1))],
|
877
|
+
),
|
878
|
+
// Again (a new fail should be reported here)
|
879
|
+
gen_assert_and_fail(&|_| {}),
|
880
|
+
gen_assert_and_fail(&|_| {}),
|
881
|
+
gen_assert_and_reply(
|
882
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
883
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
884
|
+
),
|
885
|
+
],
|
886
|
+
)
|
887
|
+
.await;
|
888
|
+
}
|
889
|
+
|
890
|
+
#[tokio::test]
|
891
|
+
async fn max_wft_respected() {
|
892
|
+
let total_wfs = 100;
|
893
|
+
let wf_ids: Vec<_> = (0..total_wfs)
|
894
|
+
.into_iter()
|
895
|
+
.map(|i| format!("fake-wf-{}", i))
|
896
|
+
.collect();
|
897
|
+
let hists = wf_ids.iter().map(|wf_id| {
|
898
|
+
let hist = canned_histories::single_timer("1");
|
899
|
+
FakeWfResponses {
|
900
|
+
wf_id: wf_id.to_string(),
|
901
|
+
hist,
|
902
|
+
response_batches: vec![1.into(), 2.into()],
|
903
|
+
}
|
904
|
+
});
|
905
|
+
let mh = MockPollCfg::new(hists.into_iter().collect(), true, 0);
|
906
|
+
let mut worker = mock_sdk_cfg(mh, |cfg| {
|
907
|
+
cfg.max_cached_workflows = total_wfs as usize;
|
908
|
+
cfg.max_outstanding_workflow_tasks = 1;
|
909
|
+
});
|
910
|
+
let active_count: &'static _ = Box::leak(Box::new(Semaphore::new(1)));
|
911
|
+
worker.register_wf(DEFAULT_WORKFLOW_TYPE, move |ctx: WfContext| async move {
|
912
|
+
drop(
|
913
|
+
active_count
|
914
|
+
.try_acquire()
|
915
|
+
.expect("No multiple concurrent workflow tasks!"),
|
916
|
+
);
|
917
|
+
ctx.timer(Duration::from_secs(1)).await;
|
918
|
+
Ok(().into())
|
919
|
+
});
|
920
|
+
|
921
|
+
for wf_id in wf_ids {
|
922
|
+
worker
|
923
|
+
.submit_wf(wf_id, DEFAULT_WORKFLOW_TYPE, vec![], Default::default())
|
924
|
+
.await
|
925
|
+
.unwrap();
|
926
|
+
}
|
927
|
+
worker.run_until_done().await.unwrap();
|
928
|
+
}
|
929
|
+
|
930
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[3]))]
|
931
|
+
#[tokio::test]
|
932
|
+
async fn activity_not_canceled_on_replay_repro(hist_batches: &'static [usize]) {
|
933
|
+
let wfid = "fake_wf_id";
|
934
|
+
let t = canned_histories::unsent_at_cancel_repro();
|
935
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
936
|
+
let activity_id = 1;
|
937
|
+
|
938
|
+
poll_and_reply(
|
939
|
+
&core,
|
940
|
+
NonSticky,
|
941
|
+
&[
|
942
|
+
gen_assert_and_reply(
|
943
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
944
|
+
// Start timer and activity
|
945
|
+
vec![
|
946
|
+
ScheduleActivity {
|
947
|
+
seq: activity_id,
|
948
|
+
activity_id: activity_id.to_string(),
|
949
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
950
|
+
..Default::default()
|
951
|
+
}
|
952
|
+
.into(),
|
953
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
954
|
+
],
|
955
|
+
),
|
956
|
+
gen_assert_and_reply(
|
957
|
+
&job_assert!(workflow_activation_job::Variant::FireTimer(_)),
|
958
|
+
vec![RequestCancelActivity { seq: activity_id }.into()],
|
959
|
+
),
|
960
|
+
gen_assert_and_reply(
|
961
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
962
|
+
ResolveActivity {
|
963
|
+
result: Some(ActivityResolution {
|
964
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
965
|
+
}),
|
966
|
+
..
|
967
|
+
}
|
968
|
+
)),
|
969
|
+
vec![start_timer_cmd(2, Duration::from_secs(1))],
|
970
|
+
),
|
971
|
+
],
|
972
|
+
)
|
973
|
+
.await;
|
974
|
+
}
|
975
|
+
|
976
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[3]))]
|
977
|
+
#[tokio::test]
|
978
|
+
async fn activity_not_canceled_when_also_completed_repro(hist_batches: &'static [usize]) {
|
979
|
+
let wfid = "fake_wf_id";
|
980
|
+
let t = canned_histories::cancel_not_sent_when_also_complete_repro();
|
981
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
982
|
+
let activity_id = 1;
|
983
|
+
|
984
|
+
poll_and_reply(
|
985
|
+
&core,
|
986
|
+
NonSticky,
|
987
|
+
&[
|
988
|
+
gen_assert_and_reply(
|
989
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
990
|
+
vec![ScheduleActivity {
|
991
|
+
seq: activity_id,
|
992
|
+
activity_id: activity_id.to_string(),
|
993
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
994
|
+
..Default::default()
|
995
|
+
}
|
996
|
+
.into()],
|
997
|
+
),
|
998
|
+
gen_assert_and_reply(
|
999
|
+
&job_assert!(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1000
|
+
vec![
|
1001
|
+
RequestCancelActivity { seq: activity_id }.into(),
|
1002
|
+
start_timer_cmd(2, Duration::from_secs(1)),
|
1003
|
+
],
|
1004
|
+
),
|
1005
|
+
gen_assert_and_reply(
|
1006
|
+
&job_assert!(workflow_activation_job::Variant::ResolveActivity(
|
1007
|
+
ResolveActivity {
|
1008
|
+
result: Some(ActivityResolution {
|
1009
|
+
status: Some(activity_resolution::Status::Cancelled(..)),
|
1010
|
+
}),
|
1011
|
+
..
|
1012
|
+
}
|
1013
|
+
)),
|
1014
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1015
|
+
),
|
1016
|
+
],
|
1017
|
+
)
|
1018
|
+
.await;
|
1019
|
+
}
|
1020
|
+
|
1021
|
+
#[tokio::test]
|
1022
|
+
async fn lots_of_workflows() {
|
1023
|
+
let total_wfs = 500;
|
1024
|
+
let hists = (0..total_wfs).into_iter().map(|i| {
|
1025
|
+
let wf_id = format!("fake-wf-{}", i);
|
1026
|
+
let hist = canned_histories::single_timer("1");
|
1027
|
+
FakeWfResponses {
|
1028
|
+
wf_id,
|
1029
|
+
hist,
|
1030
|
+
response_batches: vec![1.into(), 2.into()],
|
1031
|
+
}
|
1032
|
+
});
|
1033
|
+
let mut mock = build_multihist_mock_sg(hists, false, 0);
|
1034
|
+
mock.make_wft_stream_interminable();
|
1035
|
+
let worker = &mock_worker(mock);
|
1036
|
+
let completed_count = Arc::new(Semaphore::new(0));
|
1037
|
+
let killer = async {
|
1038
|
+
let _ = completed_count.acquire_many(total_wfs).await.unwrap();
|
1039
|
+
worker.initiate_shutdown();
|
1040
|
+
};
|
1041
|
+
let poller = fanout_tasks(5, |_| {
|
1042
|
+
let completed_count = completed_count.clone();
|
1043
|
+
async move {
|
1044
|
+
while let Ok(wft) = worker.poll_workflow_activation().await {
|
1045
|
+
let job = &wft.jobs[0];
|
1046
|
+
let reply = match job.variant {
|
1047
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_)) => {
|
1048
|
+
start_timer_cmd(1, Duration::from_secs(1))
|
1049
|
+
}
|
1050
|
+
Some(workflow_activation_job::Variant::RemoveFromCache(_)) => {
|
1051
|
+
worker
|
1052
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(
|
1053
|
+
wft.run_id,
|
1054
|
+
))
|
1055
|
+
.await
|
1056
|
+
.unwrap();
|
1057
|
+
continue;
|
1058
|
+
}
|
1059
|
+
_ => {
|
1060
|
+
completed_count.add_permits(1);
|
1061
|
+
CompleteWorkflowExecution { result: None }.into()
|
1062
|
+
}
|
1063
|
+
};
|
1064
|
+
worker
|
1065
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1066
|
+
wft.run_id, reply,
|
1067
|
+
))
|
1068
|
+
.await
|
1069
|
+
.unwrap();
|
1070
|
+
}
|
1071
|
+
}
|
1072
|
+
});
|
1073
|
+
join!(killer, poller);
|
1074
|
+
worker.shutdown().await;
|
1075
|
+
}
|
1076
|
+
|
1077
|
+
#[rstest(hist_batches, case::incremental(&[1, 2]), case::replay(&[2]))]
|
1078
|
+
#[tokio::test]
|
1079
|
+
async fn wft_timeout_repro(hist_batches: &'static [usize]) {
|
1080
|
+
let wfid = "fake_wf_id";
|
1081
|
+
let t = canned_histories::wft_timeout_repro();
|
1082
|
+
let core = build_fake_worker(wfid, t, hist_batches);
|
1083
|
+
let activity_id = 1;
|
1084
|
+
|
1085
|
+
poll_and_reply(
|
1086
|
+
&core,
|
1087
|
+
NonSticky,
|
1088
|
+
&[
|
1089
|
+
gen_assert_and_reply(
|
1090
|
+
&job_assert!(workflow_activation_job::Variant::StartWorkflow(_)),
|
1091
|
+
vec![ScheduleActivity {
|
1092
|
+
seq: activity_id,
|
1093
|
+
activity_id: activity_id.to_string(),
|
1094
|
+
cancellation_type: ActivityCancellationType::TryCancel as i32,
|
1095
|
+
..Default::default()
|
1096
|
+
}
|
1097
|
+
.into()],
|
1098
|
+
),
|
1099
|
+
gen_assert_and_reply(
|
1100
|
+
&job_assert!(
|
1101
|
+
workflow_activation_job::Variant::SignalWorkflow(_),
|
1102
|
+
workflow_activation_job::Variant::SignalWorkflow(_),
|
1103
|
+
workflow_activation_job::Variant::ResolveActivity(ResolveActivity {
|
1104
|
+
result: Some(ActivityResolution {
|
1105
|
+
status: Some(activity_resolution::Status::Completed(..)),
|
1106
|
+
}),
|
1107
|
+
..
|
1108
|
+
})
|
1109
|
+
),
|
1110
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1111
|
+
),
|
1112
|
+
],
|
1113
|
+
)
|
1114
|
+
.await;
|
1115
|
+
}
|
1116
|
+
|
1117
|
+
#[tokio::test]
|
1118
|
+
async fn complete_after_eviction() {
|
1119
|
+
let wfid = "fake_wf_id";
|
1120
|
+
let t = canned_histories::single_timer("1");
|
1121
|
+
let mut mock = mock_workflow_client();
|
1122
|
+
mock.expect_complete_workflow_task().times(0);
|
1123
|
+
let mock = single_hist_mock_sg(wfid, t, [2], mock, true);
|
1124
|
+
let core = mock_worker(mock);
|
1125
|
+
|
1126
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1127
|
+
// We just got start workflow, immediately evict
|
1128
|
+
core.request_workflow_eviction(&activation.run_id);
|
1129
|
+
// Since we got whole history, we must finish replay before eviction will appear
|
1130
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1131
|
+
activation.run_id,
|
1132
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1133
|
+
))
|
1134
|
+
.await
|
1135
|
+
.unwrap();
|
1136
|
+
let next_activation = core.poll_workflow_activation().await.unwrap();
|
1137
|
+
assert_matches!(
|
1138
|
+
next_activation.jobs.as_slice(),
|
1139
|
+
[WorkflowActivationJob {
|
1140
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1141
|
+
},]
|
1142
|
+
);
|
1143
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1144
|
+
next_activation.run_id,
|
1145
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1146
|
+
))
|
1147
|
+
.await
|
1148
|
+
.unwrap();
|
1149
|
+
|
1150
|
+
core.shutdown().await;
|
1151
|
+
}
|
1152
|
+
|
1153
|
+
#[tokio::test]
|
1154
|
+
async fn sends_appropriate_sticky_task_queue_responses() {
|
1155
|
+
// This test verifies that when completions are sent with sticky queues enabled, that they
|
1156
|
+
// include the information that tells the server to enqueue the next task on a sticky queue.
|
1157
|
+
let wfid = "fake_wf_id";
|
1158
|
+
let t = canned_histories::single_timer("1");
|
1159
|
+
let mut mock = mock_workflow_client();
|
1160
|
+
mock.expect_complete_workflow_task()
|
1161
|
+
.withf(|comp| comp.sticky_attributes.is_some())
|
1162
|
+
.times(1)
|
1163
|
+
.returning(|_| Ok(Default::default()));
|
1164
|
+
mock.expect_complete_workflow_task().times(0);
|
1165
|
+
let mut mock = single_hist_mock_sg(wfid, t, [1], mock, false);
|
1166
|
+
mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
|
1167
|
+
let core = mock_worker(mock);
|
1168
|
+
|
1169
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1170
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1171
|
+
activation.run_id,
|
1172
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1173
|
+
))
|
1174
|
+
.await
|
1175
|
+
.unwrap();
|
1176
|
+
core.shutdown().await;
|
1177
|
+
}
|
1178
|
+
|
1179
|
+
#[tokio::test]
|
1180
|
+
async fn new_server_work_while_eviction_outstanding_doesnt_overwrite_activation() {
|
1181
|
+
let wfid = "fake_wf_id";
|
1182
|
+
let t = canned_histories::single_timer("1");
|
1183
|
+
let mock = single_hist_mock_sg(wfid, t, [1, 2], mock_workflow_client(), false);
|
1184
|
+
let taskmap = mock.outstanding_task_map.clone().unwrap();
|
1185
|
+
let core = mock_worker(mock);
|
1186
|
+
|
1187
|
+
// Poll for and complete first workflow task
|
1188
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1189
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1190
|
+
activation.run_id,
|
1191
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1192
|
+
))
|
1193
|
+
.await
|
1194
|
+
.unwrap();
|
1195
|
+
let evict_act = core.poll_workflow_activation().await.unwrap();
|
1196
|
+
assert_matches!(
|
1197
|
+
evict_act.jobs.as_slice(),
|
1198
|
+
[WorkflowActivationJob {
|
1199
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1200
|
+
}]
|
1201
|
+
);
|
1202
|
+
// Ensure mock has delivered both tasks
|
1203
|
+
assert!(taskmap.all_work_delivered());
|
1204
|
+
// Now we can complete the evict
|
1205
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_act.run_id))
|
1206
|
+
.await
|
1207
|
+
.unwrap();
|
1208
|
+
// The task buffered during eviction is applied and we start over
|
1209
|
+
let start_again = core.poll_workflow_activation().await.unwrap();
|
1210
|
+
assert_matches!(
|
1211
|
+
start_again.jobs[0].variant,
|
1212
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_))
|
1213
|
+
);
|
1214
|
+
}
|
1215
|
+
|
1216
|
+
#[tokio::test]
|
1217
|
+
async fn buffered_work_drained_on_shutdown() {
|
1218
|
+
let wfid = "fake_wf_id";
|
1219
|
+
// Build a one-timer history where first task times out
|
1220
|
+
let mut t = TestHistoryBuilder::default();
|
1221
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1222
|
+
t.add_workflow_task_scheduled_and_started();
|
1223
|
+
// Need to build the first response before adding the timeout events b/c otherwise the history
|
1224
|
+
// builder will include them in the first task
|
1225
|
+
let resp_1 = hist_to_poll_resp(&t, wfid.to_owned(), 1.into()).resp;
|
1226
|
+
t.add_workflow_task_timed_out();
|
1227
|
+
t.add_full_wf_task();
|
1228
|
+
let timer_started_event_id = t.add_get_event_id(EventType::TimerStarted, None);
|
1229
|
+
t.add(
|
1230
|
+
EventType::TimerFired,
|
1231
|
+
history_event::Attributes::TimerFiredEventAttributes(TimerFiredEventAttributes {
|
1232
|
+
started_event_id: timer_started_event_id,
|
1233
|
+
timer_id: "1".to_string(),
|
1234
|
+
}),
|
1235
|
+
);
|
1236
|
+
t.add_full_wf_task();
|
1237
|
+
t.add_workflow_execution_completed();
|
1238
|
+
|
1239
|
+
let mut tasks = VecDeque::from(vec![resp_1]);
|
1240
|
+
// Extend the task list with the now timeout-included version of the task. We add a bunch of
|
1241
|
+
// them because the poll loop will spin while new tasks are available and it is buffering them
|
1242
|
+
tasks.extend(
|
1243
|
+
std::iter::repeat_with(|| hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp).take(50),
|
1244
|
+
);
|
1245
|
+
let mut mock = mock_workflow_client();
|
1246
|
+
mock.expect_complete_workflow_task()
|
1247
|
+
.returning(|_| Ok(RespondWorkflowTaskCompletedResponse::default()));
|
1248
|
+
let mut mock = MocksHolder::from_wft_stream(mock, stream::iter(tasks));
|
1249
|
+
// Cache on to avoid being super repetitive
|
1250
|
+
mock.worker_cfg(|wc| wc.max_cached_workflows = 10);
|
1251
|
+
let core = &mock_worker(mock);
|
1252
|
+
|
1253
|
+
// Poll for first WFT
|
1254
|
+
let act1 = core.poll_workflow_activation().await.unwrap();
|
1255
|
+
let poll_fut = async move {
|
1256
|
+
// Now poll again, which will start spinning, and buffer the next WFT with timer fired in it
|
1257
|
+
// - it won't stop spinning until the first task is complete
|
1258
|
+
let t = core.poll_workflow_activation().await.unwrap();
|
1259
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1260
|
+
t.run_id,
|
1261
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1262
|
+
))
|
1263
|
+
.await
|
1264
|
+
.unwrap();
|
1265
|
+
};
|
1266
|
+
let complete_first = async move {
|
1267
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1268
|
+
act1.run_id,
|
1269
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1270
|
+
))
|
1271
|
+
.await
|
1272
|
+
.unwrap();
|
1273
|
+
};
|
1274
|
+
join!(poll_fut, complete_first, async {
|
1275
|
+
// If the shutdown is sent too too fast, we might not have got a chance to even buffer work
|
1276
|
+
tokio::time::sleep(Duration::from_millis(5)).await;
|
1277
|
+
core.shutdown().await;
|
1278
|
+
});
|
1279
|
+
}
|
1280
|
+
|
1281
|
+
#[tokio::test]
|
1282
|
+
async fn fail_wft_then_recover() {
|
1283
|
+
let t = canned_histories::long_sequential_timers(1);
|
1284
|
+
let mut mh = MockPollCfg::from_resp_batches(
|
1285
|
+
"fake_wf_id",
|
1286
|
+
t,
|
1287
|
+
// We need to deliver all of history twice because of eviction
|
1288
|
+
[ResponseType::AllHistory, ResponseType::AllHistory],
|
1289
|
+
mock_workflow_client(),
|
1290
|
+
);
|
1291
|
+
mh.num_expected_fails = 1;
|
1292
|
+
mh.expect_fail_wft_matcher =
|
1293
|
+
Box::new(|_, cause, _| matches!(cause, WorkflowTaskFailedCause::NonDeterministicError));
|
1294
|
+
let mut mock = build_mock_pollers(mh);
|
1295
|
+
mock.worker_cfg(|wc| {
|
1296
|
+
wc.max_cached_workflows = 2;
|
1297
|
+
});
|
1298
|
+
let core = mock_worker(mock);
|
1299
|
+
|
1300
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1301
|
+
// Start an activity instead of a timer, triggering nondeterminism error
|
1302
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1303
|
+
act.run_id.clone(),
|
1304
|
+
vec![ScheduleActivity {
|
1305
|
+
activity_id: "fake_activity".to_string(),
|
1306
|
+
..Default::default()
|
1307
|
+
}
|
1308
|
+
.into()],
|
1309
|
+
))
|
1310
|
+
.await
|
1311
|
+
.unwrap();
|
1312
|
+
// We must handle an eviction now
|
1313
|
+
let evict_act = core.poll_workflow_activation().await.unwrap();
|
1314
|
+
assert_eq!(evict_act.run_id, act.run_id);
|
1315
|
+
assert_matches!(
|
1316
|
+
evict_act.jobs.as_slice(),
|
1317
|
+
[WorkflowActivationJob {
|
1318
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1319
|
+
}]
|
1320
|
+
);
|
1321
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(evict_act.run_id))
|
1322
|
+
.await
|
1323
|
+
.unwrap();
|
1324
|
+
|
1325
|
+
// Workflow starting over, this time issue the right command
|
1326
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1327
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1328
|
+
act.run_id,
|
1329
|
+
vec![start_timer_cmd(1, Duration::from_secs(1))],
|
1330
|
+
))
|
1331
|
+
.await
|
1332
|
+
.unwrap();
|
1333
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1334
|
+
assert_matches!(
|
1335
|
+
act.jobs.as_slice(),
|
1336
|
+
[WorkflowActivationJob {
|
1337
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1338
|
+
},]
|
1339
|
+
);
|
1340
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1341
|
+
act.run_id,
|
1342
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1343
|
+
))
|
1344
|
+
.await
|
1345
|
+
.unwrap();
|
1346
|
+
core.shutdown().await;
|
1347
|
+
}
|
1348
|
+
|
1349
|
+
#[tokio::test]
|
1350
|
+
async fn poll_response_triggers_wf_error() {
|
1351
|
+
let mut t = TestHistoryBuilder::default();
|
1352
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1353
|
+
// Add this nonsense event here to make applying the poll response fail
|
1354
|
+
t.add_external_signal_completed(100);
|
1355
|
+
t.add_full_wf_task();
|
1356
|
+
t.add_workflow_execution_completed();
|
1357
|
+
|
1358
|
+
let mh = MockPollCfg::from_resp_batches(
|
1359
|
+
"fake_wf_id",
|
1360
|
+
t,
|
1361
|
+
[ResponseType::AllHistory],
|
1362
|
+
mock_workflow_client(),
|
1363
|
+
);
|
1364
|
+
let mock = build_mock_pollers(mh);
|
1365
|
+
let core = mock_worker(mock);
|
1366
|
+
// Poll for first WFT, which is immediately an eviction
|
1367
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1368
|
+
assert_matches!(
|
1369
|
+
act.jobs.as_slice(),
|
1370
|
+
[WorkflowActivationJob {
|
1371
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1372
|
+
}]
|
1373
|
+
);
|
1374
|
+
}
|
1375
|
+
|
1376
|
+
// Verifies we can handle multiple wft timeouts in a row if lang is being very slow in responding
|
1377
|
+
#[tokio::test]
|
1378
|
+
async fn lang_slower_than_wft_timeouts() {
|
1379
|
+
let wfid = "fake_wf_id";
|
1380
|
+
let mut t = TestHistoryBuilder::default();
|
1381
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1382
|
+
t.add_workflow_task_scheduled_and_started();
|
1383
|
+
t.add_workflow_task_timed_out();
|
1384
|
+
t.add_full_wf_task();
|
1385
|
+
t.add_workflow_execution_completed();
|
1386
|
+
|
1387
|
+
let mut mock = mock_workflow_client();
|
1388
|
+
mock.expect_complete_workflow_task()
|
1389
|
+
.times(1)
|
1390
|
+
.returning(|_| Err(tonic::Status::not_found("Workflow task not found.")));
|
1391
|
+
mock.expect_complete_workflow_task()
|
1392
|
+
.times(1)
|
1393
|
+
.returning(|_| Ok(Default::default()));
|
1394
|
+
let mut mock = single_hist_mock_sg(wfid, t, [1, 1], mock, true);
|
1395
|
+
let tasksmap = mock.outstanding_task_map.clone().unwrap();
|
1396
|
+
mock.worker_cfg(|wc| {
|
1397
|
+
wc.max_cached_workflows = 2;
|
1398
|
+
});
|
1399
|
+
let core = mock_worker(mock);
|
1400
|
+
|
1401
|
+
// This completion runs into the workflow task not found error
|
1402
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1403
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
|
1404
|
+
.await
|
1405
|
+
.unwrap();
|
1406
|
+
// It will get an eviction
|
1407
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1408
|
+
assert_matches!(
|
1409
|
+
wf_task.jobs.as_slice(),
|
1410
|
+
[WorkflowActivationJob {
|
1411
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1412
|
+
}]
|
1413
|
+
);
|
1414
|
+
// Before we complete, unlock the next task from the mock so that we'll see it get buffered.
|
1415
|
+
tasksmap.release_run(&wf_task.run_id);
|
1416
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
|
1417
|
+
.await
|
1418
|
+
.unwrap();
|
1419
|
+
// The buffered WFT should be applied now
|
1420
|
+
let start_again = core.poll_workflow_activation().await.unwrap();
|
1421
|
+
assert_matches!(
|
1422
|
+
start_again.jobs[0].variant,
|
1423
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_))
|
1424
|
+
);
|
1425
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1426
|
+
start_again.run_id,
|
1427
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1428
|
+
))
|
1429
|
+
.await
|
1430
|
+
.unwrap();
|
1431
|
+
core.shutdown().await;
|
1432
|
+
}
|
1433
|
+
|
1434
|
+
#[tokio::test]
|
1435
|
+
async fn tries_cancel_of_completed_activity() {
|
1436
|
+
let mut t = TestHistoryBuilder::default();
|
1437
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1438
|
+
t.add_full_wf_task();
|
1439
|
+
let scheduled_event_id = t.add_activity_task_scheduled("1");
|
1440
|
+
t.add_we_signaled("sig", vec![]);
|
1441
|
+
let started_event_id = t.add_activity_task_started(scheduled_event_id);
|
1442
|
+
t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
|
1443
|
+
t.add_workflow_task_scheduled_and_started();
|
1444
|
+
|
1445
|
+
let mock = mock_workflow_client();
|
1446
|
+
let mut mock = single_hist_mock_sg("fake_wf_id", t, [1, 2], mock, true);
|
1447
|
+
mock.worker_cfg(|cfg| cfg.max_cached_workflows = 1);
|
1448
|
+
let core = mock_worker(mock);
|
1449
|
+
|
1450
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1451
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1452
|
+
activation.run_id,
|
1453
|
+
ScheduleActivity {
|
1454
|
+
seq: 1,
|
1455
|
+
activity_id: "1".to_string(),
|
1456
|
+
..Default::default()
|
1457
|
+
}
|
1458
|
+
.into(),
|
1459
|
+
))
|
1460
|
+
.await
|
1461
|
+
.unwrap();
|
1462
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1463
|
+
assert_matches!(
|
1464
|
+
activation.jobs.as_slice(),
|
1465
|
+
[
|
1466
|
+
WorkflowActivationJob {
|
1467
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1468
|
+
},
|
1469
|
+
WorkflowActivationJob {
|
1470
|
+
variant: Some(workflow_activation_job::Variant::ResolveActivity(_)),
|
1471
|
+
}
|
1472
|
+
]
|
1473
|
+
);
|
1474
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1475
|
+
activation.run_id,
|
1476
|
+
vec![
|
1477
|
+
RequestCancelActivity { seq: 1 }.into(),
|
1478
|
+
CompleteWorkflowExecution { result: None }.into(),
|
1479
|
+
],
|
1480
|
+
))
|
1481
|
+
.await
|
1482
|
+
.unwrap();
|
1483
|
+
|
1484
|
+
core.shutdown().await;
|
1485
|
+
}
|
1486
|
+
|
1487
|
+
#[tokio::test]
|
1488
|
+
async fn failing_wft_doesnt_eat_permit_forever() {
|
1489
|
+
let mut t = TestHistoryBuilder::default();
|
1490
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1491
|
+
t.add_workflow_task_scheduled_and_started();
|
1492
|
+
|
1493
|
+
let mock = mock_workflow_client();
|
1494
|
+
let mut mock = MockPollCfg::from_resp_batches("fake_wf_id", t, [1, 1, 1], mock);
|
1495
|
+
mock.num_expected_fails = 1;
|
1496
|
+
let mut mock = build_mock_pollers(mock);
|
1497
|
+
mock.worker_cfg(|cfg| {
|
1498
|
+
cfg.max_cached_workflows = 2;
|
1499
|
+
cfg.max_outstanding_workflow_tasks = 2;
|
1500
|
+
});
|
1501
|
+
let outstanding_mock_tasks = mock.outstanding_task_map.clone();
|
1502
|
+
let worker = mock_worker(mock);
|
1503
|
+
|
1504
|
+
let mut run_id = "".to_string();
|
1505
|
+
// Fail twice, verifying a permit is not eaten. We cannot fail the same run more than twice in a
|
1506
|
+
// row because we purposefully time out rather than spamming.
|
1507
|
+
for _ in 1..=2 {
|
1508
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1509
|
+
// Issue a nonsense completion that will trigger a WFT failure
|
1510
|
+
worker
|
1511
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1512
|
+
activation.run_id,
|
1513
|
+
RequestCancelActivity { seq: 1 }.into(),
|
1514
|
+
))
|
1515
|
+
.await
|
1516
|
+
.unwrap();
|
1517
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1518
|
+
assert_matches!(
|
1519
|
+
activation.jobs.as_slice(),
|
1520
|
+
[WorkflowActivationJob {
|
1521
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1522
|
+
},]
|
1523
|
+
);
|
1524
|
+
run_id = activation.run_id.clone();
|
1525
|
+
worker
|
1526
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
|
1527
|
+
.await
|
1528
|
+
.unwrap();
|
1529
|
+
}
|
1530
|
+
assert_eq!(worker.outstanding_workflow_tasks().await, 0);
|
1531
|
+
// 1 permit is in use because the next task is buffered and has re-used the permit
|
1532
|
+
assert_eq!(worker.available_wft_permits().await, 1);
|
1533
|
+
// We should be "out of work" because the mock service thinks we didn't complete the last task,
|
1534
|
+
// which we didn't, because we don't spam failures. The real server would eventually time out
|
1535
|
+
// the task. Mock doesn't understand that, so the WFT permit is released because eventually a
|
1536
|
+
// new one will be generated. We manually clear the mock's outstanding task list so the next
|
1537
|
+
// poll will work.
|
1538
|
+
outstanding_mock_tasks.unwrap().release_run(&run_id);
|
1539
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1540
|
+
// There should be no change in permits, since this just unbuffered the buffered task
|
1541
|
+
assert_eq!(worker.available_wft_permits().await, 1);
|
1542
|
+
worker
|
1543
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1544
|
+
activation.run_id,
|
1545
|
+
CompleteWorkflowExecution { result: None }.into(),
|
1546
|
+
))
|
1547
|
+
.await
|
1548
|
+
.unwrap();
|
1549
|
+
assert_eq!(worker.available_wft_permits().await, 2);
|
1550
|
+
|
1551
|
+
worker.shutdown().await;
|
1552
|
+
}
|
1553
|
+
|
1554
|
+
#[tokio::test]
|
1555
|
+
async fn cache_miss_will_fetch_history() {
|
1556
|
+
let mut t = TestHistoryBuilder::default();
|
1557
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1558
|
+
t.add_full_wf_task();
|
1559
|
+
t.add_we_signaled("sig", vec![]);
|
1560
|
+
t.add_full_wf_task();
|
1561
|
+
t.add_workflow_execution_completed();
|
1562
|
+
let get_exec_resp: GetWorkflowExecutionHistoryResponse = t.get_history_info(2).unwrap().into();
|
1563
|
+
|
1564
|
+
let mut mh = MockPollCfg::from_resp_batches(
|
1565
|
+
"fake_wf_id",
|
1566
|
+
t,
|
1567
|
+
[ResponseType::ToTaskNum(1), ResponseType::OneTask(2)],
|
1568
|
+
mock_workflow_client(),
|
1569
|
+
);
|
1570
|
+
mh.mock_client
|
1571
|
+
.expect_get_workflow_execution_history()
|
1572
|
+
.times(1)
|
1573
|
+
.returning(move |_, _, _| Ok(get_exec_resp.clone()));
|
1574
|
+
let mut mock = build_mock_pollers(mh);
|
1575
|
+
mock.worker_cfg(|cfg| {
|
1576
|
+
cfg.max_cached_workflows = 1;
|
1577
|
+
});
|
1578
|
+
let worker = mock_worker(mock);
|
1579
|
+
|
1580
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1581
|
+
assert_eq!(activation.history_length, 3);
|
1582
|
+
assert_matches!(
|
1583
|
+
activation.jobs.as_slice(),
|
1584
|
+
[WorkflowActivationJob {
|
1585
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
|
1586
|
+
}]
|
1587
|
+
);
|
1588
|
+
// Force an eviction (before complete matters, so that we will be sure the eviction is queued
|
1589
|
+
// up before the next fake WFT is unlocked)
|
1590
|
+
worker.request_wf_eviction(
|
1591
|
+
&activation.run_id,
|
1592
|
+
"whatever",
|
1593
|
+
EvictionReason::LangRequested,
|
1594
|
+
);
|
1595
|
+
worker
|
1596
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(&activation.run_id))
|
1597
|
+
.await
|
1598
|
+
.unwrap();
|
1599
|
+
// Handle the eviction, and the restart
|
1600
|
+
for i in 1..=2 {
|
1601
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1602
|
+
assert_eq!(activation.history_length, 3);
|
1603
|
+
if i == 1 {
|
1604
|
+
assert_matches!(
|
1605
|
+
activation.jobs.as_slice(),
|
1606
|
+
[WorkflowActivationJob {
|
1607
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1608
|
+
}]
|
1609
|
+
);
|
1610
|
+
} else {
|
1611
|
+
assert_matches!(
|
1612
|
+
activation.jobs.as_slice(),
|
1613
|
+
[WorkflowActivationJob {
|
1614
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
|
1615
|
+
}]
|
1616
|
+
);
|
1617
|
+
}
|
1618
|
+
worker
|
1619
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
|
1620
|
+
.await
|
1621
|
+
.unwrap();
|
1622
|
+
}
|
1623
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
1624
|
+
assert_eq!(activation.history_length, 7);
|
1625
|
+
assert_matches!(
|
1626
|
+
activation.jobs.as_slice(),
|
1627
|
+
[WorkflowActivationJob {
|
1628
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1629
|
+
}]
|
1630
|
+
);
|
1631
|
+
worker
|
1632
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1633
|
+
activation.run_id,
|
1634
|
+
CompleteWorkflowExecution { result: None }.into(),
|
1635
|
+
))
|
1636
|
+
.await
|
1637
|
+
.unwrap();
|
1638
|
+
assert_eq!(worker.outstanding_workflow_tasks().await, 0);
|
1639
|
+
worker.shutdown().await;
|
1640
|
+
}
|
1641
|
+
|
1642
|
+
/// This test verifies that WFTs which come as replies to completing a WFT are properly delivered
|
1643
|
+
/// via activation polling.
|
1644
|
+
#[tokio::test]
|
1645
|
+
async fn tasks_from_completion_are_delivered() {
|
1646
|
+
let wfid = "fake_wf_id";
|
1647
|
+
let mut t = TestHistoryBuilder::default();
|
1648
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1649
|
+
t.add_full_wf_task();
|
1650
|
+
t.add_we_signaled("sig", vec![]);
|
1651
|
+
t.add_full_wf_task();
|
1652
|
+
t.add_workflow_execution_completed();
|
1653
|
+
|
1654
|
+
let mut mock = mock_workflow_client();
|
1655
|
+
let complete_resp = RespondWorkflowTaskCompletedResponse {
|
1656
|
+
workflow_task: Some(hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp),
|
1657
|
+
activity_tasks: vec![],
|
1658
|
+
reset_history_event_id: 0,
|
1659
|
+
};
|
1660
|
+
mock.expect_complete_workflow_task()
|
1661
|
+
.times(1)
|
1662
|
+
.returning(move |_| Ok(complete_resp.clone()));
|
1663
|
+
mock.expect_complete_workflow_task()
|
1664
|
+
.times(1)
|
1665
|
+
.returning(|_| Ok(Default::default()));
|
1666
|
+
let mut mock = single_hist_mock_sg(wfid, t, [1], mock, true);
|
1667
|
+
mock.worker_cfg(|wc| wc.max_cached_workflows = 2);
|
1668
|
+
let core = mock_worker(mock);
|
1669
|
+
|
1670
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1671
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(wf_task.run_id))
|
1672
|
+
.await
|
1673
|
+
.unwrap();
|
1674
|
+
let wf_task = core.poll_workflow_activation().await.unwrap();
|
1675
|
+
assert_matches!(
|
1676
|
+
wf_task.jobs.as_slice(),
|
1677
|
+
[WorkflowActivationJob {
|
1678
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1679
|
+
},]
|
1680
|
+
);
|
1681
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1682
|
+
wf_task.run_id,
|
1683
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1684
|
+
))
|
1685
|
+
.await
|
1686
|
+
.unwrap();
|
1687
|
+
core.shutdown().await;
|
1688
|
+
}
|
1689
|
+
|
1690
|
+
#[tokio::test]
|
1691
|
+
async fn poll_faster_than_complete_wont_overflow_cache() {
|
1692
|
+
// Make workflow tasks for 5 different runs
|
1693
|
+
let tasks: Vec<_> = (1..=5)
|
1694
|
+
.map(|i| FakeWfResponses {
|
1695
|
+
wf_id: format!("wf-{}", i),
|
1696
|
+
hist: canned_histories::single_timer("1"),
|
1697
|
+
response_batches: vec![ResponseType::ToTaskNum(1)],
|
1698
|
+
})
|
1699
|
+
.collect();
|
1700
|
+
let mut mock_client = mock_workflow_client();
|
1701
|
+
mock_client
|
1702
|
+
.expect_complete_workflow_task()
|
1703
|
+
.times(3)
|
1704
|
+
.returning(|_| Ok(Default::default()));
|
1705
|
+
let mut mock_cfg = MockPollCfg::new(tasks, true, 0);
|
1706
|
+
mock_cfg.mock_client = mock_client;
|
1707
|
+
let mut mock = build_mock_pollers(mock_cfg);
|
1708
|
+
mock.worker_cfg(|wc| {
|
1709
|
+
wc.max_cached_workflows = 3;
|
1710
|
+
wc.max_outstanding_workflow_tasks = 3;
|
1711
|
+
});
|
1712
|
+
let core = mock_worker(mock);
|
1713
|
+
// Poll 4 times, completing once, such that max tasks are never exceeded
|
1714
|
+
let p1 = core.poll_workflow_activation().await.unwrap();
|
1715
|
+
let p2 = core.poll_workflow_activation().await.unwrap();
|
1716
|
+
let p3 = core.poll_workflow_activation().await.unwrap();
|
1717
|
+
for (i, p_res) in [&p1, &p2, &p3].into_iter().enumerate() {
|
1718
|
+
assert_matches!(
|
1719
|
+
&p_res.jobs[0].variant,
|
1720
|
+
Some(workflow_activation_job::Variant::StartWorkflow(sw))
|
1721
|
+
if sw.workflow_id == format!("wf-{}", i + 1)
|
1722
|
+
);
|
1723
|
+
}
|
1724
|
+
// Complete first task to free a wft slot. Cache size is at 3
|
1725
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1726
|
+
p1.run_id,
|
1727
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1728
|
+
))
|
1729
|
+
.await
|
1730
|
+
.unwrap();
|
1731
|
+
// Now we're at cache limit. We will poll for a task, discover it is for a new run, issue
|
1732
|
+
// an eviction, and buffer the new run task. However, the run we're trying to evict has pending
|
1733
|
+
// activations! Thus, we must complete them first before this poll will unblock, and then it
|
1734
|
+
// will unblock with the eviciton.
|
1735
|
+
let p4 = core.poll_workflow_activation();
|
1736
|
+
// Make sure the task gets buffered before we start the complete, so the LRU list is in the
|
1737
|
+
// expected order and what we expect to evict will be evicted.
|
1738
|
+
advance_fut!(p4);
|
1739
|
+
let p4 = async {
|
1740
|
+
let p4 = p4.await.unwrap();
|
1741
|
+
assert_matches!(
|
1742
|
+
&p4.jobs.as_slice(),
|
1743
|
+
[WorkflowActivationJob {
|
1744
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1745
|
+
}]
|
1746
|
+
);
|
1747
|
+
p4
|
1748
|
+
};
|
1749
|
+
let p2_pending_completer = async {
|
1750
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1751
|
+
p2.run_id,
|
1752
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1753
|
+
))
|
1754
|
+
.await
|
1755
|
+
.unwrap();
|
1756
|
+
};
|
1757
|
+
let (p4, _) = join!(p4, p2_pending_completer);
|
1758
|
+
assert_eq!(core.cached_workflows().await, 3);
|
1759
|
+
|
1760
|
+
// This poll should also block until the eviction is actually completed
|
1761
|
+
let blocking_poll = async {
|
1762
|
+
let res = core.poll_workflow_activation().await.unwrap();
|
1763
|
+
assert_matches!(
|
1764
|
+
&res.jobs[0].variant,
|
1765
|
+
Some(workflow_activation_job::Variant::StartWorkflow(sw))
|
1766
|
+
if sw.workflow_id == format!("wf-{}", 4)
|
1767
|
+
);
|
1768
|
+
res
|
1769
|
+
};
|
1770
|
+
let complete_evict = async {
|
1771
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(p4.run_id))
|
1772
|
+
.await
|
1773
|
+
.unwrap();
|
1774
|
+
};
|
1775
|
+
|
1776
|
+
let (_p5, _) = join!(blocking_poll, complete_evict);
|
1777
|
+
assert_eq!(core.cached_workflows().await, 3);
|
1778
|
+
// The next poll will get an buffer a task for a new run, and generate an eviction for p3 but
|
1779
|
+
// that eviction cannot be obtained until we complete the existing outstanding task.
|
1780
|
+
let p6 = async {
|
1781
|
+
let p6 = core.poll_workflow_activation().await.unwrap();
|
1782
|
+
assert_matches!(
|
1783
|
+
p6.jobs.as_slice(),
|
1784
|
+
[WorkflowActivationJob {
|
1785
|
+
variant: Some(workflow_activation_job::Variant::RemoveFromCache(_)),
|
1786
|
+
}]
|
1787
|
+
);
|
1788
|
+
p6
|
1789
|
+
};
|
1790
|
+
let completer = async {
|
1791
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1792
|
+
p3.run_id,
|
1793
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1794
|
+
))
|
1795
|
+
.await
|
1796
|
+
.unwrap();
|
1797
|
+
};
|
1798
|
+
let (p6, _) = join!(p6, completer);
|
1799
|
+
let complete_evict = async {
|
1800
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(p6.run_id))
|
1801
|
+
.await
|
1802
|
+
.unwrap();
|
1803
|
+
};
|
1804
|
+
let blocking_poll = async {
|
1805
|
+
// This poll will also block until the last eviction goes through, and when it does it'll
|
1806
|
+
// produce the final start workflow task
|
1807
|
+
let res = core.poll_workflow_activation().await.unwrap();
|
1808
|
+
assert_matches!(
|
1809
|
+
&res.jobs[0].variant,
|
1810
|
+
Some(workflow_activation_job::Variant::StartWorkflow(sw))
|
1811
|
+
if sw.workflow_id == "wf-5"
|
1812
|
+
);
|
1813
|
+
};
|
1814
|
+
|
1815
|
+
join!(blocking_poll, complete_evict);
|
1816
|
+
// p5 outstanding and final poll outstanding -- hence one permit available
|
1817
|
+
assert_eq!(core.available_wft_permits().await, 1);
|
1818
|
+
assert_eq!(core.cached_workflows().await, 3);
|
1819
|
+
}
|
1820
|
+
|
1821
|
+
#[tokio::test]
|
1822
|
+
async fn eviction_waits_until_replay_finished() {
|
1823
|
+
let wfid = "fake_wf_id";
|
1824
|
+
let t = canned_histories::long_sequential_timers(3);
|
1825
|
+
let mock = mock_workflow_client();
|
1826
|
+
let mock = single_hist_mock_sg(wfid, t, [3], mock, true);
|
1827
|
+
let core = mock_worker(mock);
|
1828
|
+
|
1829
|
+
let activation = core.poll_workflow_activation().await.unwrap();
|
1830
|
+
assert_eq!(activation.history_length, 3);
|
1831
|
+
// Immediately request eviction after getting start workflow
|
1832
|
+
core.request_workflow_eviction(&activation.run_id);
|
1833
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1834
|
+
activation.run_id,
|
1835
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1836
|
+
))
|
1837
|
+
.await
|
1838
|
+
.unwrap();
|
1839
|
+
let t1_fired = core.poll_workflow_activation().await.unwrap();
|
1840
|
+
assert_eq!(t1_fired.history_length, 8);
|
1841
|
+
assert_matches!(
|
1842
|
+
t1_fired.jobs.as_slice(),
|
1843
|
+
[WorkflowActivationJob {
|
1844
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1845
|
+
}]
|
1846
|
+
);
|
1847
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1848
|
+
t1_fired.run_id,
|
1849
|
+
start_timer_cmd(2, Duration::from_secs(1)),
|
1850
|
+
))
|
1851
|
+
.await
|
1852
|
+
.unwrap();
|
1853
|
+
let t2_fired = core.poll_workflow_activation().await.unwrap();
|
1854
|
+
assert_eq!(t2_fired.history_length, 13);
|
1855
|
+
assert_matches!(
|
1856
|
+
t2_fired.jobs.as_slice(),
|
1857
|
+
[WorkflowActivationJob {
|
1858
|
+
variant: Some(workflow_activation_job::Variant::FireTimer(_)),
|
1859
|
+
}]
|
1860
|
+
);
|
1861
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmds(
|
1862
|
+
t2_fired.run_id,
|
1863
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
1864
|
+
))
|
1865
|
+
.await
|
1866
|
+
.unwrap();
|
1867
|
+
|
1868
|
+
core.shutdown().await;
|
1869
|
+
}
|
1870
|
+
|
1871
|
+
#[tokio::test]
|
1872
|
+
async fn autocompletes_wft_no_work() {
|
1873
|
+
let wfid = "fake_wf_id";
|
1874
|
+
let activity_id = "1";
|
1875
|
+
|
1876
|
+
let mut t = TestHistoryBuilder::default();
|
1877
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
1878
|
+
t.add_full_wf_task();
|
1879
|
+
let scheduled_event_id = t.add_activity_task_scheduled(activity_id);
|
1880
|
+
t.add_full_wf_task();
|
1881
|
+
t.add_we_signaled("sig1", vec![]);
|
1882
|
+
t.add_full_wf_task();
|
1883
|
+
let started_event_id = t.add_activity_task_started(scheduled_event_id);
|
1884
|
+
t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
|
1885
|
+
t.add_full_wf_task();
|
1886
|
+
let mock = mock_workflow_client();
|
1887
|
+
let mut mock = single_hist_mock_sg(wfid, t, [1, 2, 3, 4], mock, true);
|
1888
|
+
mock.worker_cfg(|w| w.max_cached_workflows = 1);
|
1889
|
+
let core = mock_worker(mock);
|
1890
|
+
|
1891
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1892
|
+
assert_matches!(
|
1893
|
+
act.jobs.as_slice(),
|
1894
|
+
[WorkflowActivationJob {
|
1895
|
+
variant: Some(workflow_activation_job::Variant::StartWorkflow(_)),
|
1896
|
+
}]
|
1897
|
+
);
|
1898
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1899
|
+
act.run_id,
|
1900
|
+
ScheduleActivity {
|
1901
|
+
seq: 1,
|
1902
|
+
activity_id: activity_id.to_string(),
|
1903
|
+
cancellation_type: ActivityCancellationType::Abandon as i32,
|
1904
|
+
..Default::default()
|
1905
|
+
}
|
1906
|
+
.into(),
|
1907
|
+
))
|
1908
|
+
.await
|
1909
|
+
.unwrap();
|
1910
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1911
|
+
assert_matches!(
|
1912
|
+
act.jobs.as_slice(),
|
1913
|
+
[WorkflowActivationJob {
|
1914
|
+
variant: Some(workflow_activation_job::Variant::SignalWorkflow(_)),
|
1915
|
+
}]
|
1916
|
+
);
|
1917
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1918
|
+
act.run_id,
|
1919
|
+
RequestCancelActivity { seq: 1 }.into(),
|
1920
|
+
))
|
1921
|
+
.await
|
1922
|
+
.unwrap();
|
1923
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
1924
|
+
core.complete_workflow_activation(WorkflowActivationCompletion::empty(act.run_id))
|
1925
|
+
.await
|
1926
|
+
.unwrap();
|
1927
|
+
// The last task will autocomplete, and thus this will return shutdown since there is no more
|
1928
|
+
// work
|
1929
|
+
assert_matches!(
|
1930
|
+
core.poll_workflow_activation().await.unwrap_err(),
|
1931
|
+
PollWfError::ShutDown
|
1932
|
+
);
|
1933
|
+
|
1934
|
+
core.shutdown().await;
|
1935
|
+
}
|
1936
|
+
|
1937
|
+
#[tokio::test]
|
1938
|
+
async fn no_race_acquiring_permits() {
|
1939
|
+
let wfid = "fake_wf_id";
|
1940
|
+
let mut mock_client = mock_manual_workflow_client();
|
1941
|
+
// We need to allow two polls to happen by triggering two processing events in the workflow
|
1942
|
+
// stream, but then delivering the actual tasks after that
|
1943
|
+
let task_barr: &'static Barrier = Box::leak(Box::new(Barrier::new(2)));
|
1944
|
+
mock_client
|
1945
|
+
.expect_poll_workflow_task()
|
1946
|
+
.returning(move |_, _| {
|
1947
|
+
let t = canned_histories::single_timer("1");
|
1948
|
+
let poll_resp = hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp;
|
1949
|
+
async move {
|
1950
|
+
task_barr.wait().await;
|
1951
|
+
Ok(poll_resp.clone())
|
1952
|
+
}
|
1953
|
+
.boxed()
|
1954
|
+
});
|
1955
|
+
mock_client
|
1956
|
+
.expect_complete_workflow_task()
|
1957
|
+
.returning(|_| async move { Ok(Default::default()) }.boxed());
|
1958
|
+
|
1959
|
+
let worker = Worker::new_test(
|
1960
|
+
test_worker_cfg()
|
1961
|
+
.max_outstanding_workflow_tasks(1_usize)
|
1962
|
+
.max_cached_workflows(10_usize)
|
1963
|
+
.build()
|
1964
|
+
.unwrap(),
|
1965
|
+
mock_client,
|
1966
|
+
);
|
1967
|
+
|
1968
|
+
// Two polls in a row, both of which will get stuck on the barrier and are only allowed to
|
1969
|
+
// proceed after a call which will cause the workflow stream to process an event. Without the
|
1970
|
+
// fix, this would've meant the stream though it was OK to poll twice, but once the tasks
|
1971
|
+
// are received, it would find there was only one permit.
|
1972
|
+
let poll_1_f = async {
|
1973
|
+
let r = worker.poll_workflow_activation().await.unwrap();
|
1974
|
+
worker
|
1975
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1976
|
+
r.run_id,
|
1977
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1978
|
+
))
|
1979
|
+
.await
|
1980
|
+
.unwrap();
|
1981
|
+
};
|
1982
|
+
let poll_2_f = async {
|
1983
|
+
let r = worker.poll_workflow_activation().await.unwrap();
|
1984
|
+
worker
|
1985
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
1986
|
+
r.run_id,
|
1987
|
+
start_timer_cmd(1, Duration::from_secs(1)),
|
1988
|
+
))
|
1989
|
+
.await
|
1990
|
+
.unwrap();
|
1991
|
+
};
|
1992
|
+
let other_f = async {
|
1993
|
+
worker.cached_workflows().await;
|
1994
|
+
task_barr.wait().await;
|
1995
|
+
worker.cached_workflows().await;
|
1996
|
+
task_barr.wait().await;
|
1997
|
+
};
|
1998
|
+
join!(poll_1_f, poll_2_f, other_f);
|
1999
|
+
}
|
2000
|
+
|
2001
|
+
#[tokio::test]
|
2002
|
+
async fn continue_as_new_preserves_some_values() {
|
2003
|
+
let wfid = "fake_wf_id";
|
2004
|
+
let memo = HashMap::<String, Payload>::from([("enchi".to_string(), b"cat".into())]).into();
|
2005
|
+
let search = HashMap::<String, Payload>::from([("noisy".to_string(), b"kitty".into())]).into();
|
2006
|
+
let retry_policy = RetryPolicy {
|
2007
|
+
backoff_coefficient: 13.37,
|
2008
|
+
..Default::default()
|
2009
|
+
};
|
2010
|
+
let mut wes_attrs = default_wes_attribs();
|
2011
|
+
wes_attrs.memo = Some(memo);
|
2012
|
+
wes_attrs.search_attributes = Some(search);
|
2013
|
+
wes_attrs.retry_policy = Some(retry_policy);
|
2014
|
+
let mut mock_client = mock_workflow_client();
|
2015
|
+
let hist = {
|
2016
|
+
let mut t = TestHistoryBuilder::default();
|
2017
|
+
t.add(
|
2018
|
+
EventType::WorkflowExecutionStarted,
|
2019
|
+
wes_attrs.clone().into(),
|
2020
|
+
);
|
2021
|
+
t.add_full_wf_task();
|
2022
|
+
t
|
2023
|
+
};
|
2024
|
+
mock_client
|
2025
|
+
.expect_poll_workflow_task()
|
2026
|
+
.returning(move |_, _| {
|
2027
|
+
Ok(hist_to_poll_resp(&hist, wfid.to_owned(), ResponseType::AllHistory).resp)
|
2028
|
+
});
|
2029
|
+
mock_client
|
2030
|
+
.expect_complete_workflow_task()
|
2031
|
+
.returning(move |mut c| {
|
2032
|
+
let can_cmd = c.commands.pop().unwrap().attributes.unwrap();
|
2033
|
+
if let Attributes::ContinueAsNewWorkflowExecutionCommandAttributes(a) = can_cmd {
|
2034
|
+
assert_eq!(a.workflow_type.unwrap().name, "meow");
|
2035
|
+
assert_eq!(a.memo, wes_attrs.memo);
|
2036
|
+
assert_eq!(a.search_attributes, wes_attrs.search_attributes);
|
2037
|
+
assert_eq!(a.retry_policy, wes_attrs.retry_policy);
|
2038
|
+
} else {
|
2039
|
+
panic!("Wrong attributes type");
|
2040
|
+
}
|
2041
|
+
Ok(Default::default())
|
2042
|
+
});
|
2043
|
+
|
2044
|
+
let worker = Worker::new_test(test_worker_cfg().build().unwrap(), mock_client);
|
2045
|
+
let r = worker.poll_workflow_activation().await.unwrap();
|
2046
|
+
worker
|
2047
|
+
.complete_workflow_activation(WorkflowActivationCompletion::from_cmd(
|
2048
|
+
r.run_id,
|
2049
|
+
ContinueAsNewWorkflowExecution {
|
2050
|
+
workflow_type: "meow".to_string(),
|
2051
|
+
..Default::default()
|
2052
|
+
}
|
2053
|
+
.into(),
|
2054
|
+
))
|
2055
|
+
.await
|
2056
|
+
.unwrap();
|
2057
|
+
}
|
2058
|
+
|
2059
|
+
#[rstest]
|
2060
|
+
#[tokio::test]
|
2061
|
+
async fn ignorable_events_are_ok(#[values(true, false)] attribs_unset: bool) {
|
2062
|
+
let mut t = TestHistoryBuilder::default();
|
2063
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
2064
|
+
let id = t.add_get_event_id(
|
2065
|
+
EventType::Unspecified,
|
2066
|
+
Some(
|
2067
|
+
history_event::Attributes::WorkflowPropertiesModifiedExternallyEventAttributes(
|
2068
|
+
Default::default(),
|
2069
|
+
),
|
2070
|
+
),
|
2071
|
+
);
|
2072
|
+
t.modify_event(id, |e| e.worker_may_ignore = true);
|
2073
|
+
if attribs_unset {
|
2074
|
+
t.modify_event(id, |e| {
|
2075
|
+
e.event_type = EventType::WorkflowPropertiesModifiedExternally as i32;
|
2076
|
+
e.attributes = None;
|
2077
|
+
});
|
2078
|
+
}
|
2079
|
+
t.add_workflow_task_scheduled_and_started();
|
2080
|
+
|
2081
|
+
let mock = mock_workflow_client();
|
2082
|
+
let mock = single_hist_mock_sg("wheee", t, [ResponseType::AllHistory], mock, true);
|
2083
|
+
let core = mock_worker(mock);
|
2084
|
+
|
2085
|
+
let act = core.poll_workflow_activation().await.unwrap();
|
2086
|
+
assert_matches!(
|
2087
|
+
act.jobs[0].variant,
|
2088
|
+
Some(workflow_activation_job::Variant::StartWorkflow(_))
|
2089
|
+
);
|
2090
|
+
}
|