@temporalio/core-bridge 1.11.8 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +219 -193
- package/Cargo.toml +27 -8
- package/README.md +5 -0
- package/index.js +72 -12
- package/lib/errors.d.ts +25 -0
- package/lib/errors.js +76 -1
- package/lib/errors.js.map +1 -1
- package/lib/index.d.ts +11 -478
- package/lib/index.js +28 -5
- package/lib/index.js.map +1 -1
- package/lib/native.d.ts +330 -0
- package/lib/{worker-tuner.js → native.js} +1 -1
- package/lib/native.js.map +1 -0
- package/package.json +7 -3
- package/releases/aarch64-apple-darwin/index.node +0 -0
- package/releases/aarch64-unknown-linux-gnu/index.node +0 -0
- package/releases/x86_64-apple-darwin/index.node +0 -0
- package/releases/x86_64-pc-windows-msvc/index.node +0 -0
- package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
- package/sdk-core/.cargo/config.toml +8 -2
- package/sdk-core/.cargo/multi-worker-manual-test +15 -0
- package/sdk-core/.github/workflows/per-pr.yml +40 -11
- package/sdk-core/AGENTS.md +73 -0
- package/sdk-core/ARCHITECTURE.md +71 -23
- package/sdk-core/Cargo.toml +1 -1
- package/sdk-core/README.md +4 -4
- package/sdk-core/arch_docs/workflow_task_chunking.md +51 -0
- package/sdk-core/client/Cargo.toml +1 -1
- package/sdk-core/client/src/lib.rs +49 -13
- package/sdk-core/client/src/metrics.rs +15 -16
- package/sdk-core/client/src/proxy.rs +2 -2
- package/sdk-core/client/src/raw.rs +54 -8
- package/sdk-core/client/src/retry.rs +109 -13
- package/sdk-core/client/src/worker_registry/mod.rs +4 -4
- package/sdk-core/client/src/workflow_handle/mod.rs +1 -1
- package/sdk-core/core/Cargo.toml +28 -8
- package/sdk-core/core/src/abstractions.rs +62 -10
- package/sdk-core/core/src/core_tests/activity_tasks.rs +180 -8
- package/sdk-core/core/src/core_tests/mod.rs +4 -4
- package/sdk-core/core/src/core_tests/queries.rs +18 -4
- package/sdk-core/core/src/core_tests/workers.rs +3 -3
- package/sdk-core/core/src/core_tests/workflow_tasks.rs +191 -25
- package/sdk-core/core/src/ephemeral_server/mod.rs +10 -3
- package/sdk-core/core/src/internal_flags.rs +14 -14
- package/sdk-core/core/src/lib.rs +5 -2
- package/sdk-core/core/src/pollers/mod.rs +1 -1
- package/sdk-core/core/src/pollers/poll_buffer.rs +495 -164
- package/sdk-core/core/src/protosext/mod.rs +3 -3
- package/sdk-core/core/src/replay/mod.rs +3 -3
- package/sdk-core/core/src/telemetry/metrics.rs +13 -4
- package/sdk-core/core/src/telemetry/mod.rs +72 -70
- package/sdk-core/core/src/telemetry/otel.rs +51 -54
- package/sdk-core/core/src/test_help/mod.rs +9 -3
- package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +31 -11
- package/sdk-core/core/src/worker/activities/local_activities.rs +35 -28
- package/sdk-core/core/src/worker/activities.rs +58 -30
- package/sdk-core/core/src/worker/client/mocks.rs +3 -3
- package/sdk-core/core/src/worker/client.rs +155 -53
- package/sdk-core/core/src/worker/mod.rs +103 -95
- package/sdk-core/core/src/worker/nexus.rs +100 -73
- package/sdk-core/core/src/worker/tuner/resource_based.rs +14 -6
- package/sdk-core/core/src/worker/tuner.rs +4 -13
- package/sdk-core/core/src/worker/workflow/history_update.rs +47 -51
- package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +1 -4
- package/sdk-core/core/src/worker/workflow/machines/nexus_operation_state_machine.rs +127 -32
- package/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs +1 -2
- package/sdk-core/core/src/worker/workflow/machines/update_state_machine.rs +1 -1
- package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +55 -42
- package/sdk-core/core/src/worker/workflow/managed_run.rs +45 -35
- package/sdk-core/core/src/worker/workflow/mod.rs +200 -97
- package/sdk-core/core/src/worker/workflow/wft_poller.rs +175 -4
- package/sdk-core/core/src/worker/workflow/workflow_stream.rs +38 -36
- package/sdk-core/core-api/Cargo.toml +8 -0
- package/sdk-core/core-api/src/envconfig.rs +1544 -0
- package/sdk-core/core-api/src/lib.rs +2 -0
- package/sdk-core/core-api/src/telemetry/metrics.rs +8 -0
- package/sdk-core/core-api/src/telemetry.rs +36 -3
- package/sdk-core/core-api/src/worker.rs +301 -75
- package/sdk-core/docker/docker-compose-telem.yaml +1 -0
- package/sdk-core/etc/prometheus.yaml +6 -2
- package/sdk-core/histories/long_local_activity_with_update-0_history.bin +0 -0
- package/sdk-core/histories/long_local_activity_with_update-1_history.bin +0 -0
- package/sdk-core/histories/long_local_activity_with_update-2_history.bin +0 -0
- package/sdk-core/histories/long_local_activity_with_update-3_history.bin +0 -0
- package/sdk-core/sdk/src/activity_context.rs +5 -0
- package/sdk-core/sdk/src/interceptors.rs +73 -3
- package/sdk-core/sdk/src/lib.rs +15 -16
- package/sdk-core/sdk/src/workflow_context/options.rs +10 -0
- package/sdk-core/sdk/src/workflow_context.rs +48 -29
- package/sdk-core/sdk/src/workflow_future.rs +5 -6
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/.github/workflows/push-to-buf.yml +20 -0
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/CODEOWNERS +6 -0
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/README.md +17 -6
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/VERSION +1 -1
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/buf.lock +7 -2
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/buf.yaml +2 -0
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/cloudservice/v1/request_response.proto +78 -0
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/cloudservice/v1/service.proto +29 -0
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/identity/v1/message.proto +74 -32
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/namespace/v1/message.proto +45 -15
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/nexus/v1/message.proto +7 -1
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/operation/v1/message.proto +3 -3
- package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/region/v1/message.proto +3 -3
- package/sdk-core/sdk-core-protos/protos/api_upstream/LICENSE +1 -1
- package/sdk-core/sdk-core-protos/protos/api_upstream/buf.yaml +2 -0
- package/sdk-core/sdk-core-protos/protos/api_upstream/openapi/openapiv2.json +1103 -88
- package/sdk-core/sdk-core-protos/protos/api_upstream/openapi/openapiv3.yaml +1233 -151
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/activity/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/batch/v1/message.proto +19 -24
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/command/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/common/v1/message.proto +12 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/deployment/v1/message.proto +45 -45
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/command_type.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/common.proto +15 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/deployment.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/event_type.proto +4 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/namespace.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/nexus.proto +0 -20
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/query.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/reset.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/schedule.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/task_queue.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/update.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/workflow.proto +4 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/errordetails/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/export/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/failure/v1/message.proto +2 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/filter/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/history/v1/message.proto +75 -49
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/namespace/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/nexus/v1/message.proto +0 -20
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/protocol/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/query/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/replication/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/rules/v1/message.proto +90 -0
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/schedule/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/enhanced_stack_trace.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/task_complete_metadata.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/user_metadata.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/workflow_metadata.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +17 -38
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/update/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/version/v1/message.proto +0 -22
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflow/v1/message.proto +151 -44
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +172 -65
- package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +69 -28
- package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/activity_task/activity_task.proto +18 -0
- package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/common/common.proto +5 -0
- package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/nexus/nexus.proto +16 -1
- package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +21 -15
- package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +3 -0
- package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto +3 -0
- package/sdk-core/sdk-core-protos/src/lib.rs +60 -16
- package/sdk-core/test-utils/src/lib.rs +157 -39
- package/sdk-core/tests/cloud_tests.rs +86 -0
- package/sdk-core/tests/fuzzy_workflow.rs +23 -26
- package/sdk-core/tests/global_metric_tests.rs +116 -0
- package/sdk-core/tests/heavy_tests.rs +127 -7
- package/sdk-core/tests/integ_tests/client_tests.rs +2 -8
- package/sdk-core/tests/integ_tests/metrics_tests.rs +100 -106
- package/sdk-core/tests/integ_tests/polling_tests.rs +94 -8
- package/sdk-core/tests/integ_tests/update_tests.rs +75 -6
- package/sdk-core/tests/integ_tests/worker_tests.rs +54 -5
- package/sdk-core/tests/integ_tests/worker_versioning_tests.rs +240 -0
- package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +41 -3
- package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +168 -8
- package/sdk-core/tests/integ_tests/workflow_tests/nexus.rs +285 -15
- package/sdk-core/tests/integ_tests/workflow_tests/priority.rs +12 -4
- package/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +3 -2
- package/sdk-core/tests/integ_tests/workflow_tests.rs +124 -74
- package/sdk-core/tests/main.rs +3 -51
- package/sdk-core/tests/manual_tests.rs +430 -0
- package/sdk-core/tests/runner.rs +28 -2
- package/src/client.rs +565 -0
- package/src/helpers/abort_controller.rs +204 -0
- package/src/helpers/callbacks.rs +299 -0
- package/src/helpers/errors.rs +302 -0
- package/src/helpers/future.rs +44 -0
- package/src/helpers/handles.rs +191 -0
- package/src/helpers/inspect.rs +18 -0
- package/src/helpers/json_string.rs +58 -0
- package/src/helpers/mod.rs +20 -0
- package/src/helpers/properties.rs +71 -0
- package/src/helpers/try_from_js.rs +213 -0
- package/src/helpers/try_into_js.rs +129 -0
- package/src/lib.rs +28 -40
- package/src/logs.rs +111 -0
- package/src/metrics.rs +325 -0
- package/src/runtime.rs +409 -498
- package/src/testing.rs +315 -57
- package/src/worker.rs +907 -378
- package/ts/errors.ts +57 -0
- package/ts/index.ts +10 -596
- package/ts/native.ts +496 -0
- package/lib/worker-tuner.d.ts +0 -167
- package/lib/worker-tuner.js.map +0 -1
- package/src/conversions/slot_supplier_bridge.rs +0 -291
- package/src/conversions.rs +0 -618
- package/src/errors.rs +0 -38
- package/src/helpers.rs +0 -297
- package/ts/worker-tuner.ts +0 -193
|
@@ -26,6 +26,7 @@ use temporal_sdk::{ActivityOptions, WfContext};
|
|
|
26
26
|
use temporal_sdk_core_api::{
|
|
27
27
|
Worker as WorkerTrait,
|
|
28
28
|
errors::{CompleteActivityError, PollError},
|
|
29
|
+
worker::PollerBehavior,
|
|
29
30
|
};
|
|
30
31
|
use temporal_sdk_core_protos::{
|
|
31
32
|
DEFAULT_ACTIVITY_TYPE, DEFAULT_WORKFLOW_TYPE, TestHistoryBuilder,
|
|
@@ -146,6 +147,7 @@ async fn heartbeats_report_cancels_only_once() {
|
|
|
146
147
|
Ok(RecordActivityTaskHeartbeatResponse {
|
|
147
148
|
cancel_requested: true,
|
|
148
149
|
activity_paused: false,
|
|
150
|
+
activity_reset: false,
|
|
149
151
|
})
|
|
150
152
|
});
|
|
151
153
|
mock_client
|
|
@@ -272,6 +274,7 @@ async fn activity_cancel_interrupts_poll() {
|
|
|
272
274
|
Ok(RecordActivityTaskHeartbeatResponse {
|
|
273
275
|
cancel_requested: true,
|
|
274
276
|
activity_paused: false,
|
|
277
|
+
activity_reset: false,
|
|
275
278
|
})
|
|
276
279
|
}
|
|
277
280
|
.boxed()
|
|
@@ -393,11 +396,13 @@ async fn many_concurrent_heartbeat_cancels() {
|
|
|
393
396
|
Ok(RecordActivityTaskHeartbeatResponse {
|
|
394
397
|
cancel_requested: false,
|
|
395
398
|
activity_paused: false,
|
|
399
|
+
activity_reset: false,
|
|
396
400
|
})
|
|
397
401
|
} else {
|
|
398
402
|
Ok(RecordActivityTaskHeartbeatResponse {
|
|
399
403
|
cancel_requested: true,
|
|
400
404
|
activity_paused: false,
|
|
405
|
+
activity_reset: false,
|
|
401
406
|
})
|
|
402
407
|
}
|
|
403
408
|
}
|
|
@@ -408,7 +413,7 @@ async fn many_concurrent_heartbeat_cancels() {
|
|
|
408
413
|
test_worker_cfg()
|
|
409
414
|
.max_outstanding_activities(CONCURRENCY_NUM)
|
|
410
415
|
// Only 1 poll at a time to avoid over-polling and running out of responses
|
|
411
|
-
.
|
|
416
|
+
.activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize))
|
|
412
417
|
.build()
|
|
413
418
|
.unwrap(),
|
|
414
419
|
mock_client,
|
|
@@ -519,6 +524,7 @@ async fn can_heartbeat_acts_during_shutdown() {
|
|
|
519
524
|
Ok(RecordActivityTaskHeartbeatResponse {
|
|
520
525
|
cancel_requested: false,
|
|
521
526
|
activity_paused: false,
|
|
527
|
+
activity_reset: false,
|
|
522
528
|
})
|
|
523
529
|
});
|
|
524
530
|
mock_client
|
|
@@ -573,6 +579,7 @@ async fn complete_act_with_fail_flushes_heartbeat() {
|
|
|
573
579
|
Ok(RecordActivityTaskHeartbeatResponse {
|
|
574
580
|
cancel_requested: false,
|
|
575
581
|
activity_paused: false,
|
|
582
|
+
activity_reset: false,
|
|
576
583
|
})
|
|
577
584
|
});
|
|
578
585
|
mock_client
|
|
@@ -618,8 +625,8 @@ async fn max_tq_acts_set_passed_to_poll_properly() {
|
|
|
618
625
|
let mut mock_client = mock_workflow_client();
|
|
619
626
|
mock_client
|
|
620
627
|
.expect_poll_activity_task()
|
|
621
|
-
.returning(move |_,
|
|
622
|
-
assert_eq!(
|
|
628
|
+
.returning(move |_, ao| {
|
|
629
|
+
assert_eq!(ao.max_tasks_per_sec, Some(rate));
|
|
623
630
|
Ok(PollActivityTaskQueueResponse {
|
|
624
631
|
task_token: vec![1],
|
|
625
632
|
..Default::default()
|
|
@@ -627,7 +634,7 @@ async fn max_tq_acts_set_passed_to_poll_properly() {
|
|
|
627
634
|
});
|
|
628
635
|
|
|
629
636
|
let cfg = test_worker_cfg()
|
|
630
|
-
.
|
|
637
|
+
.activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize))
|
|
631
638
|
.max_task_queue_activities_per_second(rate)
|
|
632
639
|
.build()
|
|
633
640
|
.unwrap();
|
|
@@ -1081,7 +1088,7 @@ async fn graceful_shutdown(#[values(true, false)] at_max_outstanding: bool) {
|
|
|
1081
1088
|
config: test_worker_cfg()
|
|
1082
1089
|
.graceful_shutdown_period(grace_period)
|
|
1083
1090
|
.max_outstanding_activities(max_outstanding)
|
|
1084
|
-
.
|
|
1091
|
+
.activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize)) // Makes test logic simple
|
|
1085
1092
|
.build()
|
|
1086
1093
|
.unwrap(),
|
|
1087
1094
|
..Default::default()
|
|
@@ -1104,8 +1111,9 @@ async fn graceful_shutdown(#[values(true, false)] at_max_outstanding: bool) {
|
|
|
1104
1111
|
assert_matches!(
|
|
1105
1112
|
cancel.variant,
|
|
1106
1113
|
Some(activity_task::Variant::Cancel(Cancel {
|
|
1107
|
-
reason
|
|
1108
|
-
|
|
1114
|
+
reason,
|
|
1115
|
+
details
|
|
1116
|
+
})) if reason == ActivityCancelReason::WorkerShutdown as i32 && details.as_ref().is_some_and(|d| d.is_worker_shutdown)
|
|
1109
1117
|
);
|
|
1110
1118
|
seen_tts.insert(cancel.task_token);
|
|
1111
1119
|
}
|
|
@@ -1165,7 +1173,7 @@ async fn activities_must_be_flushed_to_server_on_shutdown(#[values(true, false)]
|
|
|
1165
1173
|
act_poller: Some(Box::from(mock_act_poller)),
|
|
1166
1174
|
config: test_worker_cfg()
|
|
1167
1175
|
.graceful_shutdown_period(grace_period)
|
|
1168
|
-
.
|
|
1176
|
+
.activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize)) // Makes test logic simple
|
|
1169
1177
|
.build()
|
|
1170
1178
|
.unwrap(),
|
|
1171
1179
|
..Default::default()
|
|
@@ -1240,3 +1248,167 @@ async fn pass_activity_summary_to_metadata() {
|
|
|
1240
1248
|
.unwrap();
|
|
1241
1249
|
worker.run_until_done().await.unwrap();
|
|
1242
1250
|
}
|
|
1251
|
+
|
|
1252
|
+
#[tokio::test]
|
|
1253
|
+
async fn heartbeat_response_can_be_paused() {
|
|
1254
|
+
let mut mock_client = mock_workflow_client();
|
|
1255
|
+
// First heartbeat returns pause only
|
|
1256
|
+
mock_client
|
|
1257
|
+
.expect_record_activity_heartbeat()
|
|
1258
|
+
.times(1)
|
|
1259
|
+
.returning(|_, _| {
|
|
1260
|
+
Ok(RecordActivityTaskHeartbeatResponse {
|
|
1261
|
+
cancel_requested: false,
|
|
1262
|
+
activity_paused: true,
|
|
1263
|
+
activity_reset: false,
|
|
1264
|
+
})
|
|
1265
|
+
});
|
|
1266
|
+
// Second heartbeat returns cancel only
|
|
1267
|
+
mock_client
|
|
1268
|
+
.expect_record_activity_heartbeat()
|
|
1269
|
+
.times(1)
|
|
1270
|
+
.returning(|_, _| {
|
|
1271
|
+
Ok(RecordActivityTaskHeartbeatResponse {
|
|
1272
|
+
cancel_requested: true,
|
|
1273
|
+
activity_paused: false,
|
|
1274
|
+
activity_reset: false,
|
|
1275
|
+
})
|
|
1276
|
+
});
|
|
1277
|
+
// Third heartbeat does all 3
|
|
1278
|
+
mock_client
|
|
1279
|
+
.expect_record_activity_heartbeat()
|
|
1280
|
+
.times(1)
|
|
1281
|
+
.returning(|_, _| {
|
|
1282
|
+
Ok(RecordActivityTaskHeartbeatResponse {
|
|
1283
|
+
cancel_requested: true,
|
|
1284
|
+
activity_paused: true,
|
|
1285
|
+
activity_reset: true,
|
|
1286
|
+
})
|
|
1287
|
+
});
|
|
1288
|
+
mock_client
|
|
1289
|
+
.expect_cancel_activity_task()
|
|
1290
|
+
.times(3)
|
|
1291
|
+
.returning(|_, _| Ok(RespondActivityTaskCanceledResponse::default()));
|
|
1292
|
+
|
|
1293
|
+
let core = mock_worker(MocksHolder::from_client_with_activities(
|
|
1294
|
+
mock_client,
|
|
1295
|
+
[
|
|
1296
|
+
PollActivityTaskQueueResponse {
|
|
1297
|
+
task_token: vec![1],
|
|
1298
|
+
activity_id: "act1".to_string(),
|
|
1299
|
+
heartbeat_timeout: Some(prost_dur!(from_millis(1))),
|
|
1300
|
+
..Default::default()
|
|
1301
|
+
}
|
|
1302
|
+
.into(),
|
|
1303
|
+
PollActivityTaskQueueResponse {
|
|
1304
|
+
task_token: vec![2],
|
|
1305
|
+
activity_id: "act2".to_string(),
|
|
1306
|
+
heartbeat_timeout: Some(prost_dur!(from_millis(1))),
|
|
1307
|
+
..Default::default()
|
|
1308
|
+
}
|
|
1309
|
+
.into(),
|
|
1310
|
+
PollActivityTaskQueueResponse {
|
|
1311
|
+
task_token: vec![3],
|
|
1312
|
+
activity_id: "act3".to_string(),
|
|
1313
|
+
heartbeat_timeout: Some(prost_dur!(from_millis(1))),
|
|
1314
|
+
..Default::default()
|
|
1315
|
+
}
|
|
1316
|
+
.into(),
|
|
1317
|
+
],
|
|
1318
|
+
));
|
|
1319
|
+
|
|
1320
|
+
// The general testing pattern for each of these cases is:
|
|
1321
|
+
// 1. Poll for activity task
|
|
1322
|
+
// 2. Record activity heartbeat, get mocked heartbeat response
|
|
1323
|
+
// 3. Sleep for 10ms (waiting for heartbeat request to be flushed)
|
|
1324
|
+
// (i.e. sleep enough for the heartbeat flush interval to have elapsed)
|
|
1325
|
+
// 4. Poll for activity task.
|
|
1326
|
+
// We expect a cancellation activity task as they are prioritized (i.e. ordered before)
|
|
1327
|
+
// regular activity tasks.
|
|
1328
|
+
// 5. Assert that the received activity task is indeed a cancellation, with the reason
|
|
1329
|
+
// and details we expect.
|
|
1330
|
+
// 6. Complete the activity with a cancellation result.
|
|
1331
|
+
//
|
|
1332
|
+
// Repeat for subsequent test case(s).
|
|
1333
|
+
|
|
1334
|
+
// Test pause only
|
|
1335
|
+
let act = core.poll_activity_task().await.unwrap();
|
|
1336
|
+
core.record_activity_heartbeat(ActivityHeartbeat {
|
|
1337
|
+
task_token: act.task_token.clone(),
|
|
1338
|
+
details: vec![vec![1_u8, 2, 3].into()],
|
|
1339
|
+
});
|
|
1340
|
+
sleep(Duration::from_millis(10)).await;
|
|
1341
|
+
let act = core.poll_activity_task().await.unwrap();
|
|
1342
|
+
assert_matches!(
|
|
1343
|
+
&act,
|
|
1344
|
+
ActivityTask {
|
|
1345
|
+
task_token,
|
|
1346
|
+
variant: Some(activity_task::Variant::Cancel(Cancel { reason, details })),
|
|
1347
|
+
} if
|
|
1348
|
+
task_token == &vec![1] &&
|
|
1349
|
+
*reason == ActivityCancelReason::Paused as i32 &&
|
|
1350
|
+
details.as_ref().is_some_and(|d| d.is_paused) &&
|
|
1351
|
+
details.as_ref().is_some_and(|d| !d.is_cancelled)
|
|
1352
|
+
);
|
|
1353
|
+
core.complete_activity_task(ActivityTaskCompletion {
|
|
1354
|
+
task_token: act.task_token,
|
|
1355
|
+
result: Some(ActivityExecutionResult::cancel_from_details(None)),
|
|
1356
|
+
})
|
|
1357
|
+
.await
|
|
1358
|
+
.unwrap();
|
|
1359
|
+
|
|
1360
|
+
// Test cancel only
|
|
1361
|
+
let act = core.poll_activity_task().await.unwrap();
|
|
1362
|
+
core.record_activity_heartbeat(ActivityHeartbeat {
|
|
1363
|
+
task_token: act.task_token.clone(),
|
|
1364
|
+
details: vec![vec![1_u8, 2, 3].into()],
|
|
1365
|
+
});
|
|
1366
|
+
sleep(Duration::from_millis(10)).await;
|
|
1367
|
+
let act = core.poll_activity_task().await.unwrap();
|
|
1368
|
+
assert_matches!(
|
|
1369
|
+
&act,
|
|
1370
|
+
ActivityTask {
|
|
1371
|
+
task_token,
|
|
1372
|
+
variant: Some(activity_task::Variant::Cancel(Cancel { reason, details })),
|
|
1373
|
+
} if
|
|
1374
|
+
task_token == &vec![2] &&
|
|
1375
|
+
*reason == ActivityCancelReason::Cancelled as i32 &&
|
|
1376
|
+
details.as_ref().is_some_and(|d| !d.is_paused) &&
|
|
1377
|
+
details.as_ref().is_some_and(|d| d.is_cancelled)
|
|
1378
|
+
);
|
|
1379
|
+
core.complete_activity_task(ActivityTaskCompletion {
|
|
1380
|
+
task_token: act.task_token,
|
|
1381
|
+
result: Some(ActivityExecutionResult::cancel_from_details(None)),
|
|
1382
|
+
})
|
|
1383
|
+
.await
|
|
1384
|
+
.unwrap();
|
|
1385
|
+
|
|
1386
|
+
// Test both pause and cancel (should prioritize cancel)
|
|
1387
|
+
let act = core.poll_activity_task().await.unwrap();
|
|
1388
|
+
core.record_activity_heartbeat(ActivityHeartbeat {
|
|
1389
|
+
task_token: act.task_token.clone(),
|
|
1390
|
+
details: vec![vec![1_u8, 2, 3].into()],
|
|
1391
|
+
});
|
|
1392
|
+
sleep(Duration::from_millis(10)).await;
|
|
1393
|
+
let act = core.poll_activity_task().await.unwrap();
|
|
1394
|
+
assert_matches!(
|
|
1395
|
+
&act,
|
|
1396
|
+
ActivityTask {
|
|
1397
|
+
task_token,
|
|
1398
|
+
variant: Some(activity_task::Variant::Cancel(Cancel { reason, details })),
|
|
1399
|
+
} if
|
|
1400
|
+
task_token == &vec![3] &&
|
|
1401
|
+
*reason == ActivityCancelReason::Cancelled as i32 &&
|
|
1402
|
+
details.as_ref().is_some_and(|d| d.is_paused) &&
|
|
1403
|
+
details.as_ref().is_some_and(|d| d.is_cancelled) &&
|
|
1404
|
+
details.as_ref().is_some_and(|d| d.is_reset)
|
|
1405
|
+
);
|
|
1406
|
+
core.complete_activity_task(ActivityTaskCompletion {
|
|
1407
|
+
task_token: act.task_token,
|
|
1408
|
+
result: Some(ActivityExecutionResult::cancel_from_details(None)),
|
|
1409
|
+
})
|
|
1410
|
+
.await
|
|
1411
|
+
.unwrap();
|
|
1412
|
+
|
|
1413
|
+
core.drain_activity_poller_and_shutdown().await;
|
|
1414
|
+
}
|
|
@@ -17,7 +17,7 @@ use crate::{
|
|
|
17
17
|
};
|
|
18
18
|
use futures_util::FutureExt;
|
|
19
19
|
use std::{sync::LazyLock, time::Duration};
|
|
20
|
-
use temporal_sdk_core_api::Worker as WorkerTrait;
|
|
20
|
+
use temporal_sdk_core_api::{Worker as WorkerTrait, worker::PollerBehavior};
|
|
21
21
|
use temporal_sdk_core_protos::coresdk::workflow_completion::WorkflowActivationCompletion;
|
|
22
22
|
use tokio::{sync::Barrier, time::sleep};
|
|
23
23
|
|
|
@@ -64,7 +64,7 @@ async fn shutdown_interrupts_both_polls() {
|
|
|
64
64
|
mock_client
|
|
65
65
|
.expect_poll_workflow_task()
|
|
66
66
|
.times(1)
|
|
67
|
-
.returning(move |_| {
|
|
67
|
+
.returning(move |_, _| {
|
|
68
68
|
async move {
|
|
69
69
|
BARR.wait().await;
|
|
70
70
|
sleep(Duration::from_secs(1)).await;
|
|
@@ -76,8 +76,8 @@ async fn shutdown_interrupts_both_polls() {
|
|
|
76
76
|
let worker = Worker::new_test(
|
|
77
77
|
test_worker_cfg()
|
|
78
78
|
// Need only 1 concurrent pollers for mock expectations to work here
|
|
79
|
-
.
|
|
80
|
-
.
|
|
79
|
+
.workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize))
|
|
80
|
+
.activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize))
|
|
81
81
|
.build()
|
|
82
82
|
.unwrap(),
|
|
83
83
|
mock_client,
|
|
@@ -10,7 +10,7 @@ use std::{
|
|
|
10
10
|
collections::{HashMap, VecDeque},
|
|
11
11
|
time::Duration,
|
|
12
12
|
};
|
|
13
|
-
use temporal_sdk_core_api::Worker as WorkerTrait;
|
|
13
|
+
use temporal_sdk_core_api::{Worker as WorkerTrait, worker::WorkerVersioningStrategy};
|
|
14
14
|
use temporal_sdk_core_protos::{
|
|
15
15
|
TestHistoryBuilder,
|
|
16
16
|
coresdk::{
|
|
@@ -881,17 +881,31 @@ async fn build_id_set_properly_on_query_on_first_task() {
|
|
|
881
881
|
let mut mock = build_mock_pollers(mh);
|
|
882
882
|
mock.worker_cfg(|wc| {
|
|
883
883
|
wc.max_cached_workflows = 10;
|
|
884
|
-
wc.
|
|
884
|
+
wc.versioning_strategy = WorkerVersioningStrategy::None {
|
|
885
|
+
build_id: "1.0".to_owned(),
|
|
886
|
+
}
|
|
885
887
|
});
|
|
886
888
|
let core = mock_worker(mock);
|
|
887
889
|
|
|
888
890
|
let task = core.poll_workflow_activation().await.unwrap();
|
|
889
|
-
assert_eq!(
|
|
891
|
+
assert_eq!(
|
|
892
|
+
task.deployment_version_for_current_task
|
|
893
|
+
.as_ref()
|
|
894
|
+
.unwrap()
|
|
895
|
+
.build_id,
|
|
896
|
+
"1.0"
|
|
897
|
+
);
|
|
890
898
|
core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
|
|
891
899
|
.await
|
|
892
900
|
.unwrap();
|
|
893
901
|
let task = core.poll_workflow_activation().await.unwrap();
|
|
894
|
-
assert_eq!(
|
|
902
|
+
assert_eq!(
|
|
903
|
+
task.deployment_version_for_current_task
|
|
904
|
+
.as_ref()
|
|
905
|
+
.unwrap()
|
|
906
|
+
.build_id,
|
|
907
|
+
"1.0"
|
|
908
|
+
);
|
|
895
909
|
core.complete_workflow_activation(WorkflowActivationCompletion::empty(task.run_id))
|
|
896
910
|
.await
|
|
897
911
|
.unwrap();
|
|
@@ -14,7 +14,7 @@ use crate::{
|
|
|
14
14
|
};
|
|
15
15
|
use futures_util::{stream, stream::StreamExt};
|
|
16
16
|
use std::{cell::RefCell, time::Duration};
|
|
17
|
-
use temporal_sdk_core_api::Worker;
|
|
17
|
+
use temporal_sdk_core_api::{Worker, worker::PollerBehavior};
|
|
18
18
|
use temporal_sdk_core_protos::{
|
|
19
19
|
coresdk::{
|
|
20
20
|
workflow_activation::workflow_activation_job,
|
|
@@ -269,11 +269,11 @@ async fn worker_can_shutdown_after_never_polling_ok(#[values(true, false)] poll_
|
|
|
269
269
|
.returning(|_, _| Err(tonic::Status::permission_denied("you shall not pass")));
|
|
270
270
|
if poll_workflow {
|
|
271
271
|
mock.expect_poll_workflow_task()
|
|
272
|
-
.returning(|_| Err(tonic::Status::permission_denied("you shall not pass")));
|
|
272
|
+
.returning(|_, _| Err(tonic::Status::permission_denied("you shall not pass")));
|
|
273
273
|
}
|
|
274
274
|
let core = worker::Worker::new_test(
|
|
275
275
|
test_worker_cfg()
|
|
276
|
-
.
|
|
276
|
+
.activity_task_poller_behavior(PollerBehavior::SimpleMaximum(1_usize))
|
|
277
277
|
.build()
|
|
278
278
|
.unwrap(),
|
|
279
279
|
mock,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
use crate::{
|
|
2
|
-
Worker, advance_fut,
|
|
2
|
+
PollWorkflowOptions, Worker, advance_fut,
|
|
3
3
|
internal_flags::CoreInternalFlags,
|
|
4
4
|
job_assert,
|
|
5
5
|
replay::TestHistoryBuilder,
|
|
@@ -34,8 +34,8 @@ use temporal_sdk_core_api::{
|
|
|
34
34
|
Worker as WorkerTrait,
|
|
35
35
|
errors::PollError,
|
|
36
36
|
worker::{
|
|
37
|
-
SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext,
|
|
38
|
-
SlotSupplierPermit, WorkflowSlotKind,
|
|
37
|
+
PollerBehavior, SlotMarkUsedContext, SlotReleaseContext, SlotReservationContext,
|
|
38
|
+
SlotSupplier, SlotSupplierPermit, WorkerVersioningStrategy, WorkflowSlotKind,
|
|
39
39
|
},
|
|
40
40
|
};
|
|
41
41
|
use temporal_sdk_core_protos::{
|
|
@@ -2056,15 +2056,17 @@ async fn no_race_acquiring_permits() {
|
|
|
2056
2056
|
// We need to allow two polls to happen by triggering two processing events in the workflow
|
|
2057
2057
|
// stream, but then delivering the actual tasks after that
|
|
2058
2058
|
let task_barr: &'static Barrier = Box::leak(Box::new(Barrier::new(2)));
|
|
2059
|
-
mock_client
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2059
|
+
mock_client
|
|
2060
|
+
.expect_poll_workflow_task()
|
|
2061
|
+
.returning(move |_, _| {
|
|
2062
|
+
let t = canned_histories::single_timer("1");
|
|
2063
|
+
let poll_resp = hist_to_poll_resp(&t, wfid.to_owned(), 2.into()).resp;
|
|
2064
|
+
async move {
|
|
2065
|
+
task_barr.wait().await;
|
|
2066
|
+
Ok(poll_resp.clone())
|
|
2067
|
+
}
|
|
2068
|
+
.boxed()
|
|
2069
|
+
});
|
|
2068
2070
|
mock_client
|
|
2069
2071
|
.expect_complete_workflow_task()
|
|
2070
2072
|
.returning(|_| async move { Ok(Default::default()) }.boxed());
|
|
@@ -2780,7 +2782,7 @@ async fn poller_wont_run_ahead_of_task_slots() {
|
|
|
2780
2782
|
let mut mock_client = mock_workflow_client();
|
|
2781
2783
|
mock_client
|
|
2782
2784
|
.expect_poll_workflow_task()
|
|
2783
|
-
.returning(move |_| Ok(bunch_of_first_tasks.next().unwrap()));
|
|
2785
|
+
.returning(move |_, _| Ok(bunch_of_first_tasks.next().unwrap()));
|
|
2784
2786
|
mock_client
|
|
2785
2787
|
.expect_complete_workflow_task()
|
|
2786
2788
|
.returning(|_| Ok(Default::default()));
|
|
@@ -2789,7 +2791,7 @@ async fn poller_wont_run_ahead_of_task_slots() {
|
|
|
2789
2791
|
test_worker_cfg()
|
|
2790
2792
|
.max_cached_workflows(10_usize)
|
|
2791
2793
|
.max_outstanding_workflow_tasks(10_usize)
|
|
2792
|
-
.
|
|
2794
|
+
.workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(10_usize))
|
|
2793
2795
|
.no_remote_activities(true)
|
|
2794
2796
|
.build()
|
|
2795
2797
|
.unwrap(),
|
|
@@ -2841,10 +2843,12 @@ async fn poller_wont_poll_until_lang_polls() {
|
|
|
2841
2843
|
// the WFT stream, we'll never join the tasks running the pollers and thus the error
|
|
2842
2844
|
// gets printed but doesn't bubble up to the test. So we set this explicit expectation
|
|
2843
2845
|
// in here to ensure it isn't called.
|
|
2844
|
-
mock_client
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2846
|
+
mock_client
|
|
2847
|
+
.expect_poll_workflow_task()
|
|
2848
|
+
.returning(move |_, _| {
|
|
2849
|
+
let _ = tx.send(());
|
|
2850
|
+
Ok(Default::default())
|
|
2851
|
+
});
|
|
2848
2852
|
|
|
2849
2853
|
let worker = Worker::new_test(
|
|
2850
2854
|
test_worker_cfg()
|
|
@@ -2945,6 +2949,7 @@ async fn use_compatible_version_flag(
|
|
|
2945
2949
|
worker.shutdown().await;
|
|
2946
2950
|
}
|
|
2947
2951
|
|
|
2952
|
+
#[allow(deprecated)]
|
|
2948
2953
|
#[tokio::test]
|
|
2949
2954
|
async fn sets_build_id_from_wft_complete() {
|
|
2950
2955
|
let wfid = "fake_wf_id";
|
|
@@ -2973,21 +2978,32 @@ async fn sets_build_id_from_wft_complete() {
|
|
|
2973
2978
|
let mut worker = mock_sdk_cfg(
|
|
2974
2979
|
MockPollCfg::from_resp_batches(wfid, t, [ResponseType::AllHistory], mock),
|
|
2975
2980
|
|cfg| {
|
|
2976
|
-
cfg.
|
|
2981
|
+
cfg.versioning_strategy = WorkerVersioningStrategy::None {
|
|
2982
|
+
build_id: "fierce-predator".to_string(),
|
|
2983
|
+
};
|
|
2977
2984
|
cfg.max_cached_workflows = 1;
|
|
2978
2985
|
},
|
|
2979
2986
|
);
|
|
2980
2987
|
|
|
2981
2988
|
worker.register_wf(DEFAULT_WORKFLOW_TYPE, |ctx: WfContext| async move {
|
|
2982
2989
|
// First task, it should be empty, since replaying and nothing in first WFT completed
|
|
2983
|
-
assert_eq!(ctx.
|
|
2990
|
+
assert_eq!(ctx.current_deployment_version(), None);
|
|
2984
2991
|
ctx.timer(Duration::from_secs(1)).await;
|
|
2985
|
-
assert_eq!(
|
|
2992
|
+
assert_eq!(
|
|
2993
|
+
ctx.current_deployment_version().unwrap().build_id,
|
|
2994
|
+
"enchi-cat"
|
|
2995
|
+
);
|
|
2986
2996
|
ctx.timer(Duration::from_secs(1)).await;
|
|
2987
2997
|
// Not replaying at this point, so we should see the worker's build id
|
|
2988
|
-
assert_eq!(
|
|
2998
|
+
assert_eq!(
|
|
2999
|
+
ctx.current_deployment_version().unwrap().build_id,
|
|
3000
|
+
"fierce-predator"
|
|
3001
|
+
);
|
|
2989
3002
|
ctx.timer(Duration::from_secs(1)).await;
|
|
2990
|
-
assert_eq!(
|
|
3003
|
+
assert_eq!(
|
|
3004
|
+
ctx.current_deployment_version().unwrap().build_id,
|
|
3005
|
+
"fierce-predator"
|
|
3006
|
+
);
|
|
2991
3007
|
Ok(().into())
|
|
2992
3008
|
});
|
|
2993
3009
|
worker
|
|
@@ -3013,7 +3029,7 @@ async fn slot_provider_cant_hand_out_more_permits_than_cache_size() {
|
|
|
3013
3029
|
let mut mock_client = mock_workflow_client();
|
|
3014
3030
|
mock_client
|
|
3015
3031
|
.expect_poll_workflow_task()
|
|
3016
|
-
.returning(move |_| Ok(bunch_of_first_tasks.next().unwrap()));
|
|
3032
|
+
.returning(move |_, _| Ok(bunch_of_first_tasks.next().unwrap()));
|
|
3017
3033
|
mock_client
|
|
3018
3034
|
.expect_complete_workflow_task()
|
|
3019
3035
|
.returning(|_| Ok(Default::default()));
|
|
@@ -3043,7 +3059,7 @@ async fn slot_provider_cant_hand_out_more_permits_than_cache_size() {
|
|
|
3043
3059
|
.workflow_slot_supplier(Arc::new(EndlessSupplier {}))
|
|
3044
3060
|
.build(),
|
|
3045
3061
|
))
|
|
3046
|
-
.
|
|
3062
|
+
.workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(10_usize))
|
|
3047
3063
|
.no_remote_activities(true)
|
|
3048
3064
|
.build()
|
|
3049
3065
|
.unwrap(),
|
|
@@ -3134,3 +3150,153 @@ async fn pass_timer_summary_to_metadata() {
|
|
|
3134
3150
|
.unwrap();
|
|
3135
3151
|
worker.run_until_done().await.unwrap();
|
|
3136
3152
|
}
|
|
3153
|
+
|
|
3154
|
+
#[tokio::test]
|
|
3155
|
+
async fn both_normal_and_sticky_pollers_poll_concurrently() {
|
|
3156
|
+
struct Counters {
|
|
3157
|
+
// How many time PollWorkflowTaskQueue has been called
|
|
3158
|
+
normal_poll_count: AtomicUsize,
|
|
3159
|
+
sticky_poll_count: AtomicUsize,
|
|
3160
|
+
|
|
3161
|
+
// How many pollers are currently active (i.e. PollWorkflowTaskQueue
|
|
3162
|
+
// has been called, but not the corresponding CompleteWorkflowTask)
|
|
3163
|
+
normal_slots_active_count: AtomicUsize,
|
|
3164
|
+
sticky_slots_active_count: AtomicUsize,
|
|
3165
|
+
|
|
3166
|
+
// Max number of pollers that were active at the same time
|
|
3167
|
+
max_total_slots_active_count: AtomicUsize,
|
|
3168
|
+
max_normal_slots_active_count: AtomicUsize,
|
|
3169
|
+
max_sticky_slots_active_count: AtomicUsize,
|
|
3170
|
+
}
|
|
3171
|
+
|
|
3172
|
+
let counters = Arc::new(Counters {
|
|
3173
|
+
normal_poll_count: AtomicUsize::new(0),
|
|
3174
|
+
sticky_poll_count: AtomicUsize::new(0),
|
|
3175
|
+
normal_slots_active_count: AtomicUsize::new(0),
|
|
3176
|
+
sticky_slots_active_count: AtomicUsize::new(0),
|
|
3177
|
+
max_total_slots_active_count: AtomicUsize::new(0),
|
|
3178
|
+
max_normal_slots_active_count: AtomicUsize::new(0),
|
|
3179
|
+
max_sticky_slots_active_count: AtomicUsize::new(0),
|
|
3180
|
+
});
|
|
3181
|
+
|
|
3182
|
+
// Create actual workflow task responses to return from polls
|
|
3183
|
+
let mut task_responses = (1..100).map(|i| {
|
|
3184
|
+
hist_to_poll_resp(
|
|
3185
|
+
&canned_histories::single_timer(&format!("timer-{i}")),
|
|
3186
|
+
format!("wf-{i}"),
|
|
3187
|
+
1.into(),
|
|
3188
|
+
)
|
|
3189
|
+
.resp
|
|
3190
|
+
});
|
|
3191
|
+
|
|
3192
|
+
let mut mock_client = mock_workflow_client();
|
|
3193
|
+
|
|
3194
|
+
// Track normal vs sticky poll requests and return actual workflow tasks
|
|
3195
|
+
let cc = Arc::clone(&counters);
|
|
3196
|
+
mock_client
|
|
3197
|
+
.expect_poll_workflow_task()
|
|
3198
|
+
.returning(move |_, opts: PollWorkflowOptions| {
|
|
3199
|
+
let mut task_response = task_responses.next().unwrap_or_default();
|
|
3200
|
+
|
|
3201
|
+
// FIXME: Atomics initially made sense, but this has grown ugly, and there's probably
|
|
3202
|
+
// cases where this may produce incorrect results due to race in operation ordering
|
|
3203
|
+
// (really didn't put any thought into this). We also can't have
|
|
3204
|
+
if opts.sticky_queue_name.is_none() {
|
|
3205
|
+
// Normal queue poll
|
|
3206
|
+
cc.normal_poll_count.fetch_add(1, Ordering::Relaxed);
|
|
3207
|
+
cc.normal_slots_active_count.fetch_add(1, Ordering::Relaxed);
|
|
3208
|
+
cc.max_normal_slots_active_count.fetch_max(
|
|
3209
|
+
cc.normal_slots_active_count.load(Ordering::Relaxed),
|
|
3210
|
+
Ordering::AcqRel,
|
|
3211
|
+
);
|
|
3212
|
+
cc.max_total_slots_active_count.fetch_max(
|
|
3213
|
+
cc.normal_slots_active_count.load(Ordering::Relaxed)
|
|
3214
|
+
+ cc.sticky_slots_active_count.load(Ordering::Relaxed),
|
|
3215
|
+
Ordering::AcqRel,
|
|
3216
|
+
);
|
|
3217
|
+
|
|
3218
|
+
task_response.task_token = [task_response.task_token, b"normal".to_vec()].concat();
|
|
3219
|
+
} else {
|
|
3220
|
+
// Sticky queue poll
|
|
3221
|
+
cc.sticky_poll_count.fetch_add(1, Ordering::Relaxed);
|
|
3222
|
+
cc.sticky_slots_active_count.fetch_add(1, Ordering::Relaxed);
|
|
3223
|
+
cc.max_sticky_slots_active_count.fetch_max(
|
|
3224
|
+
cc.sticky_slots_active_count.load(Ordering::Acquire),
|
|
3225
|
+
Ordering::AcqRel,
|
|
3226
|
+
);
|
|
3227
|
+
cc.max_total_slots_active_count.fetch_max(
|
|
3228
|
+
cc.normal_slots_active_count.load(Ordering::Relaxed)
|
|
3229
|
+
+ cc.sticky_slots_active_count.load(Ordering::Relaxed),
|
|
3230
|
+
Ordering::AcqRel,
|
|
3231
|
+
);
|
|
3232
|
+
|
|
3233
|
+
task_response.task_token = [task_response.task_token, b"sticky".to_vec()].concat();
|
|
3234
|
+
}
|
|
3235
|
+
|
|
3236
|
+
// Return actual workflow task responses
|
|
3237
|
+
Ok(task_response)
|
|
3238
|
+
});
|
|
3239
|
+
|
|
3240
|
+
let cc = Arc::clone(&counters);
|
|
3241
|
+
mock_client
|
|
3242
|
+
.expect_complete_workflow_task()
|
|
3243
|
+
.returning(move |completion| {
|
|
3244
|
+
if completion.task_token.0.ends_with(b"normal") {
|
|
3245
|
+
cc.normal_slots_active_count.fetch_sub(1, Ordering::Relaxed);
|
|
3246
|
+
} else {
|
|
3247
|
+
cc.sticky_slots_active_count.fetch_sub(1, Ordering::Relaxed);
|
|
3248
|
+
}
|
|
3249
|
+
Ok(Default::default())
|
|
3250
|
+
});
|
|
3251
|
+
|
|
3252
|
+
let worker = Worker::new(
|
|
3253
|
+
test_worker_cfg()
|
|
3254
|
+
.max_cached_workflows(500_usize) // We need cache, but don't want to deal with evictions
|
|
3255
|
+
.max_outstanding_workflow_tasks(2_usize)
|
|
3256
|
+
.workflow_task_poller_behavior(PollerBehavior::SimpleMaximum(2_usize))
|
|
3257
|
+
.nonsticky_to_sticky_poll_ratio(0.2)
|
|
3258
|
+
.no_remote_activities(true)
|
|
3259
|
+
.build()
|
|
3260
|
+
.unwrap(),
|
|
3261
|
+
Some("stickytq".to_string()),
|
|
3262
|
+
Arc::new(mock_client),
|
|
3263
|
+
None,
|
|
3264
|
+
);
|
|
3265
|
+
|
|
3266
|
+
for _ in 1..50 {
|
|
3267
|
+
let activation = worker.poll_workflow_activation().await.unwrap();
|
|
3268
|
+
let _ = worker
|
|
3269
|
+
.complete_workflow_activation(WorkflowActivationCompletion::empty(activation.run_id))
|
|
3270
|
+
.await;
|
|
3271
|
+
}
|
|
3272
|
+
|
|
3273
|
+
assert!(
|
|
3274
|
+
counters.normal_poll_count.load(Ordering::Relaxed) > 0,
|
|
3275
|
+
"Normal poller should have been called at least once"
|
|
3276
|
+
);
|
|
3277
|
+
assert!(
|
|
3278
|
+
counters.sticky_poll_count.load(Ordering::Relaxed) > 0,
|
|
3279
|
+
"Sticky poller should have been called at least once"
|
|
3280
|
+
);
|
|
3281
|
+
assert!(
|
|
3282
|
+
counters
|
|
3283
|
+
.max_normal_slots_active_count
|
|
3284
|
+
.load(Ordering::Relaxed)
|
|
3285
|
+
>= 1,
|
|
3286
|
+
"Normal poller should have been active at least once"
|
|
3287
|
+
);
|
|
3288
|
+
assert!(
|
|
3289
|
+
counters
|
|
3290
|
+
.max_sticky_slots_active_count
|
|
3291
|
+
.load(Ordering::Relaxed)
|
|
3292
|
+
>= 1,
|
|
3293
|
+
"Sticky poller should have been active at least once"
|
|
3294
|
+
);
|
|
3295
|
+
assert_eq!(
|
|
3296
|
+
counters
|
|
3297
|
+
.max_total_slots_active_count
|
|
3298
|
+
.load(Ordering::Relaxed),
|
|
3299
|
+
2,
|
|
3300
|
+
"At peak, there should be exactly 2 pollers active at the same time"
|
|
3301
|
+
);
|
|
3302
|
+
}
|