@temporalio/core-bridge 1.15.0 → 1.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +172 -70
- package/lib/native.d.ts +1 -1
- package/package.json +2 -2
- package/releases/aarch64-apple-darwin/index.node +0 -0
- package/releases/aarch64-unknown-linux-gnu/index.node +0 -0
- package/releases/x86_64-apple-darwin/index.node +0 -0
- package/releases/x86_64-pc-windows-msvc/index.node +0 -0
- package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
- package/sdk-core/.github/workflows/per-pr.yml +6 -6
- package/sdk-core/AGENTS.md +41 -30
- package/sdk-core/Cargo.toml +3 -0
- package/sdk-core/README.md +15 -9
- package/sdk-core/crates/client/Cargo.toml +4 -0
- package/sdk-core/crates/client/README.md +139 -0
- package/sdk-core/crates/client/src/async_activity_handle.rs +297 -0
- package/sdk-core/crates/client/src/callback_based.rs +7 -0
- package/sdk-core/crates/client/src/errors.rs +294 -0
- package/sdk-core/crates/client/src/{raw.rs → grpc.rs} +280 -159
- package/sdk-core/crates/client/src/lib.rs +920 -1326
- package/sdk-core/crates/client/src/metrics.rs +24 -33
- package/sdk-core/crates/client/src/options_structs.rs +457 -0
- package/sdk-core/crates/client/src/replaceable.rs +5 -4
- package/sdk-core/crates/client/src/request_extensions.rs +8 -9
- package/sdk-core/crates/client/src/retry.rs +99 -54
- package/sdk-core/crates/client/src/{worker/mod.rs → worker.rs} +1 -1
- package/sdk-core/crates/client/src/workflow_handle.rs +826 -0
- package/sdk-core/crates/common/Cargo.toml +61 -2
- package/sdk-core/crates/common/build.rs +742 -12
- package/sdk-core/crates/common/protos/api_upstream/.github/workflows/ci.yml +2 -0
- package/sdk-core/crates/common/protos/api_upstream/Makefile +2 -1
- package/sdk-core/crates/common/protos/api_upstream/buf.yaml +0 -3
- package/sdk-core/crates/common/protos/api_upstream/cmd/check-path-conflicts/main.go +137 -0
- package/sdk-core/crates/common/protos/api_upstream/openapi/openapiv2.json +1166 -770
- package/sdk-core/crates/common/protos/api_upstream/openapi/openapiv3.yaml +1243 -750
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/deployment/v1/message.proto +2 -2
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/enums/v1/workflow.proto +4 -3
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/failure/v1/message.proto +1 -0
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/history/v1/message.proto +4 -0
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/namespace/v1/message.proto +6 -0
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/nexus/v1/message.proto +16 -1
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +64 -6
- package/sdk-core/crates/common/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +88 -33
- package/sdk-core/crates/common/protos/local/temporal/sdk/core/nexus/nexus.proto +4 -2
- package/sdk-core/crates/common/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +4 -0
- package/sdk-core/crates/common/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +5 -5
- package/sdk-core/crates/common/src/activity_definition.rs +20 -0
- package/sdk-core/crates/common/src/data_converters.rs +770 -0
- package/sdk-core/crates/common/src/envconfig.rs +5 -0
- package/sdk-core/crates/common/src/lib.rs +15 -211
- package/sdk-core/crates/common/src/payload_visitor.rs +648 -0
- package/sdk-core/crates/common/src/priority.rs +110 -0
- package/sdk-core/crates/common/src/protos/canned_histories.rs +3 -0
- package/sdk-core/crates/common/src/protos/history_builder.rs +45 -0
- package/sdk-core/crates/common/src/protos/history_info.rs +2 -0
- package/sdk-core/crates/common/src/protos/mod.rs +122 -27
- package/sdk-core/crates/common/src/protos/task_token.rs +3 -3
- package/sdk-core/crates/common/src/protos/utilities.rs +11 -0
- package/sdk-core/crates/{sdk-core → common}/src/telemetry/log_export.rs +5 -7
- package/sdk-core/crates/common/src/telemetry/metrics/core.rs +125 -0
- package/sdk-core/crates/common/src/telemetry/metrics.rs +268 -223
- package/sdk-core/crates/{sdk-core → common}/src/telemetry/otel.rs +8 -13
- package/sdk-core/crates/{sdk-core → common}/src/telemetry/prometheus_meter.rs +49 -50
- package/sdk-core/crates/{sdk-core → common}/src/telemetry/prometheus_server.rs +2 -3
- package/sdk-core/crates/common/src/telemetry.rs +264 -4
- package/sdk-core/crates/common/src/worker.rs +68 -603
- package/sdk-core/crates/common/src/workflow_definition.rs +60 -0
- package/sdk-core/crates/macros/Cargo.toml +5 -1
- package/sdk-core/crates/macros/src/activities_definitions.rs +585 -0
- package/sdk-core/crates/macros/src/fsm_impl.rs +507 -0
- package/sdk-core/crates/macros/src/lib.rs +138 -512
- package/sdk-core/crates/macros/src/macro_utils.rs +106 -0
- package/sdk-core/crates/macros/src/workflow_definitions.rs +1224 -0
- package/sdk-core/crates/sdk/Cargo.toml +19 -6
- package/sdk-core/crates/sdk/README.md +415 -0
- package/sdk-core/crates/sdk/src/activities.rs +417 -0
- package/sdk-core/crates/sdk/src/interceptors.rs +1 -1
- package/sdk-core/crates/sdk/src/lib.rs +757 -442
- package/sdk-core/crates/sdk/src/workflow_context/options.rs +45 -35
- package/sdk-core/crates/sdk/src/workflow_context.rs +1033 -289
- package/sdk-core/crates/sdk/src/workflow_future.rs +277 -213
- package/sdk-core/crates/sdk/src/workflows.rs +711 -0
- package/sdk-core/crates/sdk-core/Cargo.toml +57 -64
- package/sdk-core/crates/sdk-core/benches/workflow_replay_bench.rs +41 -35
- package/sdk-core/crates/sdk-core/machine_coverage/ActivityMachine_Coverage.puml +1 -1
- package/sdk-core/crates/sdk-core/src/abstractions.rs +6 -10
- package/sdk-core/crates/sdk-core/src/core_tests/activity_tasks.rs +6 -5
- package/sdk-core/crates/sdk-core/src/core_tests/mod.rs +13 -15
- package/sdk-core/crates/sdk-core/src/core_tests/queries.rs +21 -25
- package/sdk-core/crates/sdk-core/src/core_tests/replay_flag.rs +7 -10
- package/sdk-core/crates/sdk-core/src/core_tests/updates.rs +14 -17
- package/sdk-core/crates/sdk-core/src/core_tests/workers.rs +493 -26
- package/sdk-core/crates/sdk-core/src/core_tests/workflow_tasks.rs +4 -8
- package/sdk-core/crates/sdk-core/src/ephemeral_server/mod.rs +7 -7
- package/sdk-core/crates/sdk-core/src/histfetch.rs +20 -10
- package/sdk-core/crates/sdk-core/src/lib.rs +41 -111
- package/sdk-core/crates/sdk-core/src/pollers/mod.rs +4 -9
- package/sdk-core/crates/sdk-core/src/pollers/poll_buffer.rs +118 -19
- package/sdk-core/crates/sdk-core/src/protosext/mod.rs +2 -2
- package/sdk-core/crates/sdk-core/src/replay/mod.rs +14 -5
- package/sdk-core/crates/sdk-core/src/telemetry/metrics.rs +179 -196
- package/sdk-core/crates/sdk-core/src/telemetry/mod.rs +3 -280
- package/sdk-core/crates/sdk-core/src/test_help/integ_helpers.rs +6 -9
- package/sdk-core/crates/sdk-core/src/test_help/unit_helpers.rs +3 -6
- package/sdk-core/crates/sdk-core/src/worker/activities/local_activities.rs +11 -14
- package/sdk-core/crates/sdk-core/src/worker/activities.rs +16 -19
- package/sdk-core/crates/sdk-core/src/worker/client/mocks.rs +9 -5
- package/sdk-core/crates/sdk-core/src/worker/client.rs +103 -81
- package/sdk-core/crates/sdk-core/src/worker/heartbeat.rs +7 -11
- package/sdk-core/crates/sdk-core/src/worker/mod.rs +1124 -229
- package/sdk-core/crates/sdk-core/src/worker/nexus.rs +145 -23
- package/sdk-core/crates/sdk-core/src/worker/slot_provider.rs +2 -2
- package/sdk-core/crates/sdk-core/src/worker/tuner/fixed_size.rs +2 -2
- package/sdk-core/crates/sdk-core/src/worker/tuner/resource_based.rs +13 -13
- package/sdk-core/crates/sdk-core/src/worker/tuner.rs +28 -8
- package/sdk-core/crates/sdk-core/src/worker/workflow/driven_workflow.rs +9 -3
- package/sdk-core/crates/sdk-core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs +21 -22
- package/sdk-core/crates/sdk-core/src/worker/workflow/machines/workflow_machines.rs +19 -4
- package/sdk-core/crates/sdk-core/src/worker/workflow/managed_run.rs +14 -18
- package/sdk-core/crates/sdk-core/src/worker/workflow/mod.rs +4 -6
- package/sdk-core/crates/sdk-core/src/worker/workflow/run_cache.rs +4 -7
- package/sdk-core/crates/sdk-core/src/worker/workflow/wft_extraction.rs +2 -4
- package/sdk-core/crates/sdk-core/src/worker/workflow/wft_poller.rs +8 -9
- package/sdk-core/crates/sdk-core/src/worker/workflow/workflow_stream.rs +1 -3
- package/sdk-core/crates/sdk-core/tests/activities_procmacro.rs +6 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/basic_pass.rs +54 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/invalid_self_type_fail.rs +18 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/invalid_self_type_fail.stderr +5 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/missing_context_fail.rs +14 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/missing_context_fail.stderr +5 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/multi_arg_pass.rs +48 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/no_input_pass.rs +14 -0
- package/sdk-core/crates/sdk-core/tests/activities_trybuild/no_return_type_pass.rs +19 -0
- package/sdk-core/crates/sdk-core/tests/cloud_tests.rs +14 -5
- package/sdk-core/crates/sdk-core/tests/common/activity_functions.rs +55 -0
- package/sdk-core/crates/sdk-core/tests/common/mod.rs +241 -196
- package/sdk-core/crates/sdk-core/tests/common/workflows.rs +41 -28
- package/sdk-core/crates/sdk-core/tests/global_metric_tests.rs +3 -5
- package/sdk-core/crates/sdk-core/tests/heavy_tests/fuzzy_workflow.rs +73 -64
- package/sdk-core/crates/sdk-core/tests/heavy_tests.rs +298 -252
- package/sdk-core/crates/sdk-core/tests/integ_tests/async_activity_client_tests.rs +230 -0
- package/sdk-core/crates/sdk-core/tests/integ_tests/client_tests.rs +94 -57
- package/sdk-core/crates/sdk-core/tests/integ_tests/data_converter_tests.rs +381 -0
- package/sdk-core/crates/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +16 -12
- package/sdk-core/crates/sdk-core/tests/integ_tests/heartbeat_tests.rs +48 -40
- package/sdk-core/crates/sdk-core/tests/integ_tests/metrics_tests.rs +327 -255
- package/sdk-core/crates/sdk-core/tests/integ_tests/pagination_tests.rs +50 -45
- package/sdk-core/crates/sdk-core/tests/integ_tests/polling_tests.rs +147 -126
- package/sdk-core/crates/sdk-core/tests/integ_tests/queries_tests.rs +103 -89
- package/sdk-core/crates/sdk-core/tests/integ_tests/update_tests.rs +609 -453
- package/sdk-core/crates/sdk-core/tests/integ_tests/visibility_tests.rs +80 -62
- package/sdk-core/crates/sdk-core/tests/integ_tests/worker_heartbeat_tests.rs +360 -231
- package/sdk-core/crates/sdk-core/tests/integ_tests/worker_tests.rs +248 -185
- package/sdk-core/crates/sdk-core/tests/integ_tests/worker_versioning_tests.rs +52 -43
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_client_tests.rs +180 -0
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/activities.rs +428 -315
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/cancel_external.rs +82 -56
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +56 -28
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +364 -243
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/client_interactions.rs +552 -0
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs +101 -42
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +243 -147
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/eager.rs +98 -28
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +1475 -1036
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/modify_wf_properties.rs +73 -41
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/nexus.rs +397 -238
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/patches.rs +414 -189
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/queries.rs +415 -0
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/replay.rs +96 -36
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/resets.rs +154 -137
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/signals.rs +183 -105
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +85 -38
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/timers.rs +142 -40
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +73 -54
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests.rs +363 -226
- package/sdk-core/crates/sdk-core/tests/main.rs +17 -15
- package/sdk-core/crates/sdk-core/tests/manual_tests.rs +207 -152
- package/sdk-core/crates/sdk-core/tests/shared_tests/mod.rs +65 -34
- package/sdk-core/crates/sdk-core/tests/shared_tests/priority.rs +107 -84
- package/sdk-core/crates/sdk-core/tests/workflows_procmacro.rs +6 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/async_query_fail.rs +26 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/async_query_fail.stderr +5 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/basic_pass.rs +49 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/minimal_pass.rs +21 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/mut_query_fail.rs +26 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/mut_query_fail.stderr +5 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/sync_run_fail.rs +21 -0
- package/sdk-core/crates/sdk-core/tests/workflows_trybuild/sync_run_fail.stderr +5 -0
- package/sdk-core/crates/sdk-core-c-bridge/Cargo.toml +7 -1
- package/sdk-core/crates/sdk-core-c-bridge/include/temporal-sdk-core-c-bridge.h +14 -14
- package/sdk-core/crates/sdk-core-c-bridge/src/client.rs +83 -74
- package/sdk-core/crates/sdk-core-c-bridge/src/metric.rs +9 -14
- package/sdk-core/crates/sdk-core-c-bridge/src/runtime.rs +1 -2
- package/sdk-core/crates/sdk-core-c-bridge/src/tests/context.rs +13 -13
- package/sdk-core/crates/sdk-core-c-bridge/src/tests/mod.rs +6 -6
- package/sdk-core/crates/sdk-core-c-bridge/src/tests/utils.rs +3 -4
- package/sdk-core/crates/sdk-core-c-bridge/src/worker.rs +62 -75
- package/sdk-core/rustfmt.toml +2 -1
- package/src/client.rs +205 -318
- package/src/metrics.rs +22 -30
- package/src/runtime.rs +4 -5
- package/src/worker.rs +16 -19
- package/ts/native.ts +1 -1
- package/sdk-core/crates/client/src/workflow_handle/mod.rs +0 -212
- package/sdk-core/crates/common/src/errors.rs +0 -85
- package/sdk-core/crates/common/tests/worker_task_types_test.rs +0 -129
- package/sdk-core/crates/sdk/src/activity_context.rs +0 -238
- package/sdk-core/crates/sdk/src/app_data.rs +0 -37
- package/sdk-core/crates/sdk-core/tests/integ_tests/activity_functions.rs +0 -5
- package/sdk-core/crates/sdk-core/tests/integ_tests/workflow_tests/appdata_propagation.rs +0 -61
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
use crate::common::*;
|
|
2
|
-
use futures_util::StreamExt;
|
|
3
2
|
use std::sync::{
|
|
4
3
|
Arc,
|
|
5
4
|
atomic::{AtomicUsize, Ordering},
|
|
6
5
|
};
|
|
7
|
-
use temporalio_client::WorkflowOptions;
|
|
8
6
|
use temporalio_common::protos::{
|
|
9
7
|
DEFAULT_WORKFLOW_TYPE, TestHistoryBuilder,
|
|
10
8
|
temporal::api::{
|
|
@@ -14,9 +12,31 @@ use temporalio_common::protos::{
|
|
|
14
12
|
workflowservice::v1::GetWorkflowExecutionHistoryResponse,
|
|
15
13
|
},
|
|
16
14
|
};
|
|
17
|
-
use
|
|
15
|
+
use temporalio_macros::{workflow, workflow_methods};
|
|
16
|
+
use temporalio_sdk::{SyncWorkflowContext, WorkflowContext, WorkflowResult};
|
|
18
17
|
use temporalio_sdk_core::test_help::{MockPollCfg, ResponseType, mock_worker_client};
|
|
19
18
|
|
|
19
|
+
#[workflow]
|
|
20
|
+
struct WeirdPaginationWf {
|
|
21
|
+
sig_ctr: Arc<AtomicUsize>,
|
|
22
|
+
signal_count: usize,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
#[workflow_methods(factory_only)]
|
|
26
|
+
impl WeirdPaginationWf {
|
|
27
|
+
#[run(name = DEFAULT_WORKFLOW_TYPE)]
|
|
28
|
+
async fn run(ctx: &mut WorkflowContext<Self>) -> WorkflowResult<()> {
|
|
29
|
+
ctx.wait_condition(|s| s.signal_count >= 2).await;
|
|
30
|
+
Ok(())
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
#[signal(name = "hi")]
|
|
34
|
+
fn hi(&mut self, _ctx: &mut SyncWorkflowContext<Self>, _: ()) {
|
|
35
|
+
self.signal_count += 1;
|
|
36
|
+
self.sig_ctr.fetch_add(1, Ordering::AcqRel);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
20
40
|
#[tokio::test]
|
|
21
41
|
async fn weird_pagination_doesnt_drop_wft_events() {
|
|
22
42
|
let wf_id = "fakeid";
|
|
@@ -97,7 +117,6 @@ async fn weird_pagination_doesnt_drop_wft_events() {
|
|
|
97
117
|
})
|
|
98
118
|
.times(1);
|
|
99
119
|
|
|
100
|
-
let wf_type = DEFAULT_WORKFLOW_TYPE;
|
|
101
120
|
let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::Raw(wft_resp)], mock_client);
|
|
102
121
|
let mut worker = mock_sdk_cfg(mh, |cfg| {
|
|
103
122
|
cfg.max_cached_workflows = 2;
|
|
@@ -106,32 +125,36 @@ async fn weird_pagination_doesnt_drop_wft_events() {
|
|
|
106
125
|
|
|
107
126
|
let sig_ctr = Arc::new(AtomicUsize::new(0));
|
|
108
127
|
let sig_ctr_clone = sig_ctr.clone();
|
|
109
|
-
worker.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
let mut sigchan = ctx.make_signal_channel("hi");
|
|
113
|
-
while sigchan.next().await.is_some() {
|
|
114
|
-
if sig_ctr_clone.fetch_add(1, Ordering::AcqRel) == 1 {
|
|
115
|
-
break;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
Ok(().into())
|
|
119
|
-
}
|
|
128
|
+
worker.register_workflow_with_factory(move || WeirdPaginationWf {
|
|
129
|
+
sig_ctr: sig_ctr_clone.clone(),
|
|
130
|
+
signal_count: 0,
|
|
120
131
|
});
|
|
121
132
|
|
|
122
|
-
worker
|
|
123
|
-
.submit_wf(
|
|
124
|
-
wf_id.to_owned(),
|
|
125
|
-
wf_type.to_owned(),
|
|
126
|
-
vec![],
|
|
127
|
-
WorkflowOptions::default(),
|
|
128
|
-
)
|
|
129
|
-
.await
|
|
130
|
-
.unwrap();
|
|
131
133
|
worker.run_until_done().await.unwrap();
|
|
132
134
|
assert_eq!(sig_ctr.load(Ordering::Acquire), 2);
|
|
133
135
|
}
|
|
134
136
|
|
|
137
|
+
#[workflow]
|
|
138
|
+
struct ExtremePaginationWf {
|
|
139
|
+
sig_ctr: Arc<AtomicUsize>,
|
|
140
|
+
signal_count: usize,
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
#[workflow_methods(factory_only)]
|
|
144
|
+
impl ExtremePaginationWf {
|
|
145
|
+
#[run(name = DEFAULT_WORKFLOW_TYPE)]
|
|
146
|
+
async fn run(ctx: &mut WorkflowContext<Self>) -> WorkflowResult<()> {
|
|
147
|
+
ctx.wait_condition(|s| s.signal_count >= 6).await;
|
|
148
|
+
Ok(())
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
#[signal(name = "hi")]
|
|
152
|
+
fn hi(&mut self, _ctx: &mut SyncWorkflowContext<Self>, _: ()) {
|
|
153
|
+
self.signal_count += 1;
|
|
154
|
+
self.sig_ctr.fetch_add(1, Ordering::AcqRel);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
135
158
|
#[tokio::test]
|
|
136
159
|
async fn extreme_pagination_doesnt_drop_wft_events_worker() {
|
|
137
160
|
let wf_id = "fakeid";
|
|
@@ -237,7 +260,6 @@ async fn extreme_pagination_doesnt_drop_wft_events_worker() {
|
|
|
237
260
|
wft_resp.previous_started_event_id = 3;
|
|
238
261
|
wft_resp.started_event_id = 15;
|
|
239
262
|
|
|
240
|
-
let wf_type = DEFAULT_WORKFLOW_TYPE;
|
|
241
263
|
let mh = MockPollCfg::from_resp_batches(wf_id, t, [ResponseType::Raw(wft_resp)], mock_client);
|
|
242
264
|
let mut worker = mock_sdk_cfg(mh, |cfg| {
|
|
243
265
|
cfg.max_cached_workflows = 2;
|
|
@@ -246,28 +268,11 @@ async fn extreme_pagination_doesnt_drop_wft_events_worker() {
|
|
|
246
268
|
|
|
247
269
|
let sig_ctr = Arc::new(AtomicUsize::new(0));
|
|
248
270
|
let sig_ctr_clone = sig_ctr.clone();
|
|
249
|
-
worker.
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
let mut sigchan = ctx.make_signal_channel("hi");
|
|
253
|
-
while sigchan.next().await.is_some() {
|
|
254
|
-
if sig_ctr_clone.fetch_add(1, Ordering::AcqRel) == 5 {
|
|
255
|
-
break;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
Ok(().into())
|
|
259
|
-
}
|
|
271
|
+
worker.register_workflow_with_factory(move || ExtremePaginationWf {
|
|
272
|
+
sig_ctr: sig_ctr_clone.clone(),
|
|
273
|
+
signal_count: 0,
|
|
260
274
|
});
|
|
261
275
|
|
|
262
|
-
worker
|
|
263
|
-
.submit_wf(
|
|
264
|
-
wf_id.to_owned(),
|
|
265
|
-
wf_type.to_owned(),
|
|
266
|
-
vec![],
|
|
267
|
-
WorkflowOptions::default(),
|
|
268
|
-
)
|
|
269
|
-
.await
|
|
270
|
-
.unwrap();
|
|
271
276
|
worker.run_until_done().await.unwrap();
|
|
272
277
|
assert_eq!(sig_ctr.load(Ordering::Acquire), 6);
|
|
273
278
|
}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
-
use crate::{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
},
|
|
6
|
-
integ_tests::activity_functions::echo,
|
|
1
|
+
use crate::common::{
|
|
2
|
+
CoreWfStarter, INTEG_CLIENT_NAME, INTEG_CLIENT_VERSION, activity_functions::StdActivities,
|
|
3
|
+
get_integ_client, init_core_and_create_wf, init_integ_telem, integ_dev_server_config,
|
|
4
|
+
integ_worker_config,
|
|
7
5
|
};
|
|
8
6
|
use assert_matches::assert_matches;
|
|
9
7
|
use futures_util::{FutureExt, StreamExt, future::join_all};
|
|
@@ -15,12 +13,16 @@ use std::{
|
|
|
15
13
|
},
|
|
16
14
|
time::Duration,
|
|
17
15
|
};
|
|
18
|
-
use temporalio_client::{
|
|
16
|
+
use temporalio_client::{
|
|
17
|
+
Client, Connection, ConnectionOptions, NamespacedClient, UntypedWorkflow,
|
|
18
|
+
WorkflowExecutionInfo, WorkflowStartOptions,
|
|
19
|
+
};
|
|
19
20
|
use temporalio_common::{
|
|
20
|
-
|
|
21
|
+
data_converters::RawValue,
|
|
22
|
+
prost_dur,
|
|
21
23
|
protos::{
|
|
22
24
|
coresdk::{
|
|
23
|
-
|
|
25
|
+
IntoCompletion,
|
|
24
26
|
activity_task::activity_task as act_task,
|
|
25
27
|
workflow_activation::{FireTimer, WorkflowActivationJob, workflow_activation_job},
|
|
26
28
|
workflow_commands::{ActivityCancellationType, RequestCancelActivity, StartTimer},
|
|
@@ -29,15 +31,14 @@ use temporalio_common::{
|
|
|
29
31
|
temporal::api::enums::v1::EventType,
|
|
30
32
|
test_utils::schedule_activity_cmd,
|
|
31
33
|
},
|
|
32
|
-
telemetry::{Logger, TelemetryOptions},
|
|
33
|
-
worker::PollerBehavior,
|
|
34
|
+
telemetry::{CoreLogStreamConsumer, Logger, TelemetryOptions},
|
|
34
35
|
};
|
|
35
|
-
use
|
|
36
|
+
use temporalio_macros::{workflow, workflow_methods};
|
|
37
|
+
use temporalio_sdk::{ActivityOptions, WorkflowContext, WorkflowResult};
|
|
36
38
|
use temporalio_sdk_core::{
|
|
37
|
-
|
|
39
|
+
CoreRuntime, PollerBehavior, RuntimeOptions, TunerHolder,
|
|
38
40
|
ephemeral_server::{TemporalDevServerConfig, default_cached_download},
|
|
39
41
|
init_worker,
|
|
40
|
-
telemetry::CoreLogStreamConsumer,
|
|
41
42
|
test_help::{NAMESPACE, WorkerTestHelpers, drain_pollers_and_shutdown},
|
|
42
43
|
};
|
|
43
44
|
use tokio::{sync::Notify, time::timeout};
|
|
@@ -145,87 +146,99 @@ async fn switching_worker_client_changes_poll() {
|
|
|
145
146
|
let result = std::panic::AssertUnwindSafe(async {
|
|
146
147
|
// Connect clients to both servers
|
|
147
148
|
info!("Connecting clients");
|
|
148
|
-
let
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
149
|
+
let opts1 =
|
|
150
|
+
ConnectionOptions::new(Url::parse(&format!("http://{}", server1.target)).unwrap())
|
|
151
|
+
.identity("integ_tester".to_owned())
|
|
152
|
+
.client_name("temporal-core".to_owned())
|
|
153
|
+
.client_version("0.1.0".to_owned())
|
|
154
|
+
.build();
|
|
155
|
+
let connection1 = Connection::connect(opts1).await.unwrap();
|
|
156
|
+
let client_opts1 = temporalio_client::ClientOptions::new("default").build();
|
|
157
|
+
let client1 = Client::new(connection1, client_opts1).unwrap();
|
|
158
|
+
|
|
159
|
+
let opts2 =
|
|
160
|
+
ConnectionOptions::new(Url::parse(&format!("http://{}", server2.target)).unwrap())
|
|
161
|
+
.identity("integ_tester".to_owned())
|
|
162
|
+
.client_name("temporal-core".to_owned())
|
|
163
|
+
.client_version("0.1.0".to_owned())
|
|
164
|
+
.build();
|
|
165
|
+
let connection2 = Connection::connect(opts2).await.unwrap();
|
|
166
|
+
let client_opts2 = temporalio_client::ClientOptions::new("default").build();
|
|
167
|
+
let client2 = Client::new(connection2, client_opts2).unwrap();
|
|
166
168
|
|
|
167
169
|
// Start a workflow on both servers
|
|
168
170
|
info!("Starting workflows");
|
|
169
171
|
let wf1 = client1
|
|
170
172
|
.start_workflow(
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
"my-workflow-1".to_owned()
|
|
174
|
-
|
|
175
|
-
None,
|
|
176
|
-
WorkflowOptions::default(),
|
|
173
|
+
UntypedWorkflow::new("my-workflow-type"),
|
|
174
|
+
RawValue::default(),
|
|
175
|
+
WorkflowStartOptions::new("my-task-queue".to_owned(), "my-workflow-1".to_owned())
|
|
176
|
+
.build(),
|
|
177
177
|
)
|
|
178
178
|
.await
|
|
179
179
|
.unwrap();
|
|
180
|
+
let wf1_run_id = wf1.run_id().unwrap().to_string();
|
|
180
181
|
let wf2 = client2
|
|
181
182
|
.start_workflow(
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
"my-workflow-2".to_owned()
|
|
185
|
-
|
|
186
|
-
None,
|
|
187
|
-
WorkflowOptions::default(),
|
|
183
|
+
UntypedWorkflow::new("my-workflow-type"),
|
|
184
|
+
RawValue::default(),
|
|
185
|
+
WorkflowStartOptions::new("my-task-queue".to_owned(), "my-workflow-2".to_owned())
|
|
186
|
+
.build(),
|
|
188
187
|
)
|
|
189
188
|
.await
|
|
190
189
|
.unwrap();
|
|
190
|
+
let wf2_run_id = wf2.run_id().unwrap().to_string();
|
|
191
191
|
|
|
192
192
|
// Create a worker only on the first server
|
|
193
193
|
let mut config = integ_worker_config("my-task-queue");
|
|
194
194
|
// We want a cache so we don't get extra remove-job activations
|
|
195
195
|
config.max_cached_workflows = 100_usize;
|
|
196
|
-
let worker = init_worker(
|
|
196
|
+
let worker = init_worker(
|
|
197
|
+
init_integ_telem().unwrap(),
|
|
198
|
+
config,
|
|
199
|
+
client1.connection().clone(),
|
|
200
|
+
)
|
|
201
|
+
.unwrap();
|
|
197
202
|
|
|
198
203
|
// Poll for first task, confirm it's first wf, complete, and wait for complete
|
|
199
204
|
info!("Doing initial poll");
|
|
200
205
|
let act1 = worker.poll_workflow_activation().await.unwrap();
|
|
201
|
-
assert_eq!(
|
|
206
|
+
assert_eq!(wf1_run_id, act1.run_id);
|
|
202
207
|
worker.complete_execution(&act1.run_id).await;
|
|
203
208
|
worker.handle_eviction().await;
|
|
204
209
|
info!("Waiting on first workflow complete");
|
|
205
|
-
|
|
206
|
-
.
|
|
207
|
-
.
|
|
208
|
-
.
|
|
209
|
-
|
|
210
|
+
WorkflowExecutionInfo {
|
|
211
|
+
namespace: client1.namespace(),
|
|
212
|
+
workflow_id: "my-workflow-1".into(),
|
|
213
|
+
run_id: Some(wf1_run_id.clone()),
|
|
214
|
+
first_execution_run_id: None,
|
|
215
|
+
}
|
|
216
|
+
.bind_untyped(client1.clone())
|
|
217
|
+
.get_result(Default::default())
|
|
218
|
+
.await
|
|
219
|
+
.unwrap();
|
|
210
220
|
|
|
211
221
|
// Swap client, poll for next task, confirm it's second wf, and respond w/ empty
|
|
212
222
|
info!("Replacing client and polling again");
|
|
213
|
-
worker
|
|
214
|
-
.replace_client(client2.get_client().inner().clone())
|
|
215
|
-
.unwrap();
|
|
223
|
+
worker.replace_client(client2.connection().clone()).unwrap();
|
|
216
224
|
let act2 = worker.poll_workflow_activation().await.unwrap();
|
|
217
|
-
assert_eq!(
|
|
225
|
+
assert_eq!(wf2_run_id, act2.run_id);
|
|
218
226
|
worker.complete_execution(&act2.run_id).await;
|
|
219
227
|
worker.handle_eviction().await;
|
|
220
228
|
info!("Waiting on second workflow complete");
|
|
221
|
-
|
|
222
|
-
.
|
|
223
|
-
.
|
|
224
|
-
|
|
225
|
-
|
|
229
|
+
WorkflowExecutionInfo {
|
|
230
|
+
namespace: client2.namespace(),
|
|
231
|
+
workflow_id: "my-workflow-2".into(),
|
|
232
|
+
run_id: Some(wf2_run_id),
|
|
233
|
+
first_execution_run_id: None,
|
|
234
|
+
}
|
|
235
|
+
.bind_untyped(client2.clone())
|
|
236
|
+
.get_result(Default::default())
|
|
237
|
+
.await
|
|
238
|
+
.unwrap();
|
|
226
239
|
|
|
227
240
|
// Shutdown workers and servers
|
|
228
|
-
drain_pollers_and_shutdown(&
|
|
241
|
+
drain_pollers_and_shutdown(&worker).await;
|
|
229
242
|
})
|
|
230
243
|
.catch_unwind()
|
|
231
244
|
.await;
|
|
@@ -239,54 +252,65 @@ async fn switching_worker_client_changes_poll() {
|
|
|
239
252
|
}
|
|
240
253
|
}
|
|
241
254
|
|
|
255
|
+
#[workflow]
|
|
256
|
+
#[derive(Default)]
|
|
257
|
+
struct OnlyOneWorkflowSlotAndTwoPollers;
|
|
258
|
+
|
|
259
|
+
#[workflow_methods]
|
|
260
|
+
impl OnlyOneWorkflowSlotAndTwoPollers {
|
|
261
|
+
#[run(name = "only_one_workflow_slot_and_two_pollers")]
|
|
262
|
+
async fn run(ctx: &mut WorkflowContext<Self>) -> WorkflowResult<()> {
|
|
263
|
+
for _ in 0..3 {
|
|
264
|
+
let _ = ctx
|
|
265
|
+
.start_activity(
|
|
266
|
+
StdActivities::echo,
|
|
267
|
+
"hi!".to_string(),
|
|
268
|
+
ActivityOptions {
|
|
269
|
+
start_to_close_timeout: Some(Duration::from_secs(5)),
|
|
270
|
+
..Default::default()
|
|
271
|
+
},
|
|
272
|
+
)
|
|
273
|
+
.await;
|
|
274
|
+
}
|
|
275
|
+
Ok(())
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
242
279
|
#[rstest::rstest]
|
|
243
280
|
#[tokio::test]
|
|
244
281
|
async fn small_workflow_slots_and_pollers(#[values(false, true)] use_autoscaling: bool) {
|
|
245
282
|
let wf_name = "only_one_workflow_slot_and_two_pollers";
|
|
246
283
|
let mut starter = CoreWfStarter::new(wf_name);
|
|
247
284
|
if use_autoscaling {
|
|
248
|
-
starter.
|
|
285
|
+
starter.sdk_config.workflow_task_poller_behavior = PollerBehavior::Autoscaling {
|
|
249
286
|
minimum: 1,
|
|
250
287
|
maximum: 5,
|
|
251
288
|
initial: 1,
|
|
252
289
|
};
|
|
253
290
|
} else {
|
|
254
|
-
starter.
|
|
291
|
+
starter.sdk_config.workflow_task_poller_behavior = PollerBehavior::SimpleMaximum(2);
|
|
255
292
|
}
|
|
256
|
-
starter.
|
|
257
|
-
starter.
|
|
258
|
-
starter.
|
|
259
|
-
starter.worker_config.max_outstanding_activities = Some(1_usize);
|
|
293
|
+
starter.sdk_config.activity_task_poller_behavior = PollerBehavior::SimpleMaximum(1);
|
|
294
|
+
starter.sdk_config.tuner = Arc::new(TunerHolder::fixed_size(2, 1, 1, 1));
|
|
295
|
+
starter.sdk_config.register_activities(StdActivities);
|
|
260
296
|
let mut worker = starter.worker().await;
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
activity_type: "echo_activity".to_string(),
|
|
265
|
-
start_to_close_timeout: Some(Duration::from_secs(5)),
|
|
266
|
-
input: "hi!".as_json_payload().expect("serializes fine"),
|
|
267
|
-
..Default::default()
|
|
268
|
-
})
|
|
269
|
-
.await;
|
|
270
|
-
}
|
|
271
|
-
Ok(().into())
|
|
272
|
-
});
|
|
273
|
-
worker.register_activity("echo_activity", echo);
|
|
297
|
+
|
|
298
|
+
worker.register_workflow::<OnlyOneWorkflowSlotAndTwoPollers>();
|
|
299
|
+
let task_queue = starter.get_task_queue().to_owned();
|
|
274
300
|
worker
|
|
275
|
-
.
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
WorkflowOptions::default(),
|
|
301
|
+
.submit_workflow(
|
|
302
|
+
OnlyOneWorkflowSlotAndTwoPollers::run,
|
|
303
|
+
(),
|
|
304
|
+
WorkflowStartOptions::new(task_queue.clone(), task_queue.clone()).build(),
|
|
280
305
|
)
|
|
281
306
|
.await
|
|
282
307
|
.unwrap();
|
|
283
308
|
let wf2id = format!("{}-2", starter.get_task_queue());
|
|
284
309
|
worker
|
|
285
|
-
.
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
WorkflowOptions::default(),
|
|
310
|
+
.submit_workflow(
|
|
311
|
+
OnlyOneWorkflowSlotAndTwoPollers::run,
|
|
312
|
+
(),
|
|
313
|
+
WorkflowStartOptions::new(task_queue.clone(), wf2id.clone()).build(),
|
|
290
314
|
)
|
|
291
315
|
.await
|
|
292
316
|
.unwrap();
|
|
@@ -299,16 +323,15 @@ async fn small_workflow_slots_and_pollers(#[values(false, true)] use_autoscaling
|
|
|
299
323
|
.iter()
|
|
300
324
|
.any(|e| e.event_type() == EventType::WorkflowTaskTimedOut);
|
|
301
325
|
assert!(!any_task_timeouts);
|
|
302
|
-
let
|
|
326
|
+
let events = starter
|
|
303
327
|
.get_client()
|
|
304
328
|
.await
|
|
305
|
-
.
|
|
329
|
+
.get_workflow_handle::<UntypedWorkflow>(&wf2id)
|
|
330
|
+
.fetch_history(Default::default())
|
|
306
331
|
.await
|
|
307
332
|
.unwrap()
|
|
308
|
-
.
|
|
309
|
-
|
|
310
|
-
let any_task_timeouts = history
|
|
311
|
-
.events
|
|
333
|
+
.into_events();
|
|
334
|
+
let any_task_timeouts = events
|
|
312
335
|
.iter()
|
|
313
336
|
.any(|e| e.event_type() == EventType::WorkflowTaskTimedOut);
|
|
314
337
|
assert!(!any_task_timeouts);
|
|
@@ -378,42 +401,40 @@ async fn replace_client_works_after_polling_failure() {
|
|
|
378
401
|
"http://{}",
|
|
379
402
|
initial_server.lock().unwrap().as_ref().unwrap().target
|
|
380
403
|
);
|
|
381
|
-
let
|
|
404
|
+
let opts = ConnectionOptions::new(Url::parse(&initial_server_target).unwrap())
|
|
382
405
|
.identity("client_for_initial_server".to_string())
|
|
383
|
-
.target_url(Url::parse(&initial_server_target).unwrap())
|
|
384
406
|
.client_name(INTEG_CLIENT_NAME.to_string())
|
|
385
407
|
.client_version(INTEG_CLIENT_VERSION.to_string())
|
|
386
|
-
.build()
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
408
|
+
.build();
|
|
409
|
+
let connection = Connection::connect(opts).await.unwrap();
|
|
410
|
+
let client_opts = temporalio_client::ClientOptions::new(NAMESPACE).build();
|
|
411
|
+
let client_for_initial_server = Client::new(connection, client_opts).unwrap();
|
|
390
412
|
|
|
391
413
|
let wf_name = "replace_client_works_after_polling_failure";
|
|
392
414
|
let task_queue = format!("{wf_name}_tq");
|
|
393
415
|
|
|
394
416
|
let mut config = integ_worker_config(&task_queue);
|
|
395
417
|
config.max_cached_workflows = 100_usize;
|
|
396
|
-
let worker =
|
|
397
|
-
|
|
418
|
+
let worker = Arc::new(
|
|
419
|
+
init_worker(&rt, config, client_for_initial_server.connection().clone()).unwrap(),
|
|
420
|
+
);
|
|
398
421
|
|
|
399
422
|
// Polling the initial server the first time is successful.
|
|
400
423
|
let wf_1 = client_for_initial_server
|
|
401
424
|
.start_workflow(
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
wf_name.
|
|
405
|
-
wf_name.into(),
|
|
406
|
-
None,
|
|
407
|
-
WorkflowOptions::default(),
|
|
425
|
+
UntypedWorkflow::new(wf_name),
|
|
426
|
+
RawValue::default(),
|
|
427
|
+
WorkflowStartOptions::new(task_queue.clone(), wf_name.to_string()).build(),
|
|
408
428
|
)
|
|
409
429
|
.await
|
|
410
430
|
.unwrap();
|
|
431
|
+
let wf_1_run_id = wf_1.run_id().unwrap().to_string();
|
|
411
432
|
let act_1 =
|
|
412
433
|
tokio::time::timeout(Duration::from_secs(60), worker.poll_workflow_activation())
|
|
413
434
|
.await
|
|
414
435
|
.unwrap()
|
|
415
436
|
.unwrap();
|
|
416
|
-
assert_eq!(act_1.run_id,
|
|
437
|
+
assert_eq!(act_1.run_id, wf_1_run_id);
|
|
417
438
|
|
|
418
439
|
// Initial server is shut down.
|
|
419
440
|
let mut server = initial_server.lock().unwrap().take().unwrap();
|
|
@@ -436,34 +457,34 @@ async fn replace_client_works_after_polling_failure() {
|
|
|
436
457
|
.unwrap();
|
|
437
458
|
|
|
438
459
|
// Start a new WF on main integration server.
|
|
439
|
-
let client_for_integ_server =
|
|
440
|
-
|
|
441
|
-
.
|
|
442
|
-
|
|
460
|
+
let client_for_integ_server = get_integ_client(
|
|
461
|
+
NAMESPACE.to_string(),
|
|
462
|
+
rt.telemetry().get_temporal_metric_meter(),
|
|
463
|
+
)
|
|
464
|
+
.await;
|
|
443
465
|
let wf_2 = client_for_integ_server
|
|
444
466
|
.start_workflow(
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
wf_name
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
WorkflowOptions {
|
|
451
|
-
execution_timeout: Some(Duration::from_secs(60)),
|
|
452
|
-
..Default::default()
|
|
453
|
-
},
|
|
467
|
+
UntypedWorkflow::new(wf_name),
|
|
468
|
+
RawValue::default(),
|
|
469
|
+
WorkflowStartOptions::new(task_queue, wf_name)
|
|
470
|
+
.execution_timeout(Duration::from_secs(60))
|
|
471
|
+
.build(),
|
|
454
472
|
)
|
|
455
473
|
.await
|
|
456
474
|
.unwrap();
|
|
475
|
+
let wf_2_run_id = wf_2.run_id().unwrap().to_string();
|
|
457
476
|
|
|
458
477
|
// Switch worker over to the main integration server.
|
|
459
478
|
// The polling started on the initial server should complete with a task from the new server.
|
|
460
|
-
worker
|
|
479
|
+
worker
|
|
480
|
+
.replace_client(client_for_integ_server.connection().clone())
|
|
481
|
+
.unwrap();
|
|
461
482
|
let act_2 = tokio::time::timeout(Duration::from_secs(60), poll_join_handle)
|
|
462
483
|
.await
|
|
463
484
|
.unwrap()
|
|
464
485
|
.unwrap()
|
|
465
486
|
.unwrap();
|
|
466
|
-
assert_eq!(act_2.run_id,
|
|
487
|
+
assert_eq!(act_2.run_id, wf_2_run_id);
|
|
467
488
|
})
|
|
468
489
|
}
|
|
469
490
|
.catch_unwind()
|