@temporalio/core-bridge 0.14.0 → 0.16.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +162 -38
- package/Cargo.toml +3 -3
- package/index.d.ts +14 -1
- package/index.node +0 -0
- package/package.json +8 -5
- package/releases/aarch64-apple-darwin/index.node +0 -0
- package/releases/{x86_64-pc-windows-gnu → aarch64-unknown-linux-gnu}/index.node +0 -0
- package/releases/x86_64-apple-darwin/index.node +0 -0
- package/releases/x86_64-pc-windows-msvc/index.node +0 -0
- package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
- package/scripts/build.js +77 -34
- package/sdk-core/.buildkite/docker/Dockerfile +1 -1
- package/sdk-core/Cargo.toml +6 -5
- package/sdk-core/fsm/Cargo.toml +1 -1
- package/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +2 -2
- package/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +8 -9
- package/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +13 -7
- package/sdk-core/fsm/rustfsm_trait/Cargo.toml +2 -2
- package/sdk-core/fsm/rustfsm_trait/src/lib.rs +1 -1
- package/sdk-core/protos/local/workflow_activation.proto +6 -3
- package/sdk-core/sdk-core-protos/Cargo.toml +4 -4
- package/sdk-core/sdk-core-protos/src/lib.rs +38 -50
- package/sdk-core/src/core_tests/activity_tasks.rs +5 -5
- package/sdk-core/src/core_tests/child_workflows.rs +55 -29
- package/sdk-core/src/core_tests/determinism.rs +19 -9
- package/sdk-core/src/core_tests/mod.rs +3 -3
- package/sdk-core/src/core_tests/retry.rs +14 -8
- package/sdk-core/src/core_tests/workers.rs +1 -1
- package/sdk-core/src/core_tests/workflow_tasks.rs +347 -4
- package/sdk-core/src/errors.rs +27 -44
- package/sdk-core/src/lib.rs +13 -3
- package/sdk-core/src/machines/activity_state_machine.rs +44 -5
- package/sdk-core/src/machines/child_workflow_state_machine.rs +31 -11
- package/sdk-core/src/machines/complete_workflow_state_machine.rs +1 -1
- package/sdk-core/src/machines/continue_as_new_workflow_state_machine.rs +1 -1
- package/sdk-core/src/machines/mod.rs +18 -23
- package/sdk-core/src/machines/patch_state_machine.rs +8 -8
- package/sdk-core/src/machines/signal_external_state_machine.rs +22 -1
- package/sdk-core/src/machines/timer_state_machine.rs +21 -3
- package/sdk-core/src/machines/transition_coverage.rs +3 -3
- package/sdk-core/src/machines/workflow_machines.rs +11 -11
- package/sdk-core/src/pending_activations.rs +27 -22
- package/sdk-core/src/pollers/gateway.rs +15 -7
- package/sdk-core/src/pollers/poll_buffer.rs +6 -5
- package/sdk-core/src/pollers/retry.rs +153 -120
- package/sdk-core/src/prototype_rust_sdk/workflow_context.rs +61 -46
- package/sdk-core/src/prototype_rust_sdk/workflow_future.rs +13 -12
- package/sdk-core/src/prototype_rust_sdk.rs +17 -23
- package/sdk-core/src/telemetry/metrics.rs +2 -4
- package/sdk-core/src/telemetry/mod.rs +6 -7
- package/sdk-core/src/test_help/canned_histories.rs +17 -93
- package/sdk-core/src/test_help/history_builder.rs +61 -2
- package/sdk-core/src/test_help/history_info.rs +21 -2
- package/sdk-core/src/test_help/mod.rs +26 -34
- package/sdk-core/src/worker/activities/activity_heartbeat_manager.rs +246 -138
- package/sdk-core/src/worker/activities.rs +46 -45
- package/sdk-core/src/worker/config.rs +11 -0
- package/sdk-core/src/worker/dispatcher.rs +5 -5
- package/sdk-core/src/worker/mod.rs +86 -56
- package/sdk-core/src/workflow/driven_workflow.rs +3 -3
- package/sdk-core/src/workflow/history_update.rs +1 -1
- package/sdk-core/src/workflow/mod.rs +2 -1
- package/sdk-core/src/workflow/workflow_tasks/cache_manager.rs +13 -17
- package/sdk-core/src/workflow/workflow_tasks/concurrency_manager.rs +10 -18
- package/sdk-core/src/workflow/workflow_tasks/mod.rs +72 -57
- package/sdk-core/test_utils/Cargo.toml +1 -1
- package/sdk-core/test_utils/src/lib.rs +2 -2
- package/sdk-core/tests/integ_tests/workflow_tests/activities.rs +61 -1
- package/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +2 -2
- package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +49 -0
- package/sdk-core/tests/integ_tests/workflow_tests/signals.rs +2 -2
- package/sdk-core/tests/integ_tests/workflow_tests.rs +1 -0
- package/src/conversions.rs +17 -0
- package/src/errors.rs +0 -7
- package/src/lib.rs +0 -20
|
@@ -35,17 +35,13 @@ impl WorkflowCacheManager {
|
|
|
35
35
|
// Blindly add a record into the cache, since it still has capacity.
|
|
36
36
|
self.cache.put(run_id.to_owned(), ());
|
|
37
37
|
None
|
|
38
|
-
} else if self.cache.cap()
|
|
39
|
-
let maybe_got_evicted = self.cache.peek_lru().map(|r| r.0.to_owned());
|
|
40
|
-
let already_existed = self.cache.put(run_id.to_owned(), ()).is_some();
|
|
41
|
-
if !already_existed {
|
|
42
|
-
maybe_got_evicted
|
|
43
|
-
} else {
|
|
44
|
-
None
|
|
45
|
-
}
|
|
46
|
-
} else {
|
|
38
|
+
} else if self.cache.cap() == 0 {
|
|
47
39
|
// Run id should be evicted right away as cache size is 0.
|
|
48
40
|
Some(run_id.to_owned())
|
|
41
|
+
} else {
|
|
42
|
+
let maybe_got_evicted = self.cache.peek_lru().map(|r| r.0.clone());
|
|
43
|
+
let not_cached = self.cache.put(run_id.to_owned(), ()).is_none();
|
|
44
|
+
not_cached.then(|| maybe_got_evicted).flatten()
|
|
49
45
|
};
|
|
50
46
|
|
|
51
47
|
self.metrics.cache_size(self.cache.len() as u64);
|
|
@@ -75,7 +71,7 @@ mod tests {
|
|
|
75
71
|
assert_matches!(wcm.insert("1"), None);
|
|
76
72
|
assert_matches!(wcm.insert("2"), None);
|
|
77
73
|
assert_matches!(wcm.insert("3"), Some(run_id) => {
|
|
78
|
-
assert_eq!(run_id, "1")
|
|
74
|
+
assert_eq!(run_id, "1");
|
|
79
75
|
});
|
|
80
76
|
}
|
|
81
77
|
|
|
@@ -88,7 +84,7 @@ mod tests {
|
|
|
88
84
|
wcm.remove("1");
|
|
89
85
|
assert_matches!(wcm.insert("2"), None);
|
|
90
86
|
assert_matches!(wcm.insert("3"), Some(run_id) => {
|
|
91
|
-
assert_eq!(run_id, "2")
|
|
87
|
+
assert_eq!(run_id, "2");
|
|
92
88
|
});
|
|
93
89
|
}
|
|
94
90
|
|
|
@@ -110,7 +106,7 @@ mod tests {
|
|
|
110
106
|
assert_matches!(wcm.insert("2"), None);
|
|
111
107
|
wcm.touch("1");
|
|
112
108
|
assert_matches!(wcm.insert("3"), Some(run_id) => {
|
|
113
|
-
assert_eq!(run_id, "2")
|
|
109
|
+
assert_eq!(run_id, "2");
|
|
114
110
|
});
|
|
115
111
|
}
|
|
116
112
|
|
|
@@ -123,7 +119,7 @@ mod tests {
|
|
|
123
119
|
assert_matches!(wcm.insert("1"), None);
|
|
124
120
|
assert_matches!(wcm.insert("2"), None);
|
|
125
121
|
assert_matches!(wcm.insert("3"), Some(run_id) => {
|
|
126
|
-
assert_eq!(run_id, "1")
|
|
122
|
+
assert_eq!(run_id, "1");
|
|
127
123
|
});
|
|
128
124
|
}
|
|
129
125
|
|
|
@@ -133,10 +129,10 @@ mod tests {
|
|
|
133
129
|
max_cached_workflows: 0,
|
|
134
130
|
});
|
|
135
131
|
assert_matches!(wcm.insert("1"), Some(run_id) => {
|
|
136
|
-
assert_eq!(run_id, "1")
|
|
132
|
+
assert_eq!(run_id, "1");
|
|
137
133
|
});
|
|
138
134
|
assert_matches!(wcm.insert("2"), Some(run_id) => {
|
|
139
|
-
assert_eq!(run_id, "2")
|
|
135
|
+
assert_eq!(run_id, "2");
|
|
140
136
|
});
|
|
141
137
|
}
|
|
142
138
|
|
|
@@ -144,10 +140,10 @@ mod tests {
|
|
|
144
140
|
fn non_sticky_always_pending_eviction() {
|
|
145
141
|
let mut wcm = WorkflowCacheManager::new_test(WorkflowCachingPolicy::NonSticky);
|
|
146
142
|
assert_matches!(wcm.insert("1"), Some(run_id) => {
|
|
147
|
-
assert_eq!(run_id, "1")
|
|
143
|
+
assert_eq!(run_id, "1");
|
|
148
144
|
});
|
|
149
145
|
assert_matches!(wcm.insert("2"), Some(run_id) => {
|
|
150
|
-
assert_eq!(run_id, "2")
|
|
146
|
+
assert_eq!(run_id, "2");
|
|
151
147
|
});
|
|
152
148
|
}
|
|
153
149
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
use crate::{
|
|
2
|
-
errors::
|
|
2
|
+
errors::WorkflowMissingError,
|
|
3
3
|
protosext::ValidPollWFTQResponse,
|
|
4
4
|
telemetry::metrics::{workflow_type, MetricsContext},
|
|
5
5
|
workflow::{
|
|
@@ -89,7 +89,7 @@ impl WorkflowConcurrencyManager {
|
|
|
89
89
|
pub(crate) fn get_task_mut(
|
|
90
90
|
&self,
|
|
91
91
|
run_id: &str,
|
|
92
|
-
) -> Result<impl DerefMut<Target = Option<OutstandingTask>> + '_,
|
|
92
|
+
) -> Result<impl DerefMut<Target = Option<OutstandingTask>> + '_, WorkflowMissingError> {
|
|
93
93
|
let writelock = self.runs.write();
|
|
94
94
|
if writelock.contains_key(run_id) {
|
|
95
95
|
Ok(RwLockWriteGuard::map(writelock, |hm| {
|
|
@@ -97,10 +97,8 @@ impl WorkflowConcurrencyManager {
|
|
|
97
97
|
&mut hm.get_mut(run_id).unwrap().wft
|
|
98
98
|
}))
|
|
99
99
|
} else {
|
|
100
|
-
Err(
|
|
101
|
-
source: WFMachinesError::Fatal("Workflow machines not found".to_string()),
|
|
100
|
+
Err(WorkflowMissingError {
|
|
102
101
|
run_id: run_id.to_owned(),
|
|
103
|
-
task_token: None,
|
|
104
102
|
})
|
|
105
103
|
}
|
|
106
104
|
}
|
|
@@ -146,7 +144,7 @@ impl WorkflowConcurrencyManager {
|
|
|
146
144
|
&self,
|
|
147
145
|
run_id: &str,
|
|
148
146
|
task: OutstandingTask,
|
|
149
|
-
) -> Result<(),
|
|
147
|
+
) -> Result<(), WorkflowMissingError> {
|
|
150
148
|
let mut dereffer = self.get_task_mut(run_id)?;
|
|
151
149
|
*dereffer = Some(task);
|
|
152
150
|
Ok(())
|
|
@@ -171,16 +169,14 @@ impl WorkflowConcurrencyManager {
|
|
|
171
169
|
&self,
|
|
172
170
|
run_id: &str,
|
|
173
171
|
activation: OutstandingActivation,
|
|
174
|
-
) -> Result<Option<OutstandingActivation>,
|
|
172
|
+
) -> Result<Option<OutstandingActivation>, WorkflowMissingError> {
|
|
175
173
|
let mut writelock = self.runs.write();
|
|
176
174
|
let machine_ref = writelock.get_mut(run_id);
|
|
177
175
|
if let Some(run) = machine_ref {
|
|
178
176
|
Ok(run.activation.replace(activation))
|
|
179
177
|
} else {
|
|
180
|
-
Err(
|
|
181
|
-
source: WFMachinesError::Fatal("Workflow machines not found".to_string()),
|
|
178
|
+
Err(WorkflowMissingError {
|
|
182
179
|
run_id: run_id.to_owned(),
|
|
183
|
-
task_token: None,
|
|
184
180
|
})
|
|
185
181
|
}
|
|
186
182
|
}
|
|
@@ -188,11 +184,7 @@ impl WorkflowConcurrencyManager {
|
|
|
188
184
|
pub fn delete_activation(&self, run_id: &str) -> Option<OutstandingActivation> {
|
|
189
185
|
let mut writelock = self.runs.write();
|
|
190
186
|
let machine_ref = writelock.get_mut(run_id);
|
|
191
|
-
|
|
192
|
-
run.activation.take()
|
|
193
|
-
} else {
|
|
194
|
-
None
|
|
195
|
-
}
|
|
187
|
+
machine_ref.and_then(|run| run.activation.take())
|
|
196
188
|
}
|
|
197
189
|
|
|
198
190
|
pub fn exists(&self, run_id: &str) -> bool {
|
|
@@ -274,14 +266,14 @@ impl WorkflowConcurrencyManager {
|
|
|
274
266
|
/// Remove the workflow with the provided run id from management
|
|
275
267
|
pub fn evict(&self, run_id: &str) -> Option<ValidPollWFTQResponse> {
|
|
276
268
|
let val = self.runs.write().remove(run_id);
|
|
277
|
-
val.
|
|
269
|
+
val.and_then(|v| v.buffered_resp)
|
|
278
270
|
}
|
|
279
271
|
|
|
280
272
|
/// Clear and return any buffered polling response for this run ID
|
|
281
273
|
pub fn take_buffered_poll(&self, run_id: &str) -> Option<ValidPollWFTQResponse> {
|
|
282
274
|
let mut writelock = self.runs.write();
|
|
283
275
|
let val = writelock.get_mut(run_id);
|
|
284
|
-
val.
|
|
276
|
+
val.and_then(|v| v.buffered_resp.take())
|
|
285
277
|
}
|
|
286
278
|
|
|
287
279
|
pub fn outstanding_wft(&self) -> usize {
|
|
@@ -315,6 +307,6 @@ mod tests {
|
|
|
315
307
|
)
|
|
316
308
|
.await;
|
|
317
309
|
// Should whine that the machines have nothing to do (history empty)
|
|
318
|
-
assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. })
|
|
310
|
+
assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. });
|
|
319
311
|
}
|
|
320
312
|
}
|
|
@@ -4,7 +4,7 @@ mod cache_manager;
|
|
|
4
4
|
mod concurrency_manager;
|
|
5
5
|
|
|
6
6
|
use crate::{
|
|
7
|
-
errors::WorkflowUpdateError,
|
|
7
|
+
errors::{WorkflowMissingError, WorkflowUpdateError},
|
|
8
8
|
machines::{ProtoCommand, WFCommand, WFMachinesError},
|
|
9
9
|
pending_activations::PendingActivations,
|
|
10
10
|
pollers::GatewayRef,
|
|
@@ -21,7 +21,7 @@ use crate::{
|
|
|
21
21
|
use crossbeam::queue::SegQueue;
|
|
22
22
|
use futures::FutureExt;
|
|
23
23
|
use parking_lot::Mutex;
|
|
24
|
-
use std::{fmt::Debug,
|
|
24
|
+
use std::{fmt::Debug, time::Instant};
|
|
25
25
|
use temporal_sdk_core_protos::coresdk::{
|
|
26
26
|
workflow_activation::{
|
|
27
27
|
create_evict_activation, create_query_activation, wf_activation_job, QueryWorkflow,
|
|
@@ -77,9 +77,9 @@ pub(crate) enum OutstandingActivation {
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
impl OutstandingActivation {
|
|
80
|
-
fn has_eviction(
|
|
80
|
+
const fn has_eviction(self) -> bool {
|
|
81
81
|
matches!(
|
|
82
|
-
|
|
82
|
+
self,
|
|
83
83
|
OutstandingActivation::Normal {
|
|
84
84
|
contains_eviction: true
|
|
85
85
|
}
|
|
@@ -96,7 +96,7 @@ pub struct WorkflowTaskInfo {
|
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
#[derive(Debug, derive_more::From)]
|
|
99
|
-
pub enum NewWfTaskOutcome {
|
|
99
|
+
pub(crate) enum NewWfTaskOutcome {
|
|
100
100
|
/// A new activation for the workflow should be issued to lang
|
|
101
101
|
IssueActivation(WfActivation),
|
|
102
102
|
/// The poll loop should be restarted, there is nothing to do
|
|
@@ -106,6 +106,8 @@ pub enum NewWfTaskOutcome {
|
|
|
106
106
|
Autocomplete,
|
|
107
107
|
/// Workflow task had partial history and workflow was not present in the cache.
|
|
108
108
|
CacheMiss,
|
|
109
|
+
/// The workflow task ran into problems while being applied and we must now evict the workflow
|
|
110
|
+
Evict(WorkflowUpdateError),
|
|
109
111
|
}
|
|
110
112
|
|
|
111
113
|
#[derive(Debug)]
|
|
@@ -162,9 +164,7 @@ impl WorkflowTaskManager {
|
|
|
162
164
|
}
|
|
163
165
|
}
|
|
164
166
|
|
|
165
|
-
pub(crate) fn next_pending_activation(
|
|
166
|
-
&self,
|
|
167
|
-
) -> Result<Option<WfActivation>, WorkflowUpdateError> {
|
|
167
|
+
pub(crate) fn next_pending_activation(&self) -> Option<WfActivation> {
|
|
168
168
|
// It is important that we do not issue pending activations for any workflows which already
|
|
169
169
|
// have an outstanding activation. If we did, it can result in races where an in-progress
|
|
170
170
|
// completion may appear to be the last in a task (no more pending activations) because
|
|
@@ -174,10 +174,14 @@ impl WorkflowTaskManager {
|
|
|
174
174
|
.pending_activations
|
|
175
175
|
.pop_first_matching(|rid| self.workflow_machines.get_activation(rid).is_none());
|
|
176
176
|
if let Some(act) = maybe_act.as_ref() {
|
|
177
|
-
self.insert_outstanding_activation(act)
|
|
177
|
+
if let Err(WorkflowMissingError { run_id }) = self.insert_outstanding_activation(act) {
|
|
178
|
+
self.request_eviction(&run_id, "Pending activation present for missing run");
|
|
179
|
+
// Continue trying to return a valid pending activation
|
|
180
|
+
return self.next_pending_activation();
|
|
181
|
+
}
|
|
178
182
|
self.cache_manager.lock().touch(&act.run_id);
|
|
179
183
|
}
|
|
180
|
-
|
|
184
|
+
maybe_act
|
|
181
185
|
}
|
|
182
186
|
|
|
183
187
|
pub fn next_buffered_poll(&self) -> Option<ValidPollWFTQResponse> {
|
|
@@ -192,13 +196,13 @@ impl WorkflowTaskManager {
|
|
|
192
196
|
/// the lang side. Workflow will not *actually* be evicted until lang replies to that activation
|
|
193
197
|
///
|
|
194
198
|
/// Returns, if found, the number of attempts on the current workflow task
|
|
195
|
-
pub fn request_eviction(&self, run_id: &str) -> Option<u32> {
|
|
199
|
+
pub fn request_eviction(&self, run_id: &str, reason: impl Into<String>) -> Option<u32> {
|
|
196
200
|
if self.workflow_machines.exists(run_id) {
|
|
197
201
|
if !self.activation_has_eviction(run_id) {
|
|
198
202
|
debug!(%run_id, "Eviction requested");
|
|
199
203
|
// Queue up an eviction activation
|
|
200
204
|
self.pending_activations
|
|
201
|
-
.push(create_evict_activation(run_id.to_string()));
|
|
205
|
+
.push(create_evict_activation(run_id.to_string(), reason.into()));
|
|
202
206
|
let _ = self.pending_activations_notifier.send(true);
|
|
203
207
|
}
|
|
204
208
|
self.workflow_machines
|
|
@@ -241,12 +245,12 @@ impl WorkflowTaskManager {
|
|
|
241
245
|
&self,
|
|
242
246
|
work: ValidPollWFTQResponse,
|
|
243
247
|
gateway: &GatewayRef,
|
|
244
|
-
) ->
|
|
248
|
+
) -> NewWfTaskOutcome {
|
|
245
249
|
let mut work = if let Some(w) = self.workflow_machines.buffer_resp_if_outstanding_work(work)
|
|
246
250
|
{
|
|
247
251
|
w
|
|
248
252
|
} else {
|
|
249
|
-
return
|
|
253
|
+
return NewWfTaskOutcome::TaskBuffered;
|
|
250
254
|
};
|
|
251
255
|
|
|
252
256
|
debug!(
|
|
@@ -269,9 +273,9 @@ impl WorkflowTaskManager {
|
|
|
269
273
|
Ok((info, next_activation)) => (info, next_activation),
|
|
270
274
|
Err(e) => {
|
|
271
275
|
if let WFMachinesError::CacheMiss = e.source {
|
|
272
|
-
return
|
|
276
|
+
return NewWfTaskOutcome::CacheMiss;
|
|
273
277
|
}
|
|
274
|
-
return
|
|
278
|
+
return NewWfTaskOutcome::Evict(e);
|
|
275
279
|
}
|
|
276
280
|
};
|
|
277
281
|
|
|
@@ -288,20 +292,24 @@ impl WorkflowTaskManager {
|
|
|
288
292
|
legacy_query
|
|
289
293
|
};
|
|
290
294
|
|
|
291
|
-
self.workflow_machines
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
295
|
+
self.workflow_machines
|
|
296
|
+
.insert_wft(
|
|
297
|
+
&next_activation.run_id,
|
|
298
|
+
OutstandingTask {
|
|
299
|
+
info,
|
|
300
|
+
legacy_query,
|
|
301
|
+
start_time: task_start_time,
|
|
302
|
+
},
|
|
303
|
+
)
|
|
304
|
+
.expect("Workflow machines must exist, we just created/updated them");
|
|
305
|
+
|
|
306
|
+
if next_activation.jobs.is_empty() {
|
|
307
|
+
NewWfTaskOutcome::Autocomplete
|
|
303
308
|
} else {
|
|
304
|
-
|
|
309
|
+
if let Err(wme) = self.insert_outstanding_activation(&next_activation) {
|
|
310
|
+
return NewWfTaskOutcome::Evict(wme.into());
|
|
311
|
+
}
|
|
312
|
+
NewWfTaskOutcome::IssueActivation(next_activation)
|
|
305
313
|
}
|
|
306
314
|
}
|
|
307
315
|
|
|
@@ -398,7 +406,9 @@ impl WorkflowTaskManager {
|
|
|
398
406
|
query_responses,
|
|
399
407
|
},
|
|
400
408
|
})
|
|
401
|
-
} else if
|
|
409
|
+
} else if query_responses.is_empty() {
|
|
410
|
+
None
|
|
411
|
+
} else {
|
|
402
412
|
Some(ServerCommandsWithWorkflowInfo {
|
|
403
413
|
task_token,
|
|
404
414
|
action: ActivationAction::WftComplete {
|
|
@@ -406,8 +416,6 @@ impl WorkflowTaskManager {
|
|
|
406
416
|
query_responses,
|
|
407
417
|
},
|
|
408
418
|
})
|
|
409
|
-
} else {
|
|
410
|
-
None
|
|
411
419
|
}
|
|
412
420
|
};
|
|
413
421
|
Ok(ret)
|
|
@@ -439,12 +447,9 @@ impl WorkflowTaskManager {
|
|
|
439
447
|
FailedActivationOutcome::ReportLegacyQueryFailure(tt)
|
|
440
448
|
} else {
|
|
441
449
|
// Blow up any cached data associated with the workflow
|
|
442
|
-
let should_report =
|
|
443
|
-
|
|
444
|
-
attempt <= 1
|
|
445
|
-
} else {
|
|
446
|
-
true
|
|
447
|
-
};
|
|
450
|
+
let should_report = self
|
|
451
|
+
.request_eviction(run_id, "Activation failed by lang")
|
|
452
|
+
.map_or(true, |attempt| attempt <= 1);
|
|
448
453
|
if should_report {
|
|
449
454
|
FailedActivationOutcome::Report(tt)
|
|
450
455
|
} else {
|
|
@@ -516,7 +521,7 @@ impl WorkflowTaskManager {
|
|
|
516
521
|
/// if this is called too early.
|
|
517
522
|
///
|
|
518
523
|
/// Returns true if WFT is complete
|
|
519
|
-
pub(crate) fn after_wft_report(&self, run_id: &str) ->
|
|
524
|
+
pub(crate) fn after_wft_report(&self, run_id: &str) -> bool {
|
|
520
525
|
let mut just_evicted = false;
|
|
521
526
|
|
|
522
527
|
if let Some(OutstandingActivation::Normal {
|
|
@@ -532,19 +537,23 @@ impl WorkflowTaskManager {
|
|
|
532
537
|
if !just_evicted {
|
|
533
538
|
// Check if there was a legacy query which must be fulfilled, and if there is create
|
|
534
539
|
// a new pending activation for it.
|
|
535
|
-
if let Some(ref mut ot) = self
|
|
540
|
+
if let Some(ref mut ot) = &mut *self
|
|
541
|
+
.workflow_machines
|
|
542
|
+
.get_task_mut(run_id)
|
|
543
|
+
.expect("Machine must exist")
|
|
544
|
+
{
|
|
536
545
|
if let Some(query) = ot.legacy_query.take() {
|
|
537
546
|
let na = create_query_activation(run_id.to_string(), [query]);
|
|
538
547
|
self.pending_activations.push(na);
|
|
539
548
|
let _ = self.pending_activations_notifier.send(true);
|
|
540
|
-
return
|
|
549
|
+
return false;
|
|
541
550
|
}
|
|
542
551
|
}
|
|
543
552
|
|
|
544
553
|
// Evict run id if cache is full. Non-sticky will always evict.
|
|
545
554
|
let maybe_evicted = self.cache_manager.lock().insert(run_id);
|
|
546
555
|
if let Some(evicted_run_id) = maybe_evicted {
|
|
547
|
-
self.request_eviction(&evicted_run_id);
|
|
556
|
+
self.request_eviction(&evicted_run_id, "Workflow cache full");
|
|
548
557
|
}
|
|
549
558
|
|
|
550
559
|
// If there was a buffered poll response from the server, it is now ready to
|
|
@@ -556,9 +565,9 @@ impl WorkflowTaskManager {
|
|
|
556
565
|
|
|
557
566
|
// The evict may or may not have already done this, but even when we aren't evicting
|
|
558
567
|
// we want to clear the outstanding workflow task since it's now complete.
|
|
559
|
-
return
|
|
568
|
+
return self.workflow_machines.complete_wft(run_id).is_some();
|
|
560
569
|
}
|
|
561
|
-
|
|
570
|
+
false
|
|
562
571
|
}
|
|
563
572
|
|
|
564
573
|
/// Must be called after *every* activation is replied to, regardless of whether or not we
|
|
@@ -577,7 +586,10 @@ impl WorkflowTaskManager {
|
|
|
577
586
|
self.ready_buffered_wft.push(buffd);
|
|
578
587
|
}
|
|
579
588
|
|
|
580
|
-
fn insert_outstanding_activation(
|
|
589
|
+
fn insert_outstanding_activation(
|
|
590
|
+
&self,
|
|
591
|
+
act: &WfActivation,
|
|
592
|
+
) -> Result<(), WorkflowMissingError> {
|
|
581
593
|
let act_type = if act.is_legacy_query() {
|
|
582
594
|
OutstandingActivation::LegacyQuery
|
|
583
595
|
} else {
|
|
@@ -585,25 +597,28 @@ impl WorkflowTaskManager {
|
|
|
585
597
|
contains_eviction: act.eviction_index().is_some(),
|
|
586
598
|
}
|
|
587
599
|
};
|
|
588
|
-
|
|
600
|
+
match self
|
|
589
601
|
.workflow_machines
|
|
590
|
-
.insert_activation(&act.run_id, act_type)
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
602
|
+
.insert_activation(&act.run_id, act_type)
|
|
603
|
+
{
|
|
604
|
+
Ok(None) => Ok(()),
|
|
605
|
+
Ok(Some(previous)) => {
|
|
606
|
+
// This is a panic because we have screwed up core logic if this is violated. It
|
|
607
|
+
// must be upheld.
|
|
608
|
+
panic!(
|
|
609
|
+
"Attempted to insert a new outstanding activation {}, but there already was \
|
|
610
|
+
one outstanding: {:?}",
|
|
611
|
+
act, previous
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
Err(e) => Err(e),
|
|
599
615
|
}
|
|
600
|
-
Ok(())
|
|
601
616
|
}
|
|
602
617
|
|
|
603
618
|
fn activation_has_eviction(&self, run_id: &str) -> bool {
|
|
604
619
|
self.workflow_machines
|
|
605
620
|
.get_activation(run_id)
|
|
606
|
-
.map(
|
|
621
|
+
.map(OutstandingActivation::has_eviction)
|
|
607
622
|
.unwrap_or_default()
|
|
608
623
|
}
|
|
609
624
|
}
|
|
@@ -46,7 +46,7 @@ impl CoreWfStarter {
|
|
|
46
46
|
.unwrap(),
|
|
47
47
|
worker_config: WorkerConfigBuilder::default()
|
|
48
48
|
.task_queue(task_queue)
|
|
49
|
-
.max_cached_workflows(
|
|
49
|
+
.max_cached_workflows(1000_usize)
|
|
50
50
|
.build()
|
|
51
51
|
.unwrap(),
|
|
52
52
|
wft_timeout: None,
|
|
@@ -63,7 +63,7 @@ impl CoreWfStarter {
|
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
pub async fn shutdown(&mut self) {
|
|
66
|
-
self.get_core().await.shutdown().await
|
|
66
|
+
self.get_core().await.shutdown().await;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
pub async fn get_core(&mut self) -> Arc<dyn Core> {
|
|
@@ -8,7 +8,7 @@ use temporal_sdk_core_protos::{
|
|
|
8
8
|
workflow_activation::{wf_activation_job, FireTimer, ResolveActivity, WfActivationJob},
|
|
9
9
|
workflow_commands::{ActivityCancellationType, RequestCancelActivity, StartTimer},
|
|
10
10
|
workflow_completion::WfActivationCompletion,
|
|
11
|
-
ActivityTaskCompletion, IntoCompletion,
|
|
11
|
+
ActivityHeartbeat, ActivityTaskCompletion, IntoCompletion,
|
|
12
12
|
},
|
|
13
13
|
temporal::api::{
|
|
14
14
|
common::v1::{ActivityType, Payloads},
|
|
@@ -647,3 +647,63 @@ async fn async_activity_completion_workflow() {
|
|
|
647
647
|
);
|
|
648
648
|
core.complete_execution(&task_q, &task.run_id).await;
|
|
649
649
|
}
|
|
650
|
+
|
|
651
|
+
#[tokio::test]
|
|
652
|
+
async fn activity_cancelled_after_heartbeat_times_out() {
|
|
653
|
+
let test_name = "activity_cancelled_after_heartbeat_times_out";
|
|
654
|
+
let (core, task_q) = init_core_and_create_wf(test_name).await;
|
|
655
|
+
let activity_id = "act-1";
|
|
656
|
+
let task = core.poll_workflow_activation(&task_q).await.unwrap();
|
|
657
|
+
// Complete workflow task and schedule activity
|
|
658
|
+
core.complete_workflow_activation(
|
|
659
|
+
schedule_activity_cmd(
|
|
660
|
+
0,
|
|
661
|
+
&task_q,
|
|
662
|
+
activity_id,
|
|
663
|
+
ActivityCancellationType::WaitCancellationCompleted,
|
|
664
|
+
Duration::from_secs(60),
|
|
665
|
+
Duration::from_secs(1),
|
|
666
|
+
)
|
|
667
|
+
.into_completion(task_q.clone(), task.run_id),
|
|
668
|
+
)
|
|
669
|
+
.await
|
|
670
|
+
.unwrap();
|
|
671
|
+
// Poll activity and verify that it's been scheduled with correct parameters
|
|
672
|
+
let task = core.poll_activity_task(&task_q).await.unwrap();
|
|
673
|
+
assert_matches!(
|
|
674
|
+
task.variant,
|
|
675
|
+
Some(act_task::Variant::Start(start_activity)) => {
|
|
676
|
+
assert_eq!(start_activity.activity_type, "test_activity".to_string())
|
|
677
|
+
}
|
|
678
|
+
);
|
|
679
|
+
// Delay the heartbeat
|
|
680
|
+
sleep(Duration::from_secs(2)).await;
|
|
681
|
+
core.record_activity_heartbeat(ActivityHeartbeat {
|
|
682
|
+
task_token: task.task_token.clone(),
|
|
683
|
+
task_queue: task_q.to_string(),
|
|
684
|
+
details: vec![],
|
|
685
|
+
});
|
|
686
|
+
|
|
687
|
+
// Verify activity got cancelled
|
|
688
|
+
let cancel_task = core.poll_activity_task(&task_q).await.unwrap();
|
|
689
|
+
assert_eq!(cancel_task.task_token, task.task_token.clone());
|
|
690
|
+
assert_matches!(cancel_task.variant, Some(act_task::Variant::Cancel(_)));
|
|
691
|
+
|
|
692
|
+
// Complete activity with cancelled result
|
|
693
|
+
core.complete_activity_task(ActivityTaskCompletion {
|
|
694
|
+
task_token: task.task_token.clone(),
|
|
695
|
+
task_queue: task_q.to_string(),
|
|
696
|
+
result: Some(ActivityResult::cancel_from_details(None)),
|
|
697
|
+
})
|
|
698
|
+
.await
|
|
699
|
+
.unwrap();
|
|
700
|
+
|
|
701
|
+
// Verify shutdown completes
|
|
702
|
+
core.shutdown_worker(task_q.as_str()).await;
|
|
703
|
+
core.shutdown().await;
|
|
704
|
+
// Cleanup just in case
|
|
705
|
+
core.server_gateway()
|
|
706
|
+
.terminate_workflow_execution(test_name.to_string(), None)
|
|
707
|
+
.await
|
|
708
|
+
.unwrap();
|
|
709
|
+
}
|
|
@@ -20,9 +20,9 @@ async fn parent_wf(mut ctx: WfContext) -> WorkflowResult<()> {
|
|
|
20
20
|
let started = child
|
|
21
21
|
.start(&mut ctx)
|
|
22
22
|
.await
|
|
23
|
-
.
|
|
23
|
+
.into_started()
|
|
24
24
|
.expect("Child chould start OK");
|
|
25
|
-
match started.result(
|
|
25
|
+
match started.result().await.status {
|
|
26
26
|
Some(child_workflow_result::Status::Completed(Success { .. })) => Ok(().into()),
|
|
27
27
|
_ => Err(anyhow!("Unexpected child WF status")),
|
|
28
28
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
use std::{
|
|
2
|
+
sync::atomic::{AtomicUsize, Ordering},
|
|
3
|
+
time::Duration,
|
|
4
|
+
};
|
|
5
|
+
use temporal_sdk_core::prototype_rust_sdk::{ActivityOptions, WfContext, WorkflowResult};
|
|
6
|
+
use test_utils::CoreWfStarter;
|
|
7
|
+
|
|
8
|
+
static RUN_CT: AtomicUsize = AtomicUsize::new(1);
|
|
9
|
+
pub async fn timer_wf_nondeterministic(mut ctx: WfContext) -> WorkflowResult<()> {
|
|
10
|
+
let run_ct = RUN_CT.fetch_add(1, Ordering::Relaxed);
|
|
11
|
+
|
|
12
|
+
match run_ct {
|
|
13
|
+
1 | 3 => {
|
|
14
|
+
// If we have not run yet or are on the third attempt, schedule a timer
|
|
15
|
+
ctx.timer(Duration::from_secs(1)).await;
|
|
16
|
+
if run_ct == 1 {
|
|
17
|
+
// on first attempt we need to blow up after the timer fires so we will replay
|
|
18
|
+
panic!("dying on purpose");
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
2 => {
|
|
22
|
+
// On the second attempt we should cause a nondeterminism error
|
|
23
|
+
ctx.activity(ActivityOptions {
|
|
24
|
+
activity_type: "whatever".to_string(),
|
|
25
|
+
..Default::default()
|
|
26
|
+
})
|
|
27
|
+
.await;
|
|
28
|
+
}
|
|
29
|
+
_ => panic!("Ran too many times"),
|
|
30
|
+
}
|
|
31
|
+
Ok(().into())
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
#[tokio::test]
|
|
35
|
+
async fn test_determinism_error_then_recovers() {
|
|
36
|
+
let wf_name = "test_determinism_error_then_recovers";
|
|
37
|
+
let mut starter = CoreWfStarter::new(wf_name);
|
|
38
|
+
let mut worker = starter.worker().await;
|
|
39
|
+
|
|
40
|
+
worker.register_wf(wf_name.to_owned(), timer_wf_nondeterministic);
|
|
41
|
+
worker
|
|
42
|
+
.submit_wf(wf_name.to_owned(), wf_name.to_owned(), vec![])
|
|
43
|
+
.await
|
|
44
|
+
.unwrap();
|
|
45
|
+
worker.run_until_done().await.unwrap();
|
|
46
|
+
starter.shutdown().await;
|
|
47
|
+
// 4 because we still add on the 3rd and final attempt
|
|
48
|
+
assert_eq!(RUN_CT.load(Ordering::Relaxed), 4);
|
|
49
|
+
}
|
|
@@ -78,13 +78,13 @@ async fn signals_child(mut ctx: WfContext) -> WorkflowResult<()> {
|
|
|
78
78
|
})
|
|
79
79
|
.start(&mut ctx)
|
|
80
80
|
.await
|
|
81
|
-
.
|
|
81
|
+
.into_started()
|
|
82
82
|
.expect("Must start ok");
|
|
83
83
|
started_child
|
|
84
84
|
.signal(&mut ctx, SIGNAME, b"hiya!")
|
|
85
85
|
.await
|
|
86
86
|
.unwrap();
|
|
87
|
-
started_child.result(
|
|
87
|
+
started_child.result().await.status.unwrap();
|
|
88
88
|
Ok(().into())
|
|
89
89
|
}
|
|
90
90
|
|