@temporalio/core-bridge 0.14.0 → 0.16.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/Cargo.lock +162 -38
  2. package/Cargo.toml +3 -3
  3. package/index.d.ts +14 -1
  4. package/index.node +0 -0
  5. package/package.json +8 -5
  6. package/releases/aarch64-apple-darwin/index.node +0 -0
  7. package/releases/{x86_64-pc-windows-gnu → aarch64-unknown-linux-gnu}/index.node +0 -0
  8. package/releases/x86_64-apple-darwin/index.node +0 -0
  9. package/releases/x86_64-pc-windows-msvc/index.node +0 -0
  10. package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
  11. package/scripts/build.js +77 -34
  12. package/sdk-core/.buildkite/docker/Dockerfile +1 -1
  13. package/sdk-core/Cargo.toml +6 -5
  14. package/sdk-core/fsm/Cargo.toml +1 -1
  15. package/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +2 -2
  16. package/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +8 -9
  17. package/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +13 -7
  18. package/sdk-core/fsm/rustfsm_trait/Cargo.toml +2 -2
  19. package/sdk-core/fsm/rustfsm_trait/src/lib.rs +1 -1
  20. package/sdk-core/protos/local/workflow_activation.proto +6 -3
  21. package/sdk-core/sdk-core-protos/Cargo.toml +4 -4
  22. package/sdk-core/sdk-core-protos/src/lib.rs +38 -50
  23. package/sdk-core/src/core_tests/activity_tasks.rs +5 -5
  24. package/sdk-core/src/core_tests/child_workflows.rs +55 -29
  25. package/sdk-core/src/core_tests/determinism.rs +19 -9
  26. package/sdk-core/src/core_tests/mod.rs +3 -3
  27. package/sdk-core/src/core_tests/retry.rs +14 -8
  28. package/sdk-core/src/core_tests/workers.rs +1 -1
  29. package/sdk-core/src/core_tests/workflow_tasks.rs +347 -4
  30. package/sdk-core/src/errors.rs +27 -44
  31. package/sdk-core/src/lib.rs +13 -3
  32. package/sdk-core/src/machines/activity_state_machine.rs +44 -5
  33. package/sdk-core/src/machines/child_workflow_state_machine.rs +31 -11
  34. package/sdk-core/src/machines/complete_workflow_state_machine.rs +1 -1
  35. package/sdk-core/src/machines/continue_as_new_workflow_state_machine.rs +1 -1
  36. package/sdk-core/src/machines/mod.rs +18 -23
  37. package/sdk-core/src/machines/patch_state_machine.rs +8 -8
  38. package/sdk-core/src/machines/signal_external_state_machine.rs +22 -1
  39. package/sdk-core/src/machines/timer_state_machine.rs +21 -3
  40. package/sdk-core/src/machines/transition_coverage.rs +3 -3
  41. package/sdk-core/src/machines/workflow_machines.rs +11 -11
  42. package/sdk-core/src/pending_activations.rs +27 -22
  43. package/sdk-core/src/pollers/gateway.rs +15 -7
  44. package/sdk-core/src/pollers/poll_buffer.rs +6 -5
  45. package/sdk-core/src/pollers/retry.rs +153 -120
  46. package/sdk-core/src/prototype_rust_sdk/workflow_context.rs +61 -46
  47. package/sdk-core/src/prototype_rust_sdk/workflow_future.rs +13 -12
  48. package/sdk-core/src/prototype_rust_sdk.rs +17 -23
  49. package/sdk-core/src/telemetry/metrics.rs +2 -4
  50. package/sdk-core/src/telemetry/mod.rs +6 -7
  51. package/sdk-core/src/test_help/canned_histories.rs +17 -93
  52. package/sdk-core/src/test_help/history_builder.rs +61 -2
  53. package/sdk-core/src/test_help/history_info.rs +21 -2
  54. package/sdk-core/src/test_help/mod.rs +26 -34
  55. package/sdk-core/src/worker/activities/activity_heartbeat_manager.rs +246 -138
  56. package/sdk-core/src/worker/activities.rs +46 -45
  57. package/sdk-core/src/worker/config.rs +11 -0
  58. package/sdk-core/src/worker/dispatcher.rs +5 -5
  59. package/sdk-core/src/worker/mod.rs +86 -56
  60. package/sdk-core/src/workflow/driven_workflow.rs +3 -3
  61. package/sdk-core/src/workflow/history_update.rs +1 -1
  62. package/sdk-core/src/workflow/mod.rs +2 -1
  63. package/sdk-core/src/workflow/workflow_tasks/cache_manager.rs +13 -17
  64. package/sdk-core/src/workflow/workflow_tasks/concurrency_manager.rs +10 -18
  65. package/sdk-core/src/workflow/workflow_tasks/mod.rs +72 -57
  66. package/sdk-core/test_utils/Cargo.toml +1 -1
  67. package/sdk-core/test_utils/src/lib.rs +2 -2
  68. package/sdk-core/tests/integ_tests/workflow_tests/activities.rs +61 -1
  69. package/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +2 -2
  70. package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +49 -0
  71. package/sdk-core/tests/integ_tests/workflow_tests/signals.rs +2 -2
  72. package/sdk-core/tests/integ_tests/workflow_tests.rs +1 -0
  73. package/src/conversions.rs +17 -0
  74. package/src/errors.rs +0 -7
  75. package/src/lib.rs +0 -20
@@ -35,17 +35,13 @@ impl WorkflowCacheManager {
35
35
  // Blindly add a record into the cache, since it still has capacity.
36
36
  self.cache.put(run_id.to_owned(), ());
37
37
  None
38
- } else if self.cache.cap() != 0 {
39
- let maybe_got_evicted = self.cache.peek_lru().map(|r| r.0.to_owned());
40
- let already_existed = self.cache.put(run_id.to_owned(), ()).is_some();
41
- if !already_existed {
42
- maybe_got_evicted
43
- } else {
44
- None
45
- }
46
- } else {
38
+ } else if self.cache.cap() == 0 {
47
39
  // Run id should be evicted right away as cache size is 0.
48
40
  Some(run_id.to_owned())
41
+ } else {
42
+ let maybe_got_evicted = self.cache.peek_lru().map(|r| r.0.clone());
43
+ let not_cached = self.cache.put(run_id.to_owned(), ()).is_none();
44
+ not_cached.then(|| maybe_got_evicted).flatten()
49
45
  };
50
46
 
51
47
  self.metrics.cache_size(self.cache.len() as u64);
@@ -75,7 +71,7 @@ mod tests {
75
71
  assert_matches!(wcm.insert("1"), None);
76
72
  assert_matches!(wcm.insert("2"), None);
77
73
  assert_matches!(wcm.insert("3"), Some(run_id) => {
78
- assert_eq!(run_id, "1")
74
+ assert_eq!(run_id, "1");
79
75
  });
80
76
  }
81
77
 
@@ -88,7 +84,7 @@ mod tests {
88
84
  wcm.remove("1");
89
85
  assert_matches!(wcm.insert("2"), None);
90
86
  assert_matches!(wcm.insert("3"), Some(run_id) => {
91
- assert_eq!(run_id, "2")
87
+ assert_eq!(run_id, "2");
92
88
  });
93
89
  }
94
90
 
@@ -110,7 +106,7 @@ mod tests {
110
106
  assert_matches!(wcm.insert("2"), None);
111
107
  wcm.touch("1");
112
108
  assert_matches!(wcm.insert("3"), Some(run_id) => {
113
- assert_eq!(run_id, "2")
109
+ assert_eq!(run_id, "2");
114
110
  });
115
111
  }
116
112
 
@@ -123,7 +119,7 @@ mod tests {
123
119
  assert_matches!(wcm.insert("1"), None);
124
120
  assert_matches!(wcm.insert("2"), None);
125
121
  assert_matches!(wcm.insert("3"), Some(run_id) => {
126
- assert_eq!(run_id, "1")
122
+ assert_eq!(run_id, "1");
127
123
  });
128
124
  }
129
125
 
@@ -133,10 +129,10 @@ mod tests {
133
129
  max_cached_workflows: 0,
134
130
  });
135
131
  assert_matches!(wcm.insert("1"), Some(run_id) => {
136
- assert_eq!(run_id, "1")
132
+ assert_eq!(run_id, "1");
137
133
  });
138
134
  assert_matches!(wcm.insert("2"), Some(run_id) => {
139
- assert_eq!(run_id, "2")
135
+ assert_eq!(run_id, "2");
140
136
  });
141
137
  }
142
138
 
@@ -144,10 +140,10 @@ mod tests {
144
140
  fn non_sticky_always_pending_eviction() {
145
141
  let mut wcm = WorkflowCacheManager::new_test(WorkflowCachingPolicy::NonSticky);
146
142
  assert_matches!(wcm.insert("1"), Some(run_id) => {
147
- assert_eq!(run_id, "1")
143
+ assert_eq!(run_id, "1");
148
144
  });
149
145
  assert_matches!(wcm.insert("2"), Some(run_id) => {
150
- assert_eq!(run_id, "2")
146
+ assert_eq!(run_id, "2");
151
147
  });
152
148
  }
153
149
  }
@@ -1,5 +1,5 @@
1
1
  use crate::{
2
- errors::WorkflowUpdateError,
2
+ errors::WorkflowMissingError,
3
3
  protosext::ValidPollWFTQResponse,
4
4
  telemetry::metrics::{workflow_type, MetricsContext},
5
5
  workflow::{
@@ -89,7 +89,7 @@ impl WorkflowConcurrencyManager {
89
89
  pub(crate) fn get_task_mut(
90
90
  &self,
91
91
  run_id: &str,
92
- ) -> Result<impl DerefMut<Target = Option<OutstandingTask>> + '_, WorkflowUpdateError> {
92
+ ) -> Result<impl DerefMut<Target = Option<OutstandingTask>> + '_, WorkflowMissingError> {
93
93
  let writelock = self.runs.write();
94
94
  if writelock.contains_key(run_id) {
95
95
  Ok(RwLockWriteGuard::map(writelock, |hm| {
@@ -97,10 +97,8 @@ impl WorkflowConcurrencyManager {
97
97
  &mut hm.get_mut(run_id).unwrap().wft
98
98
  }))
99
99
  } else {
100
- Err(WorkflowUpdateError {
101
- source: WFMachinesError::Fatal("Workflow machines not found".to_string()),
100
+ Err(WorkflowMissingError {
102
101
  run_id: run_id.to_owned(),
103
- task_token: None,
104
102
  })
105
103
  }
106
104
  }
@@ -146,7 +144,7 @@ impl WorkflowConcurrencyManager {
146
144
  &self,
147
145
  run_id: &str,
148
146
  task: OutstandingTask,
149
- ) -> Result<(), WorkflowUpdateError> {
147
+ ) -> Result<(), WorkflowMissingError> {
150
148
  let mut dereffer = self.get_task_mut(run_id)?;
151
149
  *dereffer = Some(task);
152
150
  Ok(())
@@ -171,16 +169,14 @@ impl WorkflowConcurrencyManager {
171
169
  &self,
172
170
  run_id: &str,
173
171
  activation: OutstandingActivation,
174
- ) -> Result<Option<OutstandingActivation>, WorkflowUpdateError> {
172
+ ) -> Result<Option<OutstandingActivation>, WorkflowMissingError> {
175
173
  let mut writelock = self.runs.write();
176
174
  let machine_ref = writelock.get_mut(run_id);
177
175
  if let Some(run) = machine_ref {
178
176
  Ok(run.activation.replace(activation))
179
177
  } else {
180
- Err(WorkflowUpdateError {
181
- source: WFMachinesError::Fatal("Workflow machines not found".to_string()),
178
+ Err(WorkflowMissingError {
182
179
  run_id: run_id.to_owned(),
183
- task_token: None,
184
180
  })
185
181
  }
186
182
  }
@@ -188,11 +184,7 @@ impl WorkflowConcurrencyManager {
188
184
  pub fn delete_activation(&self, run_id: &str) -> Option<OutstandingActivation> {
189
185
  let mut writelock = self.runs.write();
190
186
  let machine_ref = writelock.get_mut(run_id);
191
- if let Some(run) = machine_ref {
192
- run.activation.take()
193
- } else {
194
- None
195
- }
187
+ machine_ref.and_then(|run| run.activation.take())
196
188
  }
197
189
 
198
190
  pub fn exists(&self, run_id: &str) -> bool {
@@ -274,14 +266,14 @@ impl WorkflowConcurrencyManager {
274
266
  /// Remove the workflow with the provided run id from management
275
267
  pub fn evict(&self, run_id: &str) -> Option<ValidPollWFTQResponse> {
276
268
  let val = self.runs.write().remove(run_id);
277
- val.map(|v| v.buffered_resp).flatten()
269
+ val.and_then(|v| v.buffered_resp)
278
270
  }
279
271
 
280
272
  /// Clear and return any buffered polling response for this run ID
281
273
  pub fn take_buffered_poll(&self, run_id: &str) -> Option<ValidPollWFTQResponse> {
282
274
  let mut writelock = self.runs.write();
283
275
  let val = writelock.get_mut(run_id);
284
- val.map(|v| v.buffered_resp.take()).flatten()
276
+ val.and_then(|v| v.buffered_resp.take())
285
277
  }
286
278
 
287
279
  pub fn outstanding_wft(&self) -> usize {
@@ -315,6 +307,6 @@ mod tests {
315
307
  )
316
308
  .await;
317
309
  // Should whine that the machines have nothing to do (history empty)
318
- assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. })
310
+ assert_matches!(res.unwrap_err(), WFMachinesError::Fatal { .. });
319
311
  }
320
312
  }
@@ -4,7 +4,7 @@ mod cache_manager;
4
4
  mod concurrency_manager;
5
5
 
6
6
  use crate::{
7
- errors::WorkflowUpdateError,
7
+ errors::{WorkflowMissingError, WorkflowUpdateError},
8
8
  machines::{ProtoCommand, WFCommand, WFMachinesError},
9
9
  pending_activations::PendingActivations,
10
10
  pollers::GatewayRef,
@@ -21,7 +21,7 @@ use crate::{
21
21
  use crossbeam::queue::SegQueue;
22
22
  use futures::FutureExt;
23
23
  use parking_lot::Mutex;
24
- use std::{fmt::Debug, ops::DerefMut, time::Instant};
24
+ use std::{fmt::Debug, time::Instant};
25
25
  use temporal_sdk_core_protos::coresdk::{
26
26
  workflow_activation::{
27
27
  create_evict_activation, create_query_activation, wf_activation_job, QueryWorkflow,
@@ -77,9 +77,9 @@ pub(crate) enum OutstandingActivation {
77
77
  }
78
78
 
79
79
  impl OutstandingActivation {
80
- fn has_eviction(&self) -> bool {
80
+ const fn has_eviction(self) -> bool {
81
81
  matches!(
82
- &self,
82
+ self,
83
83
  OutstandingActivation::Normal {
84
84
  contains_eviction: true
85
85
  }
@@ -96,7 +96,7 @@ pub struct WorkflowTaskInfo {
96
96
  }
97
97
 
98
98
  #[derive(Debug, derive_more::From)]
99
- pub enum NewWfTaskOutcome {
99
+ pub(crate) enum NewWfTaskOutcome {
100
100
  /// A new activation for the workflow should be issued to lang
101
101
  IssueActivation(WfActivation),
102
102
  /// The poll loop should be restarted, there is nothing to do
@@ -106,6 +106,8 @@ pub enum NewWfTaskOutcome {
106
106
  Autocomplete,
107
107
  /// Workflow task had partial history and workflow was not present in the cache.
108
108
  CacheMiss,
109
+ /// The workflow task ran into problems while being applied and we must now evict the workflow
110
+ Evict(WorkflowUpdateError),
109
111
  }
110
112
 
111
113
  #[derive(Debug)]
@@ -162,9 +164,7 @@ impl WorkflowTaskManager {
162
164
  }
163
165
  }
164
166
 
165
- pub(crate) fn next_pending_activation(
166
- &self,
167
- ) -> Result<Option<WfActivation>, WorkflowUpdateError> {
167
+ pub(crate) fn next_pending_activation(&self) -> Option<WfActivation> {
168
168
  // It is important that we do not issue pending activations for any workflows which already
169
169
  // have an outstanding activation. If we did, it can result in races where an in-progress
170
170
  // completion may appear to be the last in a task (no more pending activations) because
@@ -174,10 +174,14 @@ impl WorkflowTaskManager {
174
174
  .pending_activations
175
175
  .pop_first_matching(|rid| self.workflow_machines.get_activation(rid).is_none());
176
176
  if let Some(act) = maybe_act.as_ref() {
177
- self.insert_outstanding_activation(act)?;
177
+ if let Err(WorkflowMissingError { run_id }) = self.insert_outstanding_activation(act) {
178
+ self.request_eviction(&run_id, "Pending activation present for missing run");
179
+ // Continue trying to return a valid pending activation
180
+ return self.next_pending_activation();
181
+ }
178
182
  self.cache_manager.lock().touch(&act.run_id);
179
183
  }
180
- Ok(maybe_act)
184
+ maybe_act
181
185
  }
182
186
 
183
187
  pub fn next_buffered_poll(&self) -> Option<ValidPollWFTQResponse> {
@@ -192,13 +196,13 @@ impl WorkflowTaskManager {
192
196
  /// the lang side. Workflow will not *actually* be evicted until lang replies to that activation
193
197
  ///
194
198
  /// Returns, if found, the number of attempts on the current workflow task
195
- pub fn request_eviction(&self, run_id: &str) -> Option<u32> {
199
+ pub fn request_eviction(&self, run_id: &str, reason: impl Into<String>) -> Option<u32> {
196
200
  if self.workflow_machines.exists(run_id) {
197
201
  if !self.activation_has_eviction(run_id) {
198
202
  debug!(%run_id, "Eviction requested");
199
203
  // Queue up an eviction activation
200
204
  self.pending_activations
201
- .push(create_evict_activation(run_id.to_string()));
205
+ .push(create_evict_activation(run_id.to_string(), reason.into()));
202
206
  let _ = self.pending_activations_notifier.send(true);
203
207
  }
204
208
  self.workflow_machines
@@ -241,12 +245,12 @@ impl WorkflowTaskManager {
241
245
  &self,
242
246
  work: ValidPollWFTQResponse,
243
247
  gateway: &GatewayRef,
244
- ) -> Result<NewWfTaskOutcome, WorkflowUpdateError> {
248
+ ) -> NewWfTaskOutcome {
245
249
  let mut work = if let Some(w) = self.workflow_machines.buffer_resp_if_outstanding_work(work)
246
250
  {
247
251
  w
248
252
  } else {
249
- return Ok(NewWfTaskOutcome::TaskBuffered);
253
+ return NewWfTaskOutcome::TaskBuffered;
250
254
  };
251
255
 
252
256
  debug!(
@@ -269,9 +273,9 @@ impl WorkflowTaskManager {
269
273
  Ok((info, next_activation)) => (info, next_activation),
270
274
  Err(e) => {
271
275
  if let WFMachinesError::CacheMiss = e.source {
272
- return Ok(NewWfTaskOutcome::CacheMiss);
276
+ return NewWfTaskOutcome::CacheMiss;
273
277
  }
274
- return Err(e);
278
+ return NewWfTaskOutcome::Evict(e);
275
279
  }
276
280
  };
277
281
 
@@ -288,20 +292,24 @@ impl WorkflowTaskManager {
288
292
  legacy_query
289
293
  };
290
294
 
291
- self.workflow_machines.insert_wft(
292
- &next_activation.run_id,
293
- OutstandingTask {
294
- info,
295
- legacy_query,
296
- start_time: task_start_time,
297
- },
298
- )?;
299
-
300
- if !next_activation.jobs.is_empty() {
301
- self.insert_outstanding_activation(&next_activation)?;
302
- Ok(NewWfTaskOutcome::IssueActivation(next_activation))
295
+ self.workflow_machines
296
+ .insert_wft(
297
+ &next_activation.run_id,
298
+ OutstandingTask {
299
+ info,
300
+ legacy_query,
301
+ start_time: task_start_time,
302
+ },
303
+ )
304
+ .expect("Workflow machines must exist, we just created/updated them");
305
+
306
+ if next_activation.jobs.is_empty() {
307
+ NewWfTaskOutcome::Autocomplete
303
308
  } else {
304
- Ok(NewWfTaskOutcome::Autocomplete)
309
+ if let Err(wme) = self.insert_outstanding_activation(&next_activation) {
310
+ return NewWfTaskOutcome::Evict(wme.into());
311
+ }
312
+ NewWfTaskOutcome::IssueActivation(next_activation)
305
313
  }
306
314
  }
307
315
 
@@ -398,7 +406,9 @@ impl WorkflowTaskManager {
398
406
  query_responses,
399
407
  },
400
408
  })
401
- } else if !query_responses.is_empty() {
409
+ } else if query_responses.is_empty() {
410
+ None
411
+ } else {
402
412
  Some(ServerCommandsWithWorkflowInfo {
403
413
  task_token,
404
414
  action: ActivationAction::WftComplete {
@@ -406,8 +416,6 @@ impl WorkflowTaskManager {
406
416
  query_responses,
407
417
  },
408
418
  })
409
- } else {
410
- None
411
419
  }
412
420
  };
413
421
  Ok(ret)
@@ -439,12 +447,9 @@ impl WorkflowTaskManager {
439
447
  FailedActivationOutcome::ReportLegacyQueryFailure(tt)
440
448
  } else {
441
449
  // Blow up any cached data associated with the workflow
442
- let should_report = if let Some(attempt) = self.request_eviction(run_id) {
443
- // Only report to server if the last task wasn't also a failure (avoid spam)
444
- attempt <= 1
445
- } else {
446
- true
447
- };
450
+ let should_report = self
451
+ .request_eviction(run_id, "Activation failed by lang")
452
+ .map_or(true, |attempt| attempt <= 1);
448
453
  if should_report {
449
454
  FailedActivationOutcome::Report(tt)
450
455
  } else {
@@ -516,7 +521,7 @@ impl WorkflowTaskManager {
516
521
  /// if this is called too early.
517
522
  ///
518
523
  /// Returns true if WFT is complete
519
- pub(crate) fn after_wft_report(&self, run_id: &str) -> Result<bool, WorkflowUpdateError> {
524
+ pub(crate) fn after_wft_report(&self, run_id: &str) -> bool {
520
525
  let mut just_evicted = false;
521
526
 
522
527
  if let Some(OutstandingActivation::Normal {
@@ -532,19 +537,23 @@ impl WorkflowTaskManager {
532
537
  if !just_evicted {
533
538
  // Check if there was a legacy query which must be fulfilled, and if there is create
534
539
  // a new pending activation for it.
535
- if let Some(ref mut ot) = self.workflow_machines.get_task_mut(run_id)?.deref_mut() {
540
+ if let Some(ref mut ot) = &mut *self
541
+ .workflow_machines
542
+ .get_task_mut(run_id)
543
+ .expect("Machine must exist")
544
+ {
536
545
  if let Some(query) = ot.legacy_query.take() {
537
546
  let na = create_query_activation(run_id.to_string(), [query]);
538
547
  self.pending_activations.push(na);
539
548
  let _ = self.pending_activations_notifier.send(true);
540
- return Ok(false);
549
+ return false;
541
550
  }
542
551
  }
543
552
 
544
553
  // Evict run id if cache is full. Non-sticky will always evict.
545
554
  let maybe_evicted = self.cache_manager.lock().insert(run_id);
546
555
  if let Some(evicted_run_id) = maybe_evicted {
547
- self.request_eviction(&evicted_run_id);
556
+ self.request_eviction(&evicted_run_id, "Workflow cache full");
548
557
  }
549
558
 
550
559
  // If there was a buffered poll response from the server, it is now ready to
@@ -556,9 +565,9 @@ impl WorkflowTaskManager {
556
565
 
557
566
  // The evict may or may not have already done this, but even when we aren't evicting
558
567
  // we want to clear the outstanding workflow task since it's now complete.
559
- return Ok(self.workflow_machines.complete_wft(run_id).is_some());
568
+ return self.workflow_machines.complete_wft(run_id).is_some();
560
569
  }
561
- Ok(false)
570
+ false
562
571
  }
563
572
 
564
573
  /// Must be called after *every* activation is replied to, regardless of whether or not we
@@ -577,7 +586,10 @@ impl WorkflowTaskManager {
577
586
  self.ready_buffered_wft.push(buffd);
578
587
  }
579
588
 
580
- fn insert_outstanding_activation(&self, act: &WfActivation) -> Result<(), WorkflowUpdateError> {
589
+ fn insert_outstanding_activation(
590
+ &self,
591
+ act: &WfActivation,
592
+ ) -> Result<(), WorkflowMissingError> {
581
593
  let act_type = if act.is_legacy_query() {
582
594
  OutstandingActivation::LegacyQuery
583
595
  } else {
@@ -585,25 +597,28 @@ impl WorkflowTaskManager {
585
597
  contains_eviction: act.eviction_index().is_some(),
586
598
  }
587
599
  };
588
- let previous = self
600
+ match self
589
601
  .workflow_machines
590
- .insert_activation(&act.run_id, act_type)?;
591
- if let Some(previous) = previous {
592
- // This isn't a panic because we have screwed up core logic if this is violated. It
593
- // must be upheld.
594
- panic!(
595
- "Attempted to insert a new outstanding activation {}, but there already was one \
596
- outstanding: {:?}",
597
- act, previous
598
- );
602
+ .insert_activation(&act.run_id, act_type)
603
+ {
604
+ Ok(None) => Ok(()),
605
+ Ok(Some(previous)) => {
606
+ // This is a panic because we have screwed up core logic if this is violated. It
607
+ // must be upheld.
608
+ panic!(
609
+ "Attempted to insert a new outstanding activation {}, but there already was \
610
+ one outstanding: {:?}",
611
+ act, previous
612
+ );
613
+ }
614
+ Err(e) => Err(e),
599
615
  }
600
- Ok(())
601
616
  }
602
617
 
603
618
  fn activation_has_eviction(&self, run_id: &str) -> bool {
604
619
  self.workflow_machines
605
620
  .get_activation(run_id)
606
- .map(|oa| oa.has_eviction())
621
+ .map(OutstandingActivation::has_eviction)
607
622
  .unwrap_or_default()
608
623
  }
609
624
  }
@@ -2,7 +2,7 @@
2
2
  name = "test_utils"
3
3
  version = "0.1.0"
4
4
  authors = ["Spencer Judge <spencer@temporal.io>"]
5
- edition = "2018"
5
+ edition = "2021"
6
6
 
7
7
  # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
8
8
 
@@ -46,7 +46,7 @@ impl CoreWfStarter {
46
46
  .unwrap(),
47
47
  worker_config: WorkerConfigBuilder::default()
48
48
  .task_queue(task_queue)
49
- .max_cached_workflows(1000usize)
49
+ .max_cached_workflows(1000_usize)
50
50
  .build()
51
51
  .unwrap(),
52
52
  wft_timeout: None,
@@ -63,7 +63,7 @@ impl CoreWfStarter {
63
63
  }
64
64
 
65
65
  pub async fn shutdown(&mut self) {
66
- self.get_core().await.shutdown().await
66
+ self.get_core().await.shutdown().await;
67
67
  }
68
68
 
69
69
  pub async fn get_core(&mut self) -> Arc<dyn Core> {
@@ -8,7 +8,7 @@ use temporal_sdk_core_protos::{
8
8
  workflow_activation::{wf_activation_job, FireTimer, ResolveActivity, WfActivationJob},
9
9
  workflow_commands::{ActivityCancellationType, RequestCancelActivity, StartTimer},
10
10
  workflow_completion::WfActivationCompletion,
11
- ActivityTaskCompletion, IntoCompletion,
11
+ ActivityHeartbeat, ActivityTaskCompletion, IntoCompletion,
12
12
  },
13
13
  temporal::api::{
14
14
  common::v1::{ActivityType, Payloads},
@@ -647,3 +647,63 @@ async fn async_activity_completion_workflow() {
647
647
  );
648
648
  core.complete_execution(&task_q, &task.run_id).await;
649
649
  }
650
+
651
+ #[tokio::test]
652
+ async fn activity_cancelled_after_heartbeat_times_out() {
653
+ let test_name = "activity_cancelled_after_heartbeat_times_out";
654
+ let (core, task_q) = init_core_and_create_wf(test_name).await;
655
+ let activity_id = "act-1";
656
+ let task = core.poll_workflow_activation(&task_q).await.unwrap();
657
+ // Complete workflow task and schedule activity
658
+ core.complete_workflow_activation(
659
+ schedule_activity_cmd(
660
+ 0,
661
+ &task_q,
662
+ activity_id,
663
+ ActivityCancellationType::WaitCancellationCompleted,
664
+ Duration::from_secs(60),
665
+ Duration::from_secs(1),
666
+ )
667
+ .into_completion(task_q.clone(), task.run_id),
668
+ )
669
+ .await
670
+ .unwrap();
671
+ // Poll activity and verify that it's been scheduled with correct parameters
672
+ let task = core.poll_activity_task(&task_q).await.unwrap();
673
+ assert_matches!(
674
+ task.variant,
675
+ Some(act_task::Variant::Start(start_activity)) => {
676
+ assert_eq!(start_activity.activity_type, "test_activity".to_string())
677
+ }
678
+ );
679
+ // Delay the heartbeat
680
+ sleep(Duration::from_secs(2)).await;
681
+ core.record_activity_heartbeat(ActivityHeartbeat {
682
+ task_token: task.task_token.clone(),
683
+ task_queue: task_q.to_string(),
684
+ details: vec![],
685
+ });
686
+
687
+ // Verify activity got cancelled
688
+ let cancel_task = core.poll_activity_task(&task_q).await.unwrap();
689
+ assert_eq!(cancel_task.task_token, task.task_token.clone());
690
+ assert_matches!(cancel_task.variant, Some(act_task::Variant::Cancel(_)));
691
+
692
+ // Complete activity with cancelled result
693
+ core.complete_activity_task(ActivityTaskCompletion {
694
+ task_token: task.task_token.clone(),
695
+ task_queue: task_q.to_string(),
696
+ result: Some(ActivityResult::cancel_from_details(None)),
697
+ })
698
+ .await
699
+ .unwrap();
700
+
701
+ // Verify shutdown completes
702
+ core.shutdown_worker(task_q.as_str()).await;
703
+ core.shutdown().await;
704
+ // Cleanup just in case
705
+ core.server_gateway()
706
+ .terminate_workflow_execution(test_name.to_string(), None)
707
+ .await
708
+ .unwrap();
709
+ }
@@ -20,9 +20,9 @@ async fn parent_wf(mut ctx: WfContext) -> WorkflowResult<()> {
20
20
  let started = child
21
21
  .start(&mut ctx)
22
22
  .await
23
- .as_started()
23
+ .into_started()
24
24
  .expect("Child chould start OK");
25
- match started.result(&mut ctx).await.status {
25
+ match started.result().await.status {
26
26
  Some(child_workflow_result::Status::Completed(Success { .. })) => Ok(().into()),
27
27
  _ => Err(anyhow!("Unexpected child WF status")),
28
28
  }
@@ -0,0 +1,49 @@
1
+ use std::{
2
+ sync::atomic::{AtomicUsize, Ordering},
3
+ time::Duration,
4
+ };
5
+ use temporal_sdk_core::prototype_rust_sdk::{ActivityOptions, WfContext, WorkflowResult};
6
+ use test_utils::CoreWfStarter;
7
+
8
+ static RUN_CT: AtomicUsize = AtomicUsize::new(1);
9
+ pub async fn timer_wf_nondeterministic(mut ctx: WfContext) -> WorkflowResult<()> {
10
+ let run_ct = RUN_CT.fetch_add(1, Ordering::Relaxed);
11
+
12
+ match run_ct {
13
+ 1 | 3 => {
14
+ // If we have not run yet or are on the third attempt, schedule a timer
15
+ ctx.timer(Duration::from_secs(1)).await;
16
+ if run_ct == 1 {
17
+ // on first attempt we need to blow up after the timer fires so we will replay
18
+ panic!("dying on purpose");
19
+ }
20
+ }
21
+ 2 => {
22
+ // On the second attempt we should cause a nondeterminism error
23
+ ctx.activity(ActivityOptions {
24
+ activity_type: "whatever".to_string(),
25
+ ..Default::default()
26
+ })
27
+ .await;
28
+ }
29
+ _ => panic!("Ran too many times"),
30
+ }
31
+ Ok(().into())
32
+ }
33
+
34
+ #[tokio::test]
35
+ async fn test_determinism_error_then_recovers() {
36
+ let wf_name = "test_determinism_error_then_recovers";
37
+ let mut starter = CoreWfStarter::new(wf_name);
38
+ let mut worker = starter.worker().await;
39
+
40
+ worker.register_wf(wf_name.to_owned(), timer_wf_nondeterministic);
41
+ worker
42
+ .submit_wf(wf_name.to_owned(), wf_name.to_owned(), vec![])
43
+ .await
44
+ .unwrap();
45
+ worker.run_until_done().await.unwrap();
46
+ starter.shutdown().await;
47
+ // 4 because we still add on the 3rd and final attempt
48
+ assert_eq!(RUN_CT.load(Ordering::Relaxed), 4);
49
+ }
@@ -78,13 +78,13 @@ async fn signals_child(mut ctx: WfContext) -> WorkflowResult<()> {
78
78
  })
79
79
  .start(&mut ctx)
80
80
  .await
81
- .as_started()
81
+ .into_started()
82
82
  .expect("Must start ok");
83
83
  started_child
84
84
  .signal(&mut ctx, SIGNAME, b"hiya!")
85
85
  .await
86
86
  .unwrap();
87
- started_child.result(&mut ctx).await.status.unwrap();
87
+ started_child.result().await.status.unwrap();
88
88
  Ok(().into())
89
89
  }
90
90
 
@@ -3,6 +3,7 @@ mod cancel_external;
3
3
  mod cancel_wf;
4
4
  mod child_workflows;
5
5
  mod continue_as_new;
6
+ mod determinism;
6
7
  mod patches;
7
8
  mod signals;
8
9
  mod stickyness;