@temporalio/core-bridge 1.11.8 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/Cargo.lock +219 -193
  2. package/Cargo.toml +27 -8
  3. package/README.md +5 -0
  4. package/index.js +72 -12
  5. package/lib/errors.d.ts +25 -0
  6. package/lib/errors.js +76 -1
  7. package/lib/errors.js.map +1 -1
  8. package/lib/index.d.ts +11 -478
  9. package/lib/index.js +28 -5
  10. package/lib/index.js.map +1 -1
  11. package/lib/native.d.ts +330 -0
  12. package/lib/{worker-tuner.js → native.js} +1 -1
  13. package/lib/native.js.map +1 -0
  14. package/package.json +7 -3
  15. package/releases/aarch64-apple-darwin/index.node +0 -0
  16. package/releases/aarch64-unknown-linux-gnu/index.node +0 -0
  17. package/releases/x86_64-apple-darwin/index.node +0 -0
  18. package/releases/x86_64-pc-windows-msvc/index.node +0 -0
  19. package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
  20. package/sdk-core/.cargo/config.toml +8 -2
  21. package/sdk-core/.cargo/multi-worker-manual-test +15 -0
  22. package/sdk-core/.github/workflows/per-pr.yml +40 -11
  23. package/sdk-core/AGENTS.md +73 -0
  24. package/sdk-core/ARCHITECTURE.md +71 -23
  25. package/sdk-core/Cargo.toml +1 -1
  26. package/sdk-core/README.md +4 -4
  27. package/sdk-core/arch_docs/workflow_task_chunking.md +51 -0
  28. package/sdk-core/client/Cargo.toml +1 -1
  29. package/sdk-core/client/src/lib.rs +49 -13
  30. package/sdk-core/client/src/metrics.rs +15 -16
  31. package/sdk-core/client/src/proxy.rs +2 -2
  32. package/sdk-core/client/src/raw.rs +54 -8
  33. package/sdk-core/client/src/retry.rs +109 -13
  34. package/sdk-core/client/src/worker_registry/mod.rs +4 -4
  35. package/sdk-core/client/src/workflow_handle/mod.rs +1 -1
  36. package/sdk-core/core/Cargo.toml +28 -8
  37. package/sdk-core/core/src/abstractions.rs +62 -10
  38. package/sdk-core/core/src/core_tests/activity_tasks.rs +180 -8
  39. package/sdk-core/core/src/core_tests/mod.rs +4 -4
  40. package/sdk-core/core/src/core_tests/queries.rs +18 -4
  41. package/sdk-core/core/src/core_tests/workers.rs +3 -3
  42. package/sdk-core/core/src/core_tests/workflow_tasks.rs +191 -25
  43. package/sdk-core/core/src/ephemeral_server/mod.rs +10 -3
  44. package/sdk-core/core/src/internal_flags.rs +14 -14
  45. package/sdk-core/core/src/lib.rs +5 -2
  46. package/sdk-core/core/src/pollers/mod.rs +1 -1
  47. package/sdk-core/core/src/pollers/poll_buffer.rs +495 -164
  48. package/sdk-core/core/src/protosext/mod.rs +3 -3
  49. package/sdk-core/core/src/replay/mod.rs +3 -3
  50. package/sdk-core/core/src/telemetry/metrics.rs +13 -4
  51. package/sdk-core/core/src/telemetry/mod.rs +72 -70
  52. package/sdk-core/core/src/telemetry/otel.rs +51 -54
  53. package/sdk-core/core/src/test_help/mod.rs +9 -3
  54. package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +31 -11
  55. package/sdk-core/core/src/worker/activities/local_activities.rs +35 -28
  56. package/sdk-core/core/src/worker/activities.rs +58 -30
  57. package/sdk-core/core/src/worker/client/mocks.rs +3 -3
  58. package/sdk-core/core/src/worker/client.rs +155 -53
  59. package/sdk-core/core/src/worker/mod.rs +103 -95
  60. package/sdk-core/core/src/worker/nexus.rs +100 -73
  61. package/sdk-core/core/src/worker/tuner/resource_based.rs +14 -6
  62. package/sdk-core/core/src/worker/tuner.rs +4 -13
  63. package/sdk-core/core/src/worker/workflow/history_update.rs +47 -51
  64. package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +1 -4
  65. package/sdk-core/core/src/worker/workflow/machines/nexus_operation_state_machine.rs +127 -32
  66. package/sdk-core/core/src/worker/workflow/machines/transition_coverage.rs +1 -2
  67. package/sdk-core/core/src/worker/workflow/machines/update_state_machine.rs +1 -1
  68. package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +55 -42
  69. package/sdk-core/core/src/worker/workflow/managed_run.rs +45 -35
  70. package/sdk-core/core/src/worker/workflow/mod.rs +200 -97
  71. package/sdk-core/core/src/worker/workflow/wft_poller.rs +175 -4
  72. package/sdk-core/core/src/worker/workflow/workflow_stream.rs +38 -36
  73. package/sdk-core/core-api/Cargo.toml +8 -0
  74. package/sdk-core/core-api/src/envconfig.rs +1544 -0
  75. package/sdk-core/core-api/src/lib.rs +2 -0
  76. package/sdk-core/core-api/src/telemetry/metrics.rs +8 -0
  77. package/sdk-core/core-api/src/telemetry.rs +36 -3
  78. package/sdk-core/core-api/src/worker.rs +301 -75
  79. package/sdk-core/docker/docker-compose-telem.yaml +1 -0
  80. package/sdk-core/etc/prometheus.yaml +6 -2
  81. package/sdk-core/histories/long_local_activity_with_update-0_history.bin +0 -0
  82. package/sdk-core/histories/long_local_activity_with_update-1_history.bin +0 -0
  83. package/sdk-core/histories/long_local_activity_with_update-2_history.bin +0 -0
  84. package/sdk-core/histories/long_local_activity_with_update-3_history.bin +0 -0
  85. package/sdk-core/sdk/src/activity_context.rs +5 -0
  86. package/sdk-core/sdk/src/interceptors.rs +73 -3
  87. package/sdk-core/sdk/src/lib.rs +15 -16
  88. package/sdk-core/sdk/src/workflow_context/options.rs +10 -0
  89. package/sdk-core/sdk/src/workflow_context.rs +48 -29
  90. package/sdk-core/sdk/src/workflow_future.rs +5 -6
  91. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/.github/workflows/push-to-buf.yml +20 -0
  92. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/CODEOWNERS +6 -0
  93. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/README.md +17 -6
  94. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/VERSION +1 -1
  95. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/buf.lock +7 -2
  96. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/buf.yaml +2 -0
  97. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/cloudservice/v1/request_response.proto +78 -0
  98. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/cloudservice/v1/service.proto +29 -0
  99. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/identity/v1/message.proto +74 -32
  100. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/namespace/v1/message.proto +45 -15
  101. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/nexus/v1/message.proto +7 -1
  102. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/operation/v1/message.proto +3 -3
  103. package/sdk-core/sdk-core-protos/protos/api_cloud_upstream/temporal/api/cloud/region/v1/message.proto +3 -3
  104. package/sdk-core/sdk-core-protos/protos/api_upstream/LICENSE +1 -1
  105. package/sdk-core/sdk-core-protos/protos/api_upstream/buf.yaml +2 -0
  106. package/sdk-core/sdk-core-protos/protos/api_upstream/openapi/openapiv2.json +1103 -88
  107. package/sdk-core/sdk-core-protos/protos/api_upstream/openapi/openapiv3.yaml +1233 -151
  108. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/activity/v1/message.proto +0 -22
  109. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/batch/v1/message.proto +19 -24
  110. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/command/v1/message.proto +0 -22
  111. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/common/v1/message.proto +12 -22
  112. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/deployment/v1/message.proto +45 -45
  113. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +0 -22
  114. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/command_type.proto +0 -22
  115. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/common.proto +15 -22
  116. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/deployment.proto +0 -22
  117. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/event_type.proto +4 -22
  118. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/failed_cause.proto +0 -22
  119. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/namespace.proto +0 -22
  120. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/nexus.proto +0 -20
  121. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/query.proto +0 -22
  122. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/reset.proto +0 -22
  123. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/schedule.proto +0 -22
  124. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/task_queue.proto +0 -22
  125. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/update.proto +0 -22
  126. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/enums/v1/workflow.proto +4 -22
  127. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/errordetails/v1/message.proto +0 -22
  128. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/export/v1/message.proto +0 -22
  129. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/failure/v1/message.proto +2 -22
  130. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/filter/v1/message.proto +0 -22
  131. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/history/v1/message.proto +75 -49
  132. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/namespace/v1/message.proto +0 -22
  133. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/nexus/v1/message.proto +0 -20
  134. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +0 -22
  135. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +0 -22
  136. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/protocol/v1/message.proto +0 -22
  137. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/query/v1/message.proto +0 -22
  138. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/replication/v1/message.proto +0 -22
  139. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/rules/v1/message.proto +90 -0
  140. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/schedule/v1/message.proto +0 -22
  141. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/enhanced_stack_trace.proto +0 -22
  142. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/task_complete_metadata.proto +0 -22
  143. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/user_metadata.proto +0 -22
  144. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/sdk/v1/workflow_metadata.proto +0 -22
  145. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +17 -38
  146. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/update/v1/message.proto +0 -22
  147. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/version/v1/message.proto +0 -22
  148. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflow/v1/message.proto +151 -44
  149. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +172 -65
  150. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +69 -28
  151. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/activity_task/activity_task.proto +18 -0
  152. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/common/common.proto +5 -0
  153. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/nexus/nexus.proto +16 -1
  154. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +21 -15
  155. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +3 -0
  156. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_completion/workflow_completion.proto +3 -0
  157. package/sdk-core/sdk-core-protos/src/lib.rs +60 -16
  158. package/sdk-core/test-utils/src/lib.rs +157 -39
  159. package/sdk-core/tests/cloud_tests.rs +86 -0
  160. package/sdk-core/tests/fuzzy_workflow.rs +23 -26
  161. package/sdk-core/tests/global_metric_tests.rs +116 -0
  162. package/sdk-core/tests/heavy_tests.rs +127 -7
  163. package/sdk-core/tests/integ_tests/client_tests.rs +2 -8
  164. package/sdk-core/tests/integ_tests/metrics_tests.rs +100 -106
  165. package/sdk-core/tests/integ_tests/polling_tests.rs +94 -8
  166. package/sdk-core/tests/integ_tests/update_tests.rs +75 -6
  167. package/sdk-core/tests/integ_tests/worker_tests.rs +54 -5
  168. package/sdk-core/tests/integ_tests/worker_versioning_tests.rs +240 -0
  169. package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +41 -3
  170. package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +168 -8
  171. package/sdk-core/tests/integ_tests/workflow_tests/nexus.rs +285 -15
  172. package/sdk-core/tests/integ_tests/workflow_tests/priority.rs +12 -4
  173. package/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +3 -2
  174. package/sdk-core/tests/integ_tests/workflow_tests.rs +124 -74
  175. package/sdk-core/tests/main.rs +3 -51
  176. package/sdk-core/tests/manual_tests.rs +430 -0
  177. package/sdk-core/tests/runner.rs +28 -2
  178. package/src/client.rs +565 -0
  179. package/src/helpers/abort_controller.rs +204 -0
  180. package/src/helpers/callbacks.rs +299 -0
  181. package/src/helpers/errors.rs +302 -0
  182. package/src/helpers/future.rs +44 -0
  183. package/src/helpers/handles.rs +191 -0
  184. package/src/helpers/inspect.rs +18 -0
  185. package/src/helpers/json_string.rs +58 -0
  186. package/src/helpers/mod.rs +20 -0
  187. package/src/helpers/properties.rs +71 -0
  188. package/src/helpers/try_from_js.rs +213 -0
  189. package/src/helpers/try_into_js.rs +129 -0
  190. package/src/lib.rs +28 -40
  191. package/src/logs.rs +111 -0
  192. package/src/metrics.rs +325 -0
  193. package/src/runtime.rs +409 -498
  194. package/src/testing.rs +315 -57
  195. package/src/worker.rs +907 -378
  196. package/ts/errors.ts +57 -0
  197. package/ts/index.ts +10 -596
  198. package/ts/native.ts +496 -0
  199. package/lib/worker-tuner.d.ts +0 -167
  200. package/lib/worker-tuner.js.map +0 -1
  201. package/src/conversions/slot_supplier_bridge.rs +0 -291
  202. package/src/conversions.rs +0 -618
  203. package/src/errors.rs +0 -38
  204. package/src/helpers.rs +0 -297
  205. package/ts/worker-tuner.ts +0 -193
@@ -7,13 +7,10 @@ pub use resource_based::{
7
7
  ResourceSlotOptions,
8
8
  };
9
9
 
10
- use std::sync::{Arc, OnceLock};
11
- use temporal_sdk_core_api::{
12
- telemetry::metrics::TemporalMeter,
13
- worker::{
14
- ActivitySlotKind, LocalActivitySlotKind, NexusSlotKind, SlotKind, SlotSupplier,
15
- WorkerConfig, WorkerTuner, WorkflowSlotKind,
16
- },
10
+ use std::sync::Arc;
11
+ use temporal_sdk_core_api::worker::{
12
+ ActivitySlotKind, LocalActivitySlotKind, NexusSlotKind, SlotKind, SlotSupplier, WorkerConfig,
13
+ WorkerTuner, WorkflowSlotKind,
17
14
  };
18
15
 
19
16
  /// Allows for the composition of different slot suppliers into a [WorkerTuner]
@@ -22,7 +19,6 @@ pub struct TunerHolder {
22
19
  act_supplier: Arc<dyn SlotSupplier<SlotKind = ActivitySlotKind> + Send + Sync>,
23
20
  la_supplier: Arc<dyn SlotSupplier<SlotKind = LocalActivitySlotKind> + Send + Sync>,
24
21
  nexus_supplier: Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync>,
25
- metrics: OnceLock<TemporalMeter>,
26
22
  }
27
23
 
28
24
  /// Can be used to construct a [TunerHolder] without needing to manually construct each
@@ -245,7 +241,6 @@ impl TunerBuilder {
245
241
  .nexus_slot_supplier
246
242
  .clone()
247
243
  .unwrap_or_else(|| Arc::new(FixedSizeSlotSupplier::new(100))),
248
- metrics: OnceLock::new(),
249
244
  }
250
245
  }
251
246
  }
@@ -274,8 +269,4 @@ impl WorkerTuner for TunerHolder {
274
269
  ) -> Arc<dyn SlotSupplier<SlotKind = NexusSlotKind> + Send + Sync> {
275
270
  self.nexus_supplier.clone()
276
271
  }
277
-
278
- fn attach_metrics(&self, m: TemporalMeter) {
279
- let _ = self.metrics.set(m);
280
- }
281
272
  }
@@ -456,10 +456,10 @@ impl HistoryUpdate {
456
456
  use crate::abstractions::dbg_panic;
457
457
 
458
458
  for win in self.events.as_slice().windows(2) {
459
- if let &[e1, e2] = &win {
460
- if e2.event_id != e1.event_id + 1 {
461
- dbg_panic!("HistoryUpdate isn't contiguous! {:?} -> {:?}", e1, e2);
462
- }
459
+ if let &[e1, e2] = &win
460
+ && e2.event_id != e1.event_id + 1
461
+ {
462
+ dbg_panic!("HistoryUpdate isn't contiguous! {:?} -> {:?}", e1, e2);
463
463
  }
464
464
  }
465
465
  true
@@ -632,10 +632,10 @@ impl HistoryUpdate {
632
632
  pub(crate) fn can_take_next_wft_sequence(&self, from_wft_started_id: i64) -> bool {
633
633
  let next_wft_ix =
634
634
  find_end_index_of_next_wft_seq(&self.events, from_wft_started_id, self.has_last_wft);
635
- if let NextWFTSeqEndIndex::Incomplete(_) = next_wft_ix {
636
- if !self.has_last_wft {
637
- return false;
638
- }
635
+ if let NextWFTSeqEndIndex::Incomplete(_) = next_wft_ix
636
+ && !self.has_last_wft
637
+ {
638
+ return false;
639
639
  }
640
640
  true
641
641
  }
@@ -695,7 +695,8 @@ fn find_end_index_of_next_wft_seq(
695
695
  return NextWFTSeqEndIndex::Incomplete(0);
696
696
  }
697
697
  let mut last_index = 0;
698
- let mut saw_any_command_event = false;
698
+ let mut saw_command_or_started = false;
699
+ let mut saw_command = false;
699
700
  let mut wft_started_event_id_to_index = vec![];
700
701
  for (ix, e) in events.iter().enumerate() {
701
702
  last_index = ix;
@@ -706,13 +707,22 @@ fn find_end_index_of_next_wft_seq(
706
707
  continue;
707
708
  }
708
709
 
709
- if e.is_command_event() || e.event_type() == EventType::WorkflowExecutionStarted {
710
- saw_any_command_event = true;
710
+ if e.is_command_event() {
711
+ saw_command = true;
712
+ saw_command_or_started = true;
713
+ }
714
+ if e.event_type() == EventType::WorkflowExecutionStarted {
715
+ saw_command_or_started = true;
711
716
  }
712
717
  if e.is_final_wf_execution_event() {
713
718
  return NextWFTSeqEndIndex::Complete(last_index);
714
719
  }
715
720
 
721
+ // TODO: Emergency undo for boundary calculation change. Remove if no problems after a bit.
722
+ if std::env::var("TEMPORAL_NO_WFT_BOUNDARY_CHANGE").is_ok() {
723
+ saw_command = false;
724
+ }
725
+
716
726
  if e.event_type() == EventType::WorkflowTaskStarted {
717
727
  wft_started_event_id_to_index.push((e.event_id, ix));
718
728
  if let Some(next_event) = events.get(ix + 1) {
@@ -728,13 +738,14 @@ fn find_end_index_of_next_wft_seq(
728
738
  | EventType::WorkflowExecutionCanceled
729
739
  ) {
730
740
  continue;
731
- }
732
- // If we've never seen an interesting event and the next two events are a completion
733
- // followed immediately again by scheduled, then this is a WFT heartbeat and also
734
- // doesn't conclude the sequence.
735
- else if next_event_type == EventType::WorkflowTaskCompleted {
741
+ } else if next_event_type == EventType::WorkflowTaskCompleted {
736
742
  if let Some(next_next_event) = events.get(ix + 2) {
737
- if next_next_event.event_type() == EventType::WorkflowTaskScheduled {
743
+ if !saw_command
744
+ && next_next_event.event_type() == EventType::WorkflowTaskScheduled
745
+ {
746
+ // If we've never seen an interesting event and the next two events are
747
+ // a completion followed immediately again by scheduled, then this is a
748
+ // WFT heartbeat and also doesn't conclude the sequence.
738
749
  continue;
739
750
  } else {
740
751
  // If we see an update accepted command after WFT completed, we want to
@@ -766,18 +777,18 @@ fn find_end_index_of_next_wft_seq(
766
777
  }
767
778
  return NextWFTSeqEndIndex::Complete(ix);
768
779
  }
769
- } else if !has_last_wft && !saw_any_command_event {
780
+ } else if !has_last_wft && !saw_command_or_started {
770
781
  // Don't have enough events to look ahead of the WorkflowTaskCompleted. Need
771
782
  // to fetch more.
772
783
  continue;
773
784
  }
774
785
  }
775
- } else if !has_last_wft && !saw_any_command_event {
786
+ } else if !has_last_wft && !saw_command_or_started {
776
787
  // Don't have enough events to look ahead of the WorkflowTaskStarted. Need to fetch
777
788
  // more.
778
789
  continue;
779
790
  }
780
- if saw_any_command_event {
791
+ if saw_command_or_started {
781
792
  return NextWFTSeqEndIndex::Complete(ix);
782
793
  }
783
794
  }
@@ -921,7 +932,9 @@ mod tests {
921
932
  let seq = next_check_peek(&mut update, 0);
922
933
  assert_eq!(seq.len(), 6);
923
934
  let seq = next_check_peek(&mut update, 6);
924
- assert_eq!(seq.len(), 13);
935
+ assert_eq!(seq.len(), 4);
936
+ let seq = next_check_peek(&mut update, 10);
937
+ assert_eq!(seq.len(), 9);
925
938
  let seq = next_check_peek(&mut update, 19);
926
939
  assert_eq!(seq.len(), 4);
927
940
  let seq = next_check_peek(&mut update, 23);
@@ -1062,12 +1075,16 @@ mod tests {
1062
1075
  t
1063
1076
  }
1064
1077
 
1078
+ #[rstest::rstest]
1065
1079
  #[tokio::test]
1066
- async fn needs_fetch_if_ending_in_middle_of_wft_seq() {
1080
+ async fn needs_fetch_if_ending_in_middle_of_wft_seq(
1081
+ // These values test points truncation could've occurred in the middle of the heartbeat
1082
+ #[values(18, 19, 20, 21)] truncate_at: usize,
1083
+ ) {
1067
1084
  let t = three_wfts_then_heartbeats();
1068
1085
  let mut ends_in_middle_of_seq = t.as_history_update().events;
1069
- ends_in_middle_of_seq.truncate(19);
1070
- // The update should contain the first two complete WFTs, ending on the 8th event which
1086
+ ends_in_middle_of_seq.truncate(truncate_at);
1087
+ // The update should contain the first three complete WFTs, ending on the 11th event which
1071
1088
  // is WFT started. The remaining events should be returned. False flags means the creator
1072
1089
  // knows there are more events, so we should return need fetch
1073
1090
  let (mut update, remaining) = HistoryUpdate::from_events(
@@ -1078,38 +1095,15 @@ mod tests {
1078
1095
  .workflow_task_started_event_id(),
1079
1096
  false,
1080
1097
  );
1081
- assert_eq!(remaining[0].event_id, 8);
1082
- assert_eq!(remaining.last().unwrap().event_id, 19);
1083
- let seq = update.take_next_wft_sequence(0).unwrap_events();
1084
- assert_eq!(seq.last().unwrap().event_id, 3);
1085
- let seq = update.take_next_wft_sequence(3).unwrap_events();
1086
- assert_eq!(seq.last().unwrap().event_id, 7);
1087
- let next = update.take_next_wft_sequence(7);
1088
- assert_matches!(next, NextWFT::NeedFetch);
1089
- }
1090
-
1091
- // Like the above, but if the history happens to be cut off at a wft boundary, (even though
1092
- // there may have been many heartbeats after we have no way of knowing about)
1093
- #[tokio::test]
1094
- async fn needs_fetch_after_complete_seq_with_heartbeats() {
1095
- let t = three_wfts_then_heartbeats();
1096
- let mut ends_in_middle_of_seq = t.as_history_update().events;
1097
- ends_in_middle_of_seq.truncate(20);
1098
- let (mut update, _) = HistoryUpdate::from_events(
1099
- ends_in_middle_of_seq,
1100
- 0,
1101
- t.get_full_history_info()
1102
- .unwrap()
1103
- .workflow_task_started_event_id(),
1104
- false,
1105
- );
1098
+ assert_eq!(remaining[0].event_id, 12);
1099
+ assert_eq!(remaining.last().unwrap().event_id, truncate_at as i64);
1106
1100
  let seq = update.take_next_wft_sequence(0).unwrap_events();
1107
1101
  assert_eq!(seq.last().unwrap().event_id, 3);
1108
1102
  let seq = update.take_next_wft_sequence(3).unwrap_events();
1109
1103
  assert_eq!(seq.last().unwrap().event_id, 7);
1110
1104
  let seq = update.take_next_wft_sequence(7).unwrap_events();
1111
- assert_eq!(seq.last().unwrap().event_id, 20);
1112
- let next = update.take_next_wft_sequence(20);
1105
+ assert_eq!(seq.last().unwrap().event_id, 11);
1106
+ let next = update.take_next_wft_sequence(11);
1113
1107
  assert_matches!(next, NextWFT::NeedFetch);
1114
1108
  }
1115
1109
 
@@ -1146,6 +1140,8 @@ mod tests {
1146
1140
  let seq = update.take_next_wft_sequence(3).unwrap_events();
1147
1141
  assert_eq!(seq.last().unwrap().event_id, 7);
1148
1142
  let seq = update.take_next_wft_sequence(7).unwrap_events();
1143
+ assert_eq!(seq.last().unwrap().event_id, 11);
1144
+ let seq = update.take_next_wft_sequence(11).unwrap_events();
1149
1145
  assert_eq!(seq.last().unwrap().event_id, 158);
1150
1146
  let seq = update.take_next_wft_sequence(158).unwrap_events();
1151
1147
  assert_eq!(seq.last().unwrap().event_id, 160);
@@ -259,10 +259,7 @@ impl StartCommandCreated {
259
259
  ) -> ChildWorkflowMachineTransition<Cancelled> {
260
260
  state.cancelled_before_sent = true;
261
261
  ChildWorkflowMachineTransition::commands(vec![ChildWorkflowCommand::StartCancel(Failure {
262
- message: format!(
263
- "Child Workflow Execution cancelled before scheduled: {}",
264
- reason
265
- ),
262
+ message: format!("Child Workflow Execution cancelled before scheduled: {reason}"),
266
263
  cause: Some(Box::new(Failure {
267
264
  failure_info: Some(FailureInfo::CanceledFailureInfo(
268
265
  failure::CanceledFailureInfo {
@@ -9,7 +9,7 @@ use itertools::Itertools;
9
9
  use rustfsm::{MachineError, StateMachine, TransitionResult, fsm};
10
10
  use temporal_sdk_core_protos::{
11
11
  coresdk::{
12
- nexus::{NexusOperationResult, nexus_operation_result},
12
+ nexus::{NexusOperationCancellationType, NexusOperationResult, nexus_operation_result},
13
13
  workflow_activation::{
14
14
  ResolveNexusOperation, ResolveNexusOperationStart, resolve_nexus_operation_start,
15
15
  },
@@ -51,9 +51,10 @@ fsm! {
51
51
  ScheduledEventRecorded
52
52
  --(NexusOperationTimedOut(NexusOperationTimedOutEventAttributes), on_timed_out)--> TimedOut;
53
53
  ScheduledEventRecorded
54
- --(NexusOperationStarted(NexusOperationStartedEventAttributes), on_started)--> Started;
54
+ --(NexusOperationStarted(NexusOperationStartedEventAttributes), shared on_started)--> Started;
55
55
 
56
56
  Started --(Cancel, shared on_issue_cancel)--> Started;
57
+ Started --(Cancel, shared on_issue_cancel)--> Cancelled;
57
58
  Started --(CommandRequestCancelNexusOperation)--> Started;
58
59
  Started --(NexusOperationCancelRequested)--> Started;
59
60
  Started
@@ -65,17 +66,26 @@ fsm! {
65
66
  Started
66
67
  --(NexusOperationTimedOut(NexusOperationTimedOutEventAttributes), on_timed_out)--> TimedOut;
67
68
 
69
+ Cancelled --(Cancel)--> Cancelled;
70
+ Cancelled --(CommandRequestCancelNexusOperation)--> Cancelled;
71
+ Cancelled --(NexusOperationCancelRequested)--> Cancelled;
72
+ Cancelled --(NexusOperationCompleted(NexusOperationCompletedEventAttributes), shared on_completed)--> Cancelled;
73
+ Cancelled --(NexusOperationFailed(NexusOperationFailedEventAttributes), shared on_failed)--> Cancelled;
74
+ Cancelled --(NexusOperationTimedOut(NexusOperationTimedOutEventAttributes), shared on_timed_out)--> Cancelled;
75
+ Cancelled --(NexusOperationCanceled(NexusOperationCanceledEventAttributes))--> Cancelled;
76
+
68
77
  // Ignore cancels in all terminal states
69
78
  Completed --(Cancel)--> Completed;
70
79
  Failed --(Cancel)--> Failed;
71
- Cancelled --(Cancel)--> Cancelled;
72
80
  TimedOut --(Cancel)--> TimedOut;
73
81
  }
74
82
 
75
83
  #[derive(Debug, derive_more::Display)]
76
84
  pub(super) enum NexusOperationCommand {
77
85
  #[display("Start")]
78
- Start { operation_id: String },
86
+ Start { operation_token: String },
87
+ #[display("StartSync")]
88
+ StartSync,
79
89
  #[display("CancelBeforeStart")]
80
90
  CancelBeforeStart,
81
91
  #[display("Complete")]
@@ -100,6 +110,8 @@ pub(super) struct SharedState {
100
110
 
101
111
  cancelled_before_sent: bool,
102
112
  cancel_sent: bool,
113
+ cancel_type: NexusOperationCancellationType,
114
+ operation_token: Option<String>,
103
115
  }
104
116
 
105
117
  impl NexusOperationMachine {
@@ -114,6 +126,8 @@ impl NexusOperationMachine {
114
126
  operation: attribs.operation.clone(),
115
127
  cancelled_before_sent: false,
116
128
  cancel_sent: false,
129
+ cancel_type: attribs.cancellation_type(),
130
+ operation_token: None,
117
131
  },
118
132
  );
119
133
  NewMachineWithCommand {
@@ -185,9 +199,7 @@ impl ScheduledEventRecorded {
185
199
  ca: NexusOperationCompletedEventAttributes,
186
200
  ) -> NexusOperationMachineTransition<Completed> {
187
201
  NexusOperationMachineTransition::commands([
188
- NexusOperationCommand::Start {
189
- operation_id: String::default(),
190
- },
202
+ NexusOperationCommand::StartSync,
191
203
  NexusOperationCommand::Complete(ca.result),
192
204
  ])
193
205
  }
@@ -197,9 +209,7 @@ impl ScheduledEventRecorded {
197
209
  fa: NexusOperationFailedEventAttributes,
198
210
  ) -> NexusOperationMachineTransition<Failed> {
199
211
  NexusOperationMachineTransition::commands([
200
- NexusOperationCommand::Start {
201
- operation_id: String::default(),
202
- },
212
+ NexusOperationCommand::StartSync,
203
213
  NexusOperationCommand::Fail(fa.failure.unwrap_or_else(|| Failure {
204
214
  message: "Nexus operation failed but failure field was not populated".to_owned(),
205
215
  ..Default::default()
@@ -212,9 +222,7 @@ impl ScheduledEventRecorded {
212
222
  ca: NexusOperationCanceledEventAttributes,
213
223
  ) -> NexusOperationMachineTransition<Cancelled> {
214
224
  NexusOperationMachineTransition::commands([
215
- NexusOperationCommand::Start {
216
- operation_id: String::default(),
217
- },
225
+ NexusOperationCommand::StartSync,
218
226
  NexusOperationCommand::Cancel(ca.failure.unwrap_or_else(|| Failure {
219
227
  message:
220
228
  "Nexus operation was cancelled but failure field was not populated".to_owned(),
@@ -228,9 +236,7 @@ impl ScheduledEventRecorded {
228
236
  toa: NexusOperationTimedOutEventAttributes,
229
237
  ) -> NexusOperationMachineTransition<TimedOut> {
230
238
  NexusOperationMachineTransition::commands([
231
- NexusOperationCommand::Start {
232
- operation_id: String::default(),
233
- },
239
+ NexusOperationCommand::StartSync,
234
240
  NexusOperationCommand::TimedOut(toa.failure.unwrap_or_else(|| Failure {
235
241
  message: "Nexus operation timed out but failure field was not populated".to_owned(),
236
242
  ..Default::default()
@@ -240,10 +246,12 @@ impl ScheduledEventRecorded {
240
246
 
241
247
  pub(super) fn on_started(
242
248
  self,
249
+ ss: &mut SharedState,
243
250
  sa: NexusOperationStartedEventAttributes,
244
251
  ) -> NexusOperationMachineTransition<Started> {
252
+ ss.operation_token = Some(sa.operation_token.clone());
245
253
  NexusOperationMachineTransition::commands([NexusOperationCommand::Start {
246
- operation_id: sa.operation_id,
254
+ operation_token: sa.operation_token,
247
255
  }])
248
256
  }
249
257
  }
@@ -255,12 +263,20 @@ impl Started {
255
263
  pub(crate) fn on_issue_cancel(
256
264
  &self,
257
265
  ss: &mut SharedState,
258
- ) -> NexusOperationMachineTransition<Started> {
266
+ ) -> NexusOperationMachineTransition<StartedOrCancelled> {
259
267
  if !ss.cancel_sent {
260
268
  ss.cancel_sent = true;
261
- NexusOperationMachineTransition::commands([NexusOperationCommand::IssueCancel])
269
+ let dest = if matches!(
270
+ ss.cancel_type,
271
+ NexusOperationCancellationType::Abandon | NexusOperationCancellationType::TryCancel
272
+ ) {
273
+ StartedOrCancelled::Cancelled(Default::default())
274
+ } else {
275
+ StartedOrCancelled::Started(Default::default())
276
+ };
277
+ TransitionResult::ok([NexusOperationCommand::IssueCancel], dest)
262
278
  } else {
263
- NexusOperationMachineTransition::default()
279
+ TransitionResult::ok([], StartedOrCancelled::Started(Default::default()))
264
280
  }
265
281
  }
266
282
 
@@ -321,6 +337,49 @@ pub(super) struct TimedOut;
321
337
  #[derive(Default, Clone)]
322
338
  pub(super) struct Cancelled;
323
339
 
340
+ fn completion_of_not_abandoned_err() -> WFMachinesError {
341
+ WFMachinesError::Nondeterminism(
342
+ "Nexus operation which don't have the ABANDON cancellation type cannot complete after \
343
+ being cancelled."
344
+ .to_string(),
345
+ )
346
+ }
347
+
348
+ impl Cancelled {
349
+ pub(super) fn on_completed(
350
+ self,
351
+ ss: &mut SharedState,
352
+ _: NexusOperationCompletedEventAttributes,
353
+ ) -> NexusOperationMachineTransition<Cancelled> {
354
+ if ss.cancel_type == NexusOperationCancellationType::Abandon {
355
+ return NexusOperationMachineTransition::Err(completion_of_not_abandoned_err());
356
+ }
357
+ NexusOperationMachineTransition::ok([], self)
358
+ }
359
+
360
+ pub(super) fn on_failed(
361
+ self,
362
+ ss: &mut SharedState,
363
+ _: NexusOperationFailedEventAttributes,
364
+ ) -> NexusOperationMachineTransition<Cancelled> {
365
+ if ss.cancel_type == NexusOperationCancellationType::Abandon {
366
+ return NexusOperationMachineTransition::Err(completion_of_not_abandoned_err());
367
+ }
368
+ NexusOperationMachineTransition::ok([], self)
369
+ }
370
+
371
+ pub(super) fn on_timed_out(
372
+ self,
373
+ ss: &mut SharedState,
374
+ _: NexusOperationTimedOutEventAttributes,
375
+ ) -> NexusOperationMachineTransition<Cancelled> {
376
+ if ss.cancel_type == NexusOperationCancellationType::Abandon {
377
+ return NexusOperationMachineTransition::Err(completion_of_not_abandoned_err());
378
+ }
379
+ NexusOperationMachineTransition::ok([], self)
380
+ }
381
+ }
382
+
324
383
  impl TryFrom<HistEventData> for NexusOperationMachineEvents {
325
384
  type Error = WFMachinesError;
326
385
 
@@ -412,12 +471,21 @@ impl WFMachinesAdapter for NexusOperationMachine {
412
471
  _: Option<EventInfo>,
413
472
  ) -> Result<Vec<MachineResponse>, WFMachinesError> {
414
473
  Ok(match my_command {
415
- NexusOperationCommand::Start { operation_id } => {
474
+ NexusOperationCommand::StartSync => {
416
475
  vec![
417
476
  ResolveNexusOperationStart {
418
477
  seq: self.shared_state.lang_seq_num,
419
- status: Some(resolve_nexus_operation_start::Status::OperationId(
420
- operation_id,
478
+ status: Some(resolve_nexus_operation_start::Status::StartedSync(true)),
479
+ }
480
+ .into(),
481
+ ]
482
+ }
483
+ NexusOperationCommand::Start { operation_token } => {
484
+ vec![
485
+ ResolveNexusOperationStart {
486
+ seq: self.shared_state.lang_seq_num,
487
+ status: Some(resolve_nexus_operation_start::Status::OperationToken(
488
+ operation_token,
421
489
  )),
422
490
  }
423
491
  .into(),
@@ -430,6 +498,7 @@ impl WFMachinesAdapter for NexusOperationMachine {
430
498
  status: Some(resolve_nexus_operation_start::Status::CancelledBeforeStart(
431
499
  self.cancelled_failure(
432
500
  "Nexus Operation cancelled before scheduled".to_owned(),
501
+ &self.shared_state.operation_token,
433
502
  ),
434
503
  )),
435
504
  }
@@ -440,6 +509,7 @@ impl WFMachinesAdapter for NexusOperationMachine {
440
509
  status: Some(nexus_operation_result::Status::Cancelled(
441
510
  self.cancelled_failure(
442
511
  "Nexus Operation cancelled before scheduled".to_owned(),
512
+ &self.shared_state.operation_token,
443
513
  ),
444
514
  )),
445
515
  }),
@@ -494,14 +564,39 @@ impl WFMachinesAdapter for NexusOperationMachine {
494
564
  ]
495
565
  }
496
566
  NexusOperationCommand::IssueCancel => {
497
- vec![MachineResponse::IssueNewCommand(
498
- command::Attributes::RequestCancelNexusOperationCommandAttributes(
499
- RequestCancelNexusOperationCommandAttributes {
500
- scheduled_event_id: self.shared_state.scheduled_event_id,
501
- },
567
+ let mut resps = vec![];
568
+ if self.shared_state.cancel_type != NexusOperationCancellationType::Abandon {
569
+ resps.push(MachineResponse::IssueNewCommand(
570
+ command::Attributes::RequestCancelNexusOperationCommandAttributes(
571
+ RequestCancelNexusOperationCommandAttributes {
572
+ scheduled_event_id: self.shared_state.scheduled_event_id,
573
+ },
574
+ )
575
+ .into(),
576
+ ))
577
+ }
578
+ // Immediately resolve abandon/trycancel modes
579
+ if matches!(
580
+ self.shared_state.cancel_type,
581
+ NexusOperationCancellationType::Abandon
582
+ | NexusOperationCancellationType::TryCancel
583
+ ) {
584
+ resps.push(
585
+ ResolveNexusOperation {
586
+ seq: self.shared_state.lang_seq_num,
587
+ result: Some(NexusOperationResult {
588
+ status: Some(nexus_operation_result::Status::Cancelled(
589
+ self.cancelled_failure(
590
+ "Nexus operation cancelled after starting".to_owned(),
591
+ &self.shared_state.operation_token,
592
+ ),
593
+ )),
594
+ }),
595
+ }
596
+ .into(),
502
597
  )
503
- .into(),
504
- )]
598
+ }
599
+ resps
505
600
  }
506
601
  })
507
602
  }
@@ -520,7 +615,7 @@ impl TryFrom<CommandType> for NexusOperationMachineEvents {
520
615
  }
521
616
 
522
617
  impl NexusOperationMachine {
523
- fn cancelled_failure(&self, message: String) -> Failure {
618
+ fn cancelled_failure(&self, message: String, operation_token: &Option<String>) -> Failure {
524
619
  Failure {
525
620
  message,
526
621
  cause: Some(Box::new(Failure {
@@ -534,7 +629,7 @@ impl NexusOperationMachine {
534
629
  service: self.shared_state.service.clone(),
535
630
  operation: self.shared_state.operation.clone(),
536
631
  operation_id: "".to_string(),
537
- operation_token: "".to_string(),
632
+ operation_token: operation_token.clone().unwrap_or_default(),
538
633
  },
539
634
  )),
540
635
  ..Default::default()
@@ -4,11 +4,10 @@
4
4
  //! never ever be removed from behind `#[cfg(test)]` compilation.
5
5
 
6
6
  use dashmap::{DashMap, DashSet, mapref::entry::Entry};
7
- use std::sync::LazyLock;
8
7
  use std::{
9
8
  path::PathBuf,
10
9
  sync::{
11
- Mutex,
10
+ LazyLock, Mutex,
12
11
  mpsc::{SyncSender, sync_channel},
13
12
  },
14
13
  thread::JoinHandle,
@@ -173,7 +173,7 @@ impl UpdateMachine {
173
173
  msg: UpdateMsg,
174
174
  ) -> Result<MachineResponse, WFMachinesError> {
175
175
  let accept_body = msg.pack().map_err(|e| {
176
- WFMachinesError::Fatal(format!("Failed to serialize update response: {:?}", e))
176
+ WFMachinesError::Fatal(format!("Failed to serialize update response: {e:?}"))
177
177
  })?;
178
178
  Ok(MachineResponse::IssueNewMessage(ProtocolMessage {
179
179
  id: outgoing_id.clone(),