@temporalio/core-bridge 1.13.0 → 1.13.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/Cargo.lock +239 -382
  2. package/Cargo.toml +11 -11
  3. package/lib/native.d.ts +10 -3
  4. package/package.json +3 -3
  5. package/releases/aarch64-apple-darwin/index.node +0 -0
  6. package/releases/aarch64-unknown-linux-gnu/index.node +0 -0
  7. package/releases/x86_64-apple-darwin/index.node +0 -0
  8. package/releases/x86_64-pc-windows-msvc/index.node +0 -0
  9. package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
  10. package/sdk-core/.cargo/config.toml +71 -11
  11. package/sdk-core/.clippy.toml +1 -0
  12. package/sdk-core/.github/workflows/heavy.yml +2 -0
  13. package/sdk-core/.github/workflows/per-pr.yml +50 -18
  14. package/sdk-core/ARCHITECTURE.md +44 -48
  15. package/sdk-core/Cargo.toml +26 -7
  16. package/sdk-core/README.md +4 -0
  17. package/sdk-core/arch_docs/diagrams/TimerMachine_Coverage.puml +14 -0
  18. package/sdk-core/arch_docs/diagrams/initial_event_history.png +0 -0
  19. package/sdk-core/arch_docs/sdks_intro.md +299 -0
  20. package/sdk-core/client/Cargo.toml +8 -7
  21. package/sdk-core/client/src/callback_based.rs +1 -2
  22. package/sdk-core/client/src/lib.rs +485 -299
  23. package/sdk-core/client/src/metrics.rs +32 -8
  24. package/sdk-core/client/src/proxy.rs +124 -5
  25. package/sdk-core/client/src/raw.rs +598 -307
  26. package/sdk-core/client/src/replaceable.rs +253 -0
  27. package/sdk-core/client/src/retry.rs +9 -6
  28. package/sdk-core/client/src/worker_registry/mod.rs +19 -3
  29. package/sdk-core/client/src/workflow_handle/mod.rs +20 -17
  30. package/sdk-core/core/Cargo.toml +100 -31
  31. package/sdk-core/core/src/core_tests/activity_tasks.rs +55 -225
  32. package/sdk-core/core/src/core_tests/mod.rs +2 -8
  33. package/sdk-core/core/src/core_tests/queries.rs +3 -5
  34. package/sdk-core/core/src/core_tests/replay_flag.rs +3 -62
  35. package/sdk-core/core/src/core_tests/updates.rs +4 -5
  36. package/sdk-core/core/src/core_tests/workers.rs +4 -3
  37. package/sdk-core/core/src/core_tests/workflow_cancels.rs +10 -7
  38. package/sdk-core/core/src/core_tests/workflow_tasks.rs +28 -291
  39. package/sdk-core/core/src/ephemeral_server/mod.rs +15 -3
  40. package/sdk-core/core/src/internal_flags.rs +11 -1
  41. package/sdk-core/core/src/lib.rs +50 -36
  42. package/sdk-core/core/src/pollers/mod.rs +5 -5
  43. package/sdk-core/core/src/pollers/poll_buffer.rs +2 -2
  44. package/sdk-core/core/src/protosext/mod.rs +13 -5
  45. package/sdk-core/core/src/protosext/protocol_messages.rs +4 -11
  46. package/sdk-core/core/src/retry_logic.rs +256 -108
  47. package/sdk-core/core/src/telemetry/metrics.rs +1 -0
  48. package/sdk-core/core/src/telemetry/mod.rs +8 -2
  49. package/sdk-core/core/src/telemetry/prometheus_meter.rs +2 -2
  50. package/sdk-core/core/src/test_help/integ_helpers.rs +971 -0
  51. package/sdk-core/core/src/test_help/mod.rs +10 -1100
  52. package/sdk-core/core/src/test_help/unit_helpers.rs +218 -0
  53. package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +42 -6
  54. package/sdk-core/core/src/worker/activities/local_activities.rs +19 -19
  55. package/sdk-core/core/src/worker/activities.rs +10 -3
  56. package/sdk-core/core/src/worker/client/mocks.rs +3 -3
  57. package/sdk-core/core/src/worker/client.rs +130 -93
  58. package/sdk-core/core/src/worker/heartbeat.rs +12 -13
  59. package/sdk-core/core/src/worker/mod.rs +31 -21
  60. package/sdk-core/core/src/worker/nexus.rs +14 -3
  61. package/sdk-core/core/src/worker/slot_provider.rs +9 -0
  62. package/sdk-core/core/src/worker/tuner.rs +159 -0
  63. package/sdk-core/core/src/worker/workflow/history_update.rs +3 -265
  64. package/sdk-core/core/src/worker/workflow/machines/activity_state_machine.rs +1 -54
  65. package/sdk-core/core/src/worker/workflow/machines/cancel_external_state_machine.rs +0 -82
  66. package/sdk-core/core/src/worker/workflow/machines/cancel_workflow_state_machine.rs +0 -67
  67. package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +1 -192
  68. package/sdk-core/core/src/worker/workflow/machines/continue_as_new_workflow_state_machine.rs +0 -43
  69. package/sdk-core/core/src/worker/workflow/machines/local_activity_state_machine.rs +6 -554
  70. package/sdk-core/core/src/worker/workflow/machines/modify_workflow_properties_state_machine.rs +0 -71
  71. package/sdk-core/core/src/worker/workflow/machines/nexus_operation_state_machine.rs +102 -3
  72. package/sdk-core/core/src/worker/workflow/machines/patch_state_machine.rs +10 -539
  73. package/sdk-core/core/src/worker/workflow/machines/signal_external_state_machine.rs +0 -139
  74. package/sdk-core/core/src/worker/workflow/machines/timer_state_machine.rs +1 -119
  75. package/sdk-core/core/src/worker/workflow/machines/upsert_search_attributes_state_machine.rs +6 -63
  76. package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +9 -4
  77. package/sdk-core/core/src/worker/workflow/mod.rs +5 -1
  78. package/sdk-core/core/src/worker/workflow/workflow_stream.rs +8 -3
  79. package/sdk-core/core-api/Cargo.toml +4 -4
  80. package/sdk-core/core-api/src/envconfig.rs +153 -54
  81. package/sdk-core/core-api/src/lib.rs +68 -0
  82. package/sdk-core/core-api/src/telemetry/metrics.rs +2 -1
  83. package/sdk-core/core-api/src/telemetry.rs +13 -0
  84. package/sdk-core/core-c-bridge/Cargo.toml +13 -8
  85. package/sdk-core/core-c-bridge/include/temporal-sdk-core-c-bridge.h +184 -22
  86. package/sdk-core/core-c-bridge/src/client.rs +462 -184
  87. package/sdk-core/core-c-bridge/src/envconfig.rs +314 -0
  88. package/sdk-core/core-c-bridge/src/lib.rs +1 -0
  89. package/sdk-core/core-c-bridge/src/random.rs +4 -4
  90. package/sdk-core/core-c-bridge/src/runtime.rs +22 -23
  91. package/sdk-core/core-c-bridge/src/testing.rs +1 -4
  92. package/sdk-core/core-c-bridge/src/tests/context.rs +31 -31
  93. package/sdk-core/core-c-bridge/src/tests/mod.rs +32 -28
  94. package/sdk-core/core-c-bridge/src/tests/utils.rs +7 -7
  95. package/sdk-core/core-c-bridge/src/worker.rs +319 -66
  96. package/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +6 -1
  97. package/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/dupe_transitions_fail.stderr +5 -5
  98. package/sdk-core/sdk/Cargo.toml +8 -2
  99. package/sdk-core/sdk/src/activity_context.rs +1 -1
  100. package/sdk-core/sdk/src/app_data.rs +1 -1
  101. package/sdk-core/sdk/src/interceptors.rs +1 -4
  102. package/sdk-core/sdk/src/lib.rs +1 -5
  103. package/sdk-core/sdk/src/workflow_context/options.rs +10 -1
  104. package/sdk-core/sdk/src/workflow_future.rs +1 -1
  105. package/sdk-core/sdk-core-protos/Cargo.toml +6 -6
  106. package/sdk-core/sdk-core-protos/build.rs +10 -23
  107. package/sdk-core/sdk-core-protos/protos/api_upstream/.github/workflows/create-release.yml +9 -1
  108. package/sdk-core/sdk-core-protos/protos/api_upstream/openapi/openapiv2.json +254 -5
  109. package/sdk-core/sdk-core-protos/protos/api_upstream/openapi/openapiv3.yaml +234 -5
  110. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/common/v1/message.proto +1 -1
  111. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/deployment/v1/message.proto +6 -0
  112. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/namespace/v1/message.proto +6 -2
  113. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +60 -2
  114. package/sdk-core/sdk-core-protos/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +30 -6
  115. package/sdk-core/sdk-core-protos/protos/local/temporal/sdk/core/workflow_activation/workflow_activation.proto +2 -0
  116. package/sdk-core/{test-utils → sdk-core-protos}/src/canned_histories.rs +5 -5
  117. package/sdk-core/sdk-core-protos/src/history_builder.rs +2 -2
  118. package/sdk-core/sdk-core-protos/src/lib.rs +25 -9
  119. package/sdk-core/sdk-core-protos/src/test_utils.rs +89 -0
  120. package/sdk-core/sdk-core-protos/src/utilities.rs +14 -5
  121. package/sdk-core/tests/c_bridge_smoke_test.c +10 -0
  122. package/sdk-core/tests/cloud_tests.rs +10 -8
  123. package/sdk-core/tests/common/http_proxy.rs +134 -0
  124. package/sdk-core/{test-utils/src/lib.rs → tests/common/mod.rs} +214 -281
  125. package/sdk-core/{test-utils/src → tests/common}/workflows.rs +4 -3
  126. package/sdk-core/tests/fuzzy_workflow.rs +1 -1
  127. package/sdk-core/tests/global_metric_tests.rs +8 -7
  128. package/sdk-core/tests/heavy_tests.rs +7 -3
  129. package/sdk-core/tests/integ_tests/client_tests.rs +111 -24
  130. package/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +14 -9
  131. package/sdk-core/tests/integ_tests/heartbeat_tests.rs +4 -4
  132. package/sdk-core/tests/integ_tests/metrics_tests.rs +114 -14
  133. package/sdk-core/tests/integ_tests/pagination_tests.rs +273 -0
  134. package/sdk-core/tests/integ_tests/polling_tests.rs +311 -93
  135. package/sdk-core/tests/integ_tests/queries_tests.rs +4 -4
  136. package/sdk-core/tests/integ_tests/update_tests.rs +13 -7
  137. package/sdk-core/tests/integ_tests/visibility_tests.rs +26 -9
  138. package/sdk-core/tests/integ_tests/worker_tests.rs +668 -13
  139. package/sdk-core/tests/integ_tests/worker_versioning_tests.rs +40 -24
  140. package/sdk-core/tests/integ_tests/workflow_tests/activities.rs +244 -11
  141. package/sdk-core/tests/integ_tests/workflow_tests/appdata_propagation.rs +1 -1
  142. package/sdk-core/tests/integ_tests/workflow_tests/cancel_external.rs +78 -2
  143. package/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +61 -2
  144. package/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +465 -7
  145. package/sdk-core/tests/integ_tests/workflow_tests/continue_as_new.rs +41 -2
  146. package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +315 -3
  147. package/sdk-core/tests/integ_tests/workflow_tests/eager.rs +1 -1
  148. package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +1990 -14
  149. package/sdk-core/tests/integ_tests/workflow_tests/modify_wf_properties.rs +65 -2
  150. package/sdk-core/tests/integ_tests/workflow_tests/nexus.rs +123 -23
  151. package/sdk-core/tests/integ_tests/workflow_tests/patches.rs +525 -3
  152. package/sdk-core/tests/integ_tests/workflow_tests/replay.rs +65 -16
  153. package/sdk-core/tests/integ_tests/workflow_tests/resets.rs +32 -23
  154. package/sdk-core/tests/integ_tests/workflow_tests/signals.rs +126 -5
  155. package/sdk-core/tests/integ_tests/workflow_tests/stickyness.rs +1 -2
  156. package/sdk-core/tests/integ_tests/workflow_tests/timers.rs +124 -8
  157. package/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +62 -2
  158. package/sdk-core/tests/integ_tests/workflow_tests.rs +67 -8
  159. package/sdk-core/tests/main.rs +26 -17
  160. package/sdk-core/tests/manual_tests.rs +5 -1
  161. package/sdk-core/tests/runner.rs +22 -40
  162. package/sdk-core/tests/shared_tests/mod.rs +1 -1
  163. package/sdk-core/tests/shared_tests/priority.rs +1 -1
  164. package/sdk-core/{core/benches/workflow_replay.rs → tests/workflow_replay_bench.rs} +10 -5
  165. package/src/client.rs +97 -20
  166. package/src/helpers/callbacks.rs +4 -4
  167. package/src/helpers/errors.rs +7 -1
  168. package/src/helpers/handles.rs +1 -0
  169. package/src/helpers/try_from_js.rs +4 -3
  170. package/src/lib.rs +3 -2
  171. package/src/metrics.rs +3 -0
  172. package/src/runtime.rs +5 -2
  173. package/src/worker.rs +9 -12
  174. package/ts/native.ts +13 -3
  175. package/sdk-core/arch_docs/diagrams/workflow_internals.svg +0 -1
  176. package/sdk-core/core/src/core_tests/child_workflows.rs +0 -281
  177. package/sdk-core/core/src/core_tests/determinism.rs +0 -318
  178. package/sdk-core/core/src/core_tests/local_activities.rs +0 -1442
  179. package/sdk-core/test-utils/Cargo.toml +0 -38
  180. package/sdk-core/test-utils/src/histfetch.rs +0 -28
  181. package/sdk-core/test-utils/src/interceptors.rs +0 -46
@@ -3,6 +3,7 @@ pub(crate) mod protocol_messages;
3
3
  use crate::{
4
4
  CompleteActivityError, TaskToken,
5
5
  protosext::protocol_messages::IncomingProtocolMessage,
6
+ retry_logic::ValidatedRetryPolicy,
6
7
  worker::{LEGACY_QUERY_ID, LocalActivityExecutionResult},
7
8
  };
8
9
  use anyhow::anyhow;
@@ -32,11 +33,12 @@ use temporal_sdk_core_protos::{
32
33
  workflow_completion,
33
34
  },
34
35
  temporal::api::{
35
- common::v1::{Payload, RetryPolicy, WorkflowExecution},
36
+ common::v1::{Payload, WorkflowExecution},
36
37
  enums::v1::EventType,
37
38
  failure::v1::Failure,
38
39
  history::v1::{History, HistoryEvent, MarkerRecordedEventAttributes, history_event},
39
40
  query::v1::WorkflowQuery,
41
+ sdk::v1::UserMetadata,
40
42
  workflowservice::v1::PollWorkflowTaskQueueResponse,
41
43
  },
42
44
  utilities::TryIntoOrNone,
@@ -134,7 +136,7 @@ impl TryFrom<PollWorkflowTaskQueueResponse> for ValidPollWFTQResponse {
134
136
  _cant_construct_me: (),
135
137
  })
136
138
  }
137
- _ => Err(anyhow!("Unable to interpret poll response: {:?}", value)),
139
+ _ => Err(anyhow!("Unable to interpret poll response: {value:?}")),
138
140
  }
139
141
  }
140
142
  }
@@ -317,9 +319,10 @@ pub(crate) struct ValidScheduleLA {
317
319
  pub(crate) arguments: Vec<Payload>,
318
320
  pub(crate) schedule_to_start_timeout: Option<Duration>,
319
321
  pub(crate) close_timeouts: LACloseTimeouts,
320
- pub(crate) retry_policy: RetryPolicy,
322
+ pub(crate) retry_policy: ValidatedRetryPolicy,
321
323
  pub(crate) local_retry_threshold: Duration,
322
324
  pub(crate) cancellation_type: ActivityCancellationType,
325
+ pub(crate) user_metadata: Option<UserMetadata>,
323
326
  }
324
327
 
325
328
  #[derive(Debug, Clone, Copy)]
@@ -349,7 +352,10 @@ impl Default for LACloseTimeouts {
349
352
  }
350
353
 
351
354
  impl ValidScheduleLA {
352
- pub(crate) fn from_schedule_la(v: ScheduleLocalActivity) -> Result<Self, anyhow::Error> {
355
+ pub(crate) fn from_schedule_la(
356
+ v: ScheduleLocalActivity,
357
+ user_metadata: Option<UserMetadata>,
358
+ ) -> Result<Self, anyhow::Error> {
353
359
  let original_schedule_time = v
354
360
  .original_schedule_time
355
361
  .map(|x| {
@@ -403,7 +409,8 @@ impl ValidScheduleLA {
403
409
  ));
404
410
  }
405
411
  };
406
- let retry_policy = v.retry_policy.unwrap_or_default();
412
+ let retry_policy =
413
+ ValidatedRetryPolicy::from_proto_with_defaults(v.retry_policy.unwrap_or_default());
407
414
  let local_retry_threshold = v
408
415
  .local_retry_threshold
409
416
  .try_into_or_none()
@@ -423,6 +430,7 @@ impl ValidScheduleLA {
423
430
  retry_policy,
424
431
  local_retry_threshold,
425
432
  cancellation_type,
433
+ user_metadata,
426
434
  })
427
435
  }
428
436
  }
@@ -1,4 +1,4 @@
1
- use anyhow::{anyhow, bail};
1
+ use anyhow::anyhow;
2
2
  use std::collections::HashMap;
3
3
  use temporal_sdk_core_protos::temporal::api::{
4
4
  common::v1::Payload,
@@ -108,16 +108,9 @@ impl TryFrom<Option<prost_types::Any>> for IncomingProtocolMessageBody {
108
108
 
109
109
  fn try_from(v: Option<prost_types::Any>) -> Result<Self, Self::Error> {
110
110
  let v = v.ok_or_else(|| anyhow!("Protocol message body must be populated"))?;
111
- // Undo explicit type url checks when https://github.com/fdeantoni/prost-wkt/issues/48 is
112
- // fixed
113
- Ok(match v.type_url.as_str() {
114
- "type.googleapis.com/temporal.api.update.v1.Request" => {
115
- IncomingProtocolMessageBody::UpdateRequest(
116
- v.unpack_as(update::v1::Request::default())?.try_into()?,
117
- )
118
- }
119
- o => bail!("Could not understand protocol message type {}", o),
120
- })
111
+ Ok(IncomingProtocolMessageBody::UpdateRequest(
112
+ v.to_msg::<update::v1::Request>()?.try_into()?,
113
+ ))
121
114
  }
122
115
  }
123
116
 
@@ -1,45 +1,87 @@
1
- use std::time::Duration;
2
- use temporal_sdk_core_protos::{
3
- temporal::api::{common::v1::RetryPolicy, failure::v1::ApplicationFailureInfo},
4
- utilities::TryIntoOrNone,
1
+ use std::{num::NonZero, time::Duration};
2
+ use temporal_sdk_core_protos::temporal::api::{
3
+ common::v1::RetryPolicy, failure::v1::ApplicationFailureInfo,
5
4
  };
6
5
 
7
- pub(crate) trait RetryPolicyExt {
6
+ /// Represents a retry policy where all fields have valid values. Durations are stored in std type.
7
+ /// Upholds the following invariants:
8
+ /// - `maximum_interval` >= `initial_interval`
9
+ /// - `backoff_coefficient` >= 1
10
+ /// - `maximum_attempts` >= 0
11
+ #[derive(Debug, Clone)]
12
+ pub(crate) struct ValidatedRetryPolicy {
13
+ initial_interval: Duration,
14
+ backoff_coefficient: f64,
15
+ maximum_interval: Duration,
16
+ maximum_attempts: u32,
17
+ non_retryable_error_types: Vec<String>,
18
+ }
19
+
20
+ impl ValidatedRetryPolicy {
21
+ /// Validates and converts retry policy. If some field is invalid, it's replaced with a default value:
22
+ /// - `initial_interval`: 1 second
23
+ /// - `backoff_coefficient`: 2.0
24
+ /// - `maximum_interval`: 100 * `initial_interval` if missing or inconvertible, 1 * `initial_interval` if too small
25
+ /// - `maximum_attempts`: 0 (unlimited)
26
+ pub(crate) fn from_proto_with_defaults(retry_policy: RetryPolicy) -> Self {
27
+ let initial_interval = retry_policy
28
+ .initial_interval
29
+ .and_then(|i| i.try_into().ok())
30
+ .unwrap_or_else(|| Duration::from_secs(1));
31
+
32
+ let backoff_coefficient = if retry_policy.backoff_coefficient >= 1.0 {
33
+ retry_policy.backoff_coefficient
34
+ } else {
35
+ 2.0
36
+ };
37
+
38
+ let maximum_interval = if let Some(maximum_interval) = retry_policy
39
+ .maximum_interval
40
+ .and_then(|i| Duration::try_from(i).ok())
41
+ {
42
+ maximum_interval.max(initial_interval)
43
+ } else {
44
+ let maximum_interval = initial_interval.saturating_mul(100);
45
+ // Verifying that serialization to proto will work. It may fail for extremely large
46
+ // durations, so in that case we fall back to maximum_interval = initial_interval.
47
+ if prost_types::Duration::try_from(maximum_interval).is_ok() {
48
+ maximum_interval
49
+ } else {
50
+ initial_interval
51
+ }
52
+ };
53
+
54
+ Self {
55
+ initial_interval,
56
+ backoff_coefficient,
57
+ maximum_interval,
58
+ maximum_attempts: retry_policy.maximum_attempts.try_into().unwrap_or(0),
59
+ non_retryable_error_types: retry_policy.non_retryable_error_types,
60
+ }
61
+ }
62
+
8
63
  /// Ask this retry policy if a retry should be performed. Caller provides the current attempt
9
64
  /// number - the first attempt should start at 1.
10
65
  ///
11
- /// Returns `None` if it should not, otherwise a duration indicating how long to wait before
12
- /// performing the retry.
13
- ///
14
- /// Applies defaults to missing fields:
15
- /// `initial_interval` - 1 second
16
- /// `maximum_interval` - 100 x initial_interval
17
- /// `backoff_coefficient` - 2.0
18
- fn should_retry(
19
- &self,
20
- attempt_number: usize,
21
- application_failure: Option<&ApplicationFailureInfo>,
22
- ) -> Option<Duration>;
23
- }
24
-
25
- impl RetryPolicyExt for RetryPolicy {
26
- fn should_retry(
66
+ /// Returns `None` if it should not retry, otherwise returns a duration indicating how long to
67
+ /// wait before performing the retry.
68
+ pub(crate) fn should_retry(
27
69
  &self,
28
- attempt_number: usize,
70
+ attempt_number: NonZero<u32>,
29
71
  application_failure: Option<&ApplicationFailureInfo>,
30
72
  ) -> Option<Duration> {
73
+ if self.maximum_attempts > 0 && attempt_number.get() >= self.maximum_attempts {
74
+ return None;
75
+ }
76
+
31
77
  let non_retryable = application_failure
32
78
  .map(|f| f.non_retryable)
33
79
  .unwrap_or_default();
34
80
  if non_retryable {
35
81
  return None;
36
82
  }
37
- let err_type_str = application_failure.map_or("", |f| &f.r#type);
38
- let realmax = self.maximum_attempts.max(0);
39
- if realmax > 0 && attempt_number >= realmax as usize {
40
- return None;
41
- }
42
83
 
84
+ let err_type_str = application_failure.map_or("", |f| &f.r#type);
43
85
  for pat in &self.non_retryable_error_types {
44
86
  if err_type_str.to_lowercase() == pat.to_lowercase() {
45
87
  return None;
@@ -47,46 +89,48 @@ impl RetryPolicyExt for RetryPolicy {
47
89
  }
48
90
 
49
91
  if let Some(explicit_delay) = application_failure.and_then(|af| af.next_retry_delay) {
50
- return explicit_delay.try_into().ok();
92
+ match explicit_delay.try_into() {
93
+ Ok(delay) => return Some(delay),
94
+ Err(e) => error!(
95
+ "Failed to convert retry delay of application failure. Normal delay calculation will be used. Conversion error: `{}`. Application failure: {:?}",
96
+ e, application_failure
97
+ ),
98
+ }
51
99
  }
52
100
 
53
- let converted_interval = self
54
- .initial_interval
55
- .try_into_or_none()
56
- .or(Some(Duration::from_secs(1)));
57
- if attempt_number == 1 {
58
- return converted_interval;
101
+ if attempt_number.get() == 1 {
102
+ return Some(self.initial_interval);
59
103
  }
60
- let coeff = if self.backoff_coefficient != 0. {
61
- self.backoff_coefficient
62
- } else {
63
- 2.0
64
- };
65
104
 
66
- if let Some(interval) = converted_interval {
67
- let max_iv = self
68
- .maximum_interval
69
- .try_into_or_none()
70
- .unwrap_or_else(|| interval.saturating_mul(100));
71
- let mul_factor = coeff.powi(attempt_number as i32 - 1);
72
- let tried_mul = try_from_secs_f64(mul_factor * interval.as_secs_f64());
73
- Some(tried_mul.unwrap_or(max_iv).min(max_iv))
74
- } else {
75
- // No retries if initial interval is not specified
76
- None
77
- }
105
+ let delay = i32::try_from(attempt_number.get())
106
+ .ok()
107
+ .and_then(|attempt| {
108
+ let factor = self.backoff_coefficient.powi(attempt - 1);
109
+ Duration::try_from_secs_f64(factor * self.initial_interval.as_secs_f64()).ok()
110
+ })
111
+ .map(|interval| interval.min(self.maximum_interval))
112
+ .unwrap_or(self.maximum_interval);
113
+
114
+ Some(delay)
78
115
  }
79
116
  }
80
117
 
81
- const NANOS_PER_SEC: u32 = 1_000_000_000;
82
- /// modified from rust stdlib since this feature is currently nightly only
83
- fn try_from_secs_f64(secs: f64) -> Option<Duration> {
84
- const MAX_NANOS_F64: f64 = ((u64::MAX as u128 + 1) * (NANOS_PER_SEC as u128)) as f64;
85
- let nanos = secs * (NANOS_PER_SEC as f64);
86
- if !nanos.is_finite() || !(0.0..MAX_NANOS_F64).contains(&nanos) {
87
- None
88
- } else {
89
- Some(Duration::from_secs_f64(secs))
118
+ impl Default for ValidatedRetryPolicy {
119
+ fn default() -> Self {
120
+ Self::from_proto_with_defaults(RetryPolicy::default())
121
+ }
122
+ }
123
+
124
+ impl From<ValidatedRetryPolicy> for RetryPolicy {
125
+ fn from(value: ValidatedRetryPolicy) -> Self {
126
+ // All fields were tested on struct initialization to convert successfully. Unwraps are safe.
127
+ Self {
128
+ initial_interval: Some(value.initial_interval.try_into().unwrap()),
129
+ backoff_coefficient: value.backoff_coefficient,
130
+ maximum_interval: Some(value.maximum_interval.try_into().unwrap()),
131
+ maximum_attempts: value.maximum_attempts.try_into().unwrap(),
132
+ non_retryable_error_types: value.non_retryable_error_types,
133
+ }
90
134
  }
91
135
  }
92
136
 
@@ -94,84 +138,186 @@ fn try_from_secs_f64(secs: f64) -> Option<Duration> {
94
138
  mod tests {
95
139
  use super::*;
96
140
  use crate::prost_dur;
141
+ use std::{num::NonZero, time::Duration};
142
+
143
+ macro_rules! nz {
144
+ ($x:expr) => {
145
+ NonZero::new($x).unwrap()
146
+ };
147
+ }
148
+
149
+ #[test]
150
+ fn applies_defaults_to_default_retry_policy() {
151
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy::default());
152
+ assert_eq!(rp.initial_interval, Duration::from_secs(1));
153
+ assert_eq!(rp.backoff_coefficient, 2.0);
154
+ assert_eq!(rp.maximum_interval, Duration::from_secs(100));
155
+ assert_eq!(rp.maximum_attempts, 0);
156
+ assert!(rp.non_retryable_error_types.is_empty());
157
+
158
+ let rp = ValidatedRetryPolicy::default();
159
+ assert_eq!(rp.initial_interval, Duration::from_secs(1));
160
+ assert_eq!(rp.backoff_coefficient, 2.0);
161
+ assert_eq!(rp.maximum_interval, Duration::from_secs(100));
162
+ assert_eq!(rp.maximum_attempts, 0);
163
+ assert!(rp.non_retryable_error_types.is_empty());
164
+ }
165
+
166
+ #[test]
167
+ fn applies_defaults_to_invalid_fields_only() {
168
+ let base_rp = RetryPolicy {
169
+ initial_interval: Some(prost_dur!(from_secs(2))),
170
+ backoff_coefficient: 1.5,
171
+ maximum_interval: Some(prost_dur!(from_secs(4))),
172
+ maximum_attempts: 2,
173
+ non_retryable_error_types: vec!["error".into()],
174
+ };
175
+ let base_values = ValidatedRetryPolicy::from_proto_with_defaults(base_rp.clone());
176
+ assert_eq!(base_values.initial_interval, Duration::from_secs(2));
177
+ assert_eq!(base_values.backoff_coefficient, 1.5);
178
+ assert_eq!(base_values.maximum_interval, Duration::from_secs(4));
179
+ assert_eq!(base_values.maximum_attempts, 2);
180
+ assert_eq!(
181
+ base_values.non_retryable_error_types,
182
+ vec!["error".to_owned()]
183
+ );
184
+
185
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
186
+ initial_interval: Some(prost_types::Duration {
187
+ seconds: -5,
188
+ nanos: 0,
189
+ }),
190
+ ..base_rp.clone()
191
+ });
192
+ assert_eq!(rp.initial_interval, Duration::from_secs(1));
193
+ assert_eq!(rp.backoff_coefficient, base_values.backoff_coefficient);
194
+ assert_eq!(rp.maximum_interval, base_values.maximum_interval);
195
+ assert_eq!(rp.maximum_attempts, base_values.maximum_attempts);
196
+ assert_eq!(
197
+ rp.non_retryable_error_types,
198
+ base_values.non_retryable_error_types
199
+ );
200
+
201
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
202
+ backoff_coefficient: 0.5,
203
+ ..base_rp.clone()
204
+ });
205
+ assert_eq!(rp.initial_interval, base_values.initial_interval);
206
+ assert_eq!(rp.backoff_coefficient, 2.0);
207
+ assert_eq!(rp.maximum_interval, base_values.maximum_interval);
208
+ assert_eq!(rp.maximum_attempts, base_values.maximum_attempts);
209
+ assert_eq!(
210
+ rp.non_retryable_error_types,
211
+ base_values.non_retryable_error_types
212
+ );
213
+
214
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
215
+ maximum_interval: Some(prost_types::Duration {
216
+ seconds: -5,
217
+ nanos: 0,
218
+ }),
219
+ ..base_rp.clone()
220
+ });
221
+ assert_eq!(rp.initial_interval, base_values.initial_interval);
222
+ assert_eq!(rp.backoff_coefficient, base_values.backoff_coefficient);
223
+ assert_eq!(rp.maximum_interval, 100 * base_values.initial_interval);
224
+ assert_eq!(rp.maximum_attempts, base_values.maximum_attempts);
225
+ assert_eq!(
226
+ rp.non_retryable_error_types,
227
+ base_values.non_retryable_error_types
228
+ );
229
+
230
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
231
+ maximum_interval: Some(prost_dur!(from_secs(1))), // valid but less than initial interval
232
+ ..base_rp.clone()
233
+ });
234
+ assert_eq!(rp.initial_interval, base_values.initial_interval);
235
+ assert_eq!(rp.backoff_coefficient, base_values.backoff_coefficient);
236
+ assert_eq!(rp.maximum_interval, base_values.initial_interval);
237
+ assert_eq!(rp.maximum_attempts, base_values.maximum_attempts);
238
+ assert_eq!(
239
+ rp.non_retryable_error_types,
240
+ base_values.non_retryable_error_types
241
+ );
242
+
243
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
244
+ maximum_attempts: -5,
245
+ ..base_rp.clone()
246
+ });
247
+ assert_eq!(rp.initial_interval, base_values.initial_interval);
248
+ assert_eq!(rp.backoff_coefficient, base_values.backoff_coefficient);
249
+ assert_eq!(rp.maximum_interval, base_values.maximum_interval);
250
+ assert_eq!(rp.maximum_attempts, 0);
251
+ assert_eq!(
252
+ rp.non_retryable_error_types,
253
+ base_values.non_retryable_error_types
254
+ );
255
+
256
+ // non_retryable_error_types is always valid
257
+ }
97
258
 
98
259
  #[test]
99
260
  fn calcs_backoffs_properly() {
100
- let rp = RetryPolicy {
261
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
101
262
  initial_interval: Some(prost_dur!(from_secs(1))),
102
263
  backoff_coefficient: 2.0,
103
264
  maximum_interval: Some(prost_dur!(from_secs(10))),
104
265
  maximum_attempts: 10,
105
266
  non_retryable_error_types: vec![],
106
- };
107
- let res = rp.should_retry(1, None).unwrap();
108
- assert_eq!(res.as_millis(), 1_000);
109
- let res = rp.should_retry(2, None).unwrap();
110
- assert_eq!(res.as_millis(), 2_000);
111
- let res = rp.should_retry(3, None).unwrap();
112
- assert_eq!(res.as_millis(), 4_000);
113
- let res = rp.should_retry(4, None).unwrap();
114
- assert_eq!(res.as_millis(), 8_000);
115
- let res = rp.should_retry(5, None).unwrap();
116
- assert_eq!(res.as_millis(), 10_000);
117
- let res = rp.should_retry(6, None).unwrap();
118
- assert_eq!(res.as_millis(), 10_000);
267
+ });
268
+ assert_eq!(rp.should_retry(nz!(1), None), Some(Duration::from_secs(1)));
269
+ assert_eq!(rp.should_retry(nz!(2), None), Some(Duration::from_secs(2)));
270
+ assert_eq!(rp.should_retry(nz!(3), None), Some(Duration::from_secs(4)));
271
+ assert_eq!(rp.should_retry(nz!(4), None), Some(Duration::from_secs(8)));
272
+ assert_eq!(rp.should_retry(nz!(5), None), Some(Duration::from_secs(10)));
273
+ assert_eq!(rp.should_retry(nz!(6), None), Some(Duration::from_secs(10)));
119
274
  // Max attempts - no retry
120
- assert!(rp.should_retry(10, None).is_none());
121
- }
122
-
123
- #[test]
124
- fn no_interval_no_backoff() {
125
- let rp = RetryPolicy {
126
- initial_interval: None,
127
- backoff_coefficient: 0.,
128
- maximum_interval: None,
129
- maximum_attempts: 10,
130
- non_retryable_error_types: vec![],
131
- };
132
- assert!(rp.should_retry(1, None).is_some());
275
+ assert!(rp.should_retry(nz!(10), None).is_none());
133
276
  }
134
277
 
135
278
  #[test]
136
279
  fn max_attempts_zero_retry_forever() {
137
- let rp = RetryPolicy {
280
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
138
281
  initial_interval: Some(prost_dur!(from_secs(1))),
139
282
  backoff_coefficient: 1.2,
140
283
  maximum_interval: None,
141
284
  maximum_attempts: 0,
142
285
  non_retryable_error_types: vec![],
143
- };
144
- for i in 0..50 {
145
- assert!(rp.should_retry(i, None).is_some());
286
+ });
287
+ for i in 1..50 {
288
+ assert!(rp.should_retry(nz!(i), None).is_some());
146
289
  }
147
290
  }
148
291
 
149
292
  #[test]
150
- fn no_overflows() {
151
- let rp = RetryPolicy {
293
+ fn delay_calculation_does_not_overflow() {
294
+ let maximum_interval = Duration::from_secs(1000 * 365 * 24 * 60 * 60);
295
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
152
296
  initial_interval: Some(prost_dur!(from_secs(1))),
153
297
  backoff_coefficient: 10.,
154
- maximum_interval: None,
298
+ maximum_interval: Some(maximum_interval.try_into().unwrap()),
155
299
  maximum_attempts: 0,
156
300
  non_retryable_error_types: vec![],
157
- };
158
- for i in 0..50 {
159
- assert!(rp.should_retry(i, None).is_some());
301
+ });
302
+ for i in 1..50 {
303
+ assert!(rp.should_retry(nz!(i), None).unwrap() <= maximum_interval);
160
304
  }
305
+ assert_eq!(rp.should_retry(nz!(50), None), Some(maximum_interval));
306
+ assert_eq!(rp.should_retry(nz!(u32::MAX), None), Some(maximum_interval));
161
307
  }
162
308
 
163
309
  #[test]
164
310
  fn no_retry_err_str_match() {
165
- let rp = RetryPolicy {
311
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
166
312
  initial_interval: Some(prost_dur!(from_secs(1))),
167
313
  backoff_coefficient: 2.0,
168
314
  maximum_interval: Some(prost_dur!(from_secs(10))),
169
315
  maximum_attempts: 10,
170
316
  non_retryable_error_types: vec!["no retry".to_string()],
171
- };
317
+ });
172
318
  assert!(
173
319
  rp.should_retry(
174
- 1,
320
+ nz!(1),
175
321
  Some(&ApplicationFailureInfo {
176
322
  r#type: "no retry".to_string(),
177
323
  non_retryable: false,
@@ -184,16 +330,16 @@ mod tests {
184
330
 
185
331
  #[test]
186
332
  fn no_non_retryable_application_failure() {
187
- let rp = RetryPolicy {
333
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
188
334
  initial_interval: Some(prost_dur!(from_secs(1))),
189
335
  backoff_coefficient: 2.0,
190
336
  maximum_interval: Some(prost_dur!(from_secs(10))),
191
337
  maximum_attempts: 10,
192
338
  non_retryable_error_types: vec![],
193
- };
339
+ });
194
340
  assert!(
195
341
  rp.should_retry(
196
- 1,
342
+ nz!(1),
197
343
  Some(&ApplicationFailureInfo {
198
344
  r#type: "".to_string(),
199
345
  non_retryable: true,
@@ -206,19 +352,21 @@ mod tests {
206
352
 
207
353
  #[test]
208
354
  fn explicit_delay_is_used() {
209
- let rp = RetryPolicy {
355
+ let rp = ValidatedRetryPolicy::from_proto_with_defaults(RetryPolicy {
210
356
  initial_interval: Some(prost_dur!(from_secs(1))),
211
357
  backoff_coefficient: 2.0,
212
358
  maximum_attempts: 2,
213
359
  ..Default::default()
214
- };
360
+ });
215
361
  let afi = &ApplicationFailureInfo {
216
362
  r#type: "".to_string(),
217
363
  next_retry_delay: Some(prost_dur!(from_secs(50))),
218
364
  ..Default::default()
219
365
  };
220
- let res = rp.should_retry(1, Some(afi)).unwrap();
221
- assert_eq!(res.as_millis(), 50_000);
222
- assert!(rp.should_retry(2, Some(afi)).is_none());
366
+ assert_eq!(
367
+ rp.should_retry(nz!(1), Some(afi)),
368
+ Some(Duration::from_secs(50))
369
+ );
370
+ assert!(rp.should_retry(nz!(2), Some(afi)).is_none());
223
371
  }
224
372
  }
@@ -1104,6 +1104,7 @@ mod tests {
1104
1104
  METRIC_PREFIX.to_string(),
1105
1105
  Some(call_buffer.clone()),
1106
1106
  true,
1107
+ temporal_sdk_core_api::telemetry::TaskQueueLabelStrategy::UseNormal,
1107
1108
  );
1108
1109
  let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
1109
1110
  mc.forced_cache_eviction();
@@ -40,7 +40,8 @@ use std::{
40
40
  },
41
41
  };
42
42
  use temporal_sdk_core_api::telemetry::{
43
- CoreLog, CoreTelemetry, Logger, TelemetryOptions, TelemetryOptionsBuilder,
43
+ CoreLog, CoreTelemetry, Logger, TaskQueueLabelStrategy, TelemetryOptions,
44
+ TelemetryOptionsBuilder,
44
45
  metrics::{CoreMeter, MetricKeyValue, NewAttributes, TemporalMeter},
45
46
  };
46
47
  use tracing::{Level, Subscriber};
@@ -67,6 +68,7 @@ pub struct TelemetryInstance {
67
68
  /// the user has not opted into any tracing configuration.
68
69
  trace_subscriber: Option<Arc<dyn Subscriber + Send + Sync>>,
69
70
  attach_service_name: bool,
71
+ task_queue_label_strategy: TaskQueueLabelStrategy,
70
72
  }
71
73
 
72
74
  impl TelemetryInstance {
@@ -76,6 +78,7 @@ impl TelemetryInstance {
76
78
  metric_prefix: String,
77
79
  metrics: Option<Arc<dyn CoreMeter + 'static>>,
78
80
  attach_service_name: bool,
81
+ task_queue_label_strategy: TaskQueueLabelStrategy,
79
82
  ) -> Self {
80
83
  Self {
81
84
  metric_prefix,
@@ -83,6 +86,7 @@ impl TelemetryInstance {
83
86
  metrics,
84
87
  trace_subscriber,
85
88
  attach_service_name,
89
+ task_queue_label_strategy,
86
90
  }
87
91
  }
88
92
 
@@ -110,6 +114,7 @@ impl TelemetryInstance {
110
114
  Arc::new(PrefixedMetricsMeter::new(self.metric_prefix.clone(), m))
111
115
  as Arc<dyn CoreMeter>,
112
116
  attribs,
117
+ self.task_queue_label_strategy,
113
118
  )
114
119
  })
115
120
  }
@@ -119,7 +124,7 @@ impl TelemetryInstance {
119
124
  self.metrics.clone().map(|m| {
120
125
  let kvs = self.default_kvs();
121
126
  let attribs = NewAttributes::new(kvs);
122
- TemporalMeter::new(m, attribs)
127
+ TemporalMeter::new(m, attribs, self.task_queue_label_strategy)
123
128
  })
124
129
  }
125
130
 
@@ -240,6 +245,7 @@ pub fn telemetry_init(opts: TelemetryOptions) -> Result<TelemetryInstance, anyho
240
245
  opts.metric_prefix,
241
246
  opts.metrics,
242
247
  opts.attach_service_name,
248
+ opts.task_queue_label_strategy,
243
249
  ))
244
250
  }
245
251
 
@@ -315,8 +315,7 @@ where
315
315
  Ok(labels)
316
316
  } else {
317
317
  let e = anyhow!(
318
- "Must use Prometheus attributes with a Prometheus metric implementation. Got: {:?}",
319
- attributes
318
+ "Must use Prometheus attributes with a Prometheus metric implementation. Got: {attributes:?}"
320
319
  );
321
320
  dbg_panic!("{:?}", e);
322
321
  Err(e)
@@ -821,6 +820,7 @@ mod tests {
821
820
  METRIC_PREFIX.to_string(),
822
821
  Some(Arc::new(meter)),
823
822
  true,
823
+ temporal_sdk_core_api::telemetry::TaskQueueLabelStrategy::UseNormal,
824
824
  );
825
825
  let mc = MetricsContext::top_level("foo".to_string(), "q".to_string(), &telem_instance);
826
826
  mc.worker_registered();