@temporalio/core-bridge 1.1.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +765 -128
- package/Cargo.toml +2 -2
- package/common.js +7 -3
- package/index.d.ts +118 -5
- package/index.js +2 -6
- package/package.json +2 -3
- package/releases/aarch64-apple-darwin/index.node +0 -0
- package/releases/aarch64-unknown-linux-gnu/index.node +0 -0
- package/releases/x86_64-apple-darwin/index.node +0 -0
- package/releases/x86_64-pc-windows-msvc/index.node +0 -0
- package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
- package/scripts/build.js +4 -3
- package/sdk-core/.buildkite/docker/Dockerfile +2 -1
- package/sdk-core/.buildkite/pipeline.yml +2 -0
- package/sdk-core/.cargo/config.toml +1 -1
- package/sdk-core/ARCHITECTURE.md +2 -2
- package/sdk-core/README.md +12 -0
- package/sdk-core/bridge-ffi/Cargo.toml +2 -2
- package/sdk-core/bridge-ffi/src/lib.rs +2 -2
- package/sdk-core/client/Cargo.toml +7 -5
- package/sdk-core/client/src/lib.rs +354 -226
- package/sdk-core/client/src/metrics.rs +13 -11
- package/sdk-core/client/src/raw.rs +352 -107
- package/sdk-core/client/src/retry.rs +188 -147
- package/sdk-core/client/src/workflow_handle/mod.rs +1 -1
- package/sdk-core/core/Cargo.toml +28 -15
- package/sdk-core/core/src/core_tests/activity_tasks.rs +98 -33
- package/sdk-core/core/src/core_tests/child_workflows.rs +125 -3
- package/sdk-core/core/src/core_tests/local_activities.rs +6 -6
- package/sdk-core/core/src/core_tests/workers.rs +3 -2
- package/sdk-core/core/src/core_tests/workflow_tasks.rs +70 -2
- package/sdk-core/core/src/ephemeral_server/mod.rs +515 -0
- package/sdk-core/core/src/lib.rs +62 -28
- package/sdk-core/core/src/pollers/mod.rs +2 -0
- package/sdk-core/core/src/pollers/poll_buffer.rs +4 -4
- package/sdk-core/core/src/replay/mod.rs +3 -3
- package/sdk-core/core/src/retry_logic.rs +10 -9
- package/sdk-core/core/src/telemetry/metrics.rs +48 -39
- package/sdk-core/core/src/telemetry/mod.rs +46 -12
- package/sdk-core/core/src/telemetry/prometheus_server.rs +17 -13
- package/sdk-core/core/src/test_help/mod.rs +18 -8
- package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +10 -10
- package/sdk-core/core/src/worker/activities/local_activities.rs +13 -13
- package/sdk-core/core/src/worker/activities.rs +6 -12
- package/sdk-core/core/src/worker/client/mocks.rs +1 -0
- package/sdk-core/core/src/worker/client.rs +193 -64
- package/sdk-core/core/src/worker/mod.rs +14 -19
- package/sdk-core/core/src/worker/workflow/driven_workflow.rs +3 -0
- package/sdk-core/core/src/worker/workflow/history_update.rs +5 -5
- package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +133 -85
- package/sdk-core/core/src/worker/workflow/machines/mod.rs +3 -2
- package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +160 -105
- package/sdk-core/core/src/worker/workflow/managed_run.rs +2 -1
- package/sdk-core/core/src/worker/workflow/mod.rs +62 -58
- package/sdk-core/core/src/worker/workflow/run_cache.rs +5 -3
- package/sdk-core/core/src/worker/workflow/workflow_stream.rs +7 -5
- package/sdk-core/core-api/Cargo.toml +3 -3
- package/sdk-core/core-api/src/errors.rs +3 -11
- package/sdk-core/core-api/src/worker.rs +7 -0
- package/sdk-core/protos/api_upstream/.buildkite/Dockerfile +1 -1
- package/sdk-core/protos/api_upstream/.github/CODEOWNERS +1 -1
- package/sdk-core/protos/api_upstream/.github/PULL_REQUEST_TEMPLATE.md +2 -6
- package/sdk-core/protos/api_upstream/.github/workflows/trigger-api-go-update.yml +29 -0
- package/sdk-core/protos/api_upstream/Makefile +2 -2
- package/sdk-core/protos/api_upstream/buf.yaml +1 -0
- package/sdk-core/protos/api_upstream/temporal/api/batch/v1/message.proto +86 -0
- package/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto +26 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +46 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/command_type.proto +7 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/event_type.proto +14 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/update.proto +51 -0
- package/sdk-core/protos/api_upstream/temporal/api/failure/v1/message.proto +18 -0
- package/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto +57 -1
- package/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +1 -3
- package/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +4 -2
- package/sdk-core/protos/api_upstream/temporal/api/replication/v1/message.proto +11 -0
- package/sdk-core/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +23 -0
- package/sdk-core/protos/api_upstream/temporal/api/update/v1/message.proto +46 -0
- package/sdk-core/protos/api_upstream/temporal/api/workflow/v1/message.proto +1 -0
- package/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +172 -0
- package/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +30 -0
- package/sdk-core/protos/grpc/health/v1/health.proto +63 -0
- package/sdk-core/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +18 -15
- package/sdk-core/protos/testsrv_upstream/Makefile +80 -0
- package/sdk-core/protos/testsrv_upstream/api-linter.yaml +38 -0
- package/sdk-core/protos/testsrv_upstream/buf.yaml +13 -0
- package/sdk-core/protos/testsrv_upstream/dependencies/gogoproto/gogo.proto +141 -0
- package/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/request_response.proto +63 -0
- package/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/service.proto +90 -0
- package/sdk-core/sdk/Cargo.toml +2 -2
- package/sdk-core/sdk/src/lib.rs +2 -2
- package/sdk-core/sdk/src/workflow_context/options.rs +36 -8
- package/sdk-core/sdk/src/workflow_context.rs +30 -6
- package/sdk-core/sdk/src/workflow_future.rs +4 -4
- package/sdk-core/sdk-core-protos/Cargo.toml +5 -5
- package/sdk-core/sdk-core-protos/build.rs +9 -1
- package/sdk-core/sdk-core-protos/src/history_builder.rs +6 -1
- package/sdk-core/sdk-core-protos/src/lib.rs +93 -32
- package/sdk-core/test-utils/Cargo.toml +3 -3
- package/sdk-core/test-utils/src/canned_histories.rs +58 -0
- package/sdk-core/test-utils/src/lib.rs +35 -12
- package/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +128 -0
- package/sdk-core/tests/integ_tests/heartbeat_tests.rs +55 -5
- package/sdk-core/tests/integ_tests/polling_tests.rs +2 -1
- package/sdk-core/tests/integ_tests/queries_tests.rs +5 -5
- package/sdk-core/tests/integ_tests/visibility_tests.rs +93 -0
- package/sdk-core/tests/integ_tests/workflow_tests/activities.rs +93 -10
- package/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +1 -1
- package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +14 -14
- package/sdk-core/tests/integ_tests/workflow_tests/replay.rs +2 -6
- package/sdk-core/tests/integ_tests/workflow_tests/resets.rs +12 -12
- package/sdk-core/tests/integ_tests/workflow_tests/signals.rs +12 -1
- package/sdk-core/tests/integ_tests/workflow_tests/timers.rs +3 -3
- package/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +8 -2
- package/sdk-core/tests/integ_tests/workflow_tests.rs +19 -4
- package/sdk-core/tests/load_tests.rs +2 -1
- package/sdk-core/tests/main.rs +17 -0
- package/sdk-core/tests/runner.rs +93 -0
- package/src/conversions.rs +157 -94
- package/src/helpers.rs +190 -0
- package/src/lib.rs +10 -912
- package/src/runtime.rs +436 -0
- package/src/testing.rs +67 -0
- package/src/worker.rs +465 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
use crate::{
|
|
2
2
|
pollers::{self, Poller},
|
|
3
|
-
worker::client::
|
|
3
|
+
worker::client::WorkerClient,
|
|
4
4
|
};
|
|
5
5
|
use futures::{prelude::stream::FuturesUnordered, StreamExt};
|
|
6
6
|
use std::{
|
|
@@ -199,7 +199,7 @@ impl Poller<PollWorkflowTaskQueueResponse> for WorkflowTaskPoller {
|
|
|
199
199
|
|
|
200
200
|
pub type PollWorkflowTaskBuffer = LongPollBuffer<PollWorkflowTaskQueueResponse>;
|
|
201
201
|
pub(crate) fn new_workflow_task_buffer(
|
|
202
|
-
client: Arc<
|
|
202
|
+
client: Arc<dyn WorkerClient>,
|
|
203
203
|
task_queue: String,
|
|
204
204
|
is_sticky: bool,
|
|
205
205
|
concurrent_pollers: usize,
|
|
@@ -220,7 +220,7 @@ pub(crate) fn new_workflow_task_buffer(
|
|
|
220
220
|
|
|
221
221
|
pub type PollActivityTaskBuffer = LongPollBuffer<PollActivityTaskQueueResponse>;
|
|
222
222
|
pub(crate) fn new_activity_task_buffer(
|
|
223
|
-
client: Arc<
|
|
223
|
+
client: Arc<dyn WorkerClient>,
|
|
224
224
|
task_queue: String,
|
|
225
225
|
concurrent_pollers: usize,
|
|
226
226
|
buffer_size: usize,
|
|
@@ -262,7 +262,7 @@ mod tests {
|
|
|
262
262
|
});
|
|
263
263
|
|
|
264
264
|
let pb = new_workflow_task_buffer(
|
|
265
|
-
Arc::new(mock_client
|
|
265
|
+
Arc::new(mock_client),
|
|
266
266
|
"someq".to_string(),
|
|
267
267
|
false,
|
|
268
268
|
1,
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
//! to replay canned histories. It should be used by Lang SDKs to provide replay capabilities to
|
|
3
3
|
//! users during testing.
|
|
4
4
|
|
|
5
|
-
use crate::
|
|
5
|
+
use crate::worker::client::{mocks::mock_manual_workflow_client, WorkerClient};
|
|
6
6
|
use futures::FutureExt;
|
|
7
7
|
use std::{
|
|
8
8
|
sync::{
|
|
@@ -29,7 +29,7 @@ pub use temporal_sdk_core_protos::{
|
|
|
29
29
|
pub(crate) fn mock_client_from_history(
|
|
30
30
|
history: &History,
|
|
31
31
|
task_queue: impl Into<String>,
|
|
32
|
-
) ->
|
|
32
|
+
) -> impl WorkerClient {
|
|
33
33
|
let mut mg = mock_manual_workflow_client();
|
|
34
34
|
|
|
35
35
|
let hist_info = HistoryInfo::new_from_history(history, None).unwrap();
|
|
@@ -67,5 +67,5 @@ pub(crate) fn mock_client_from_history(
|
|
|
67
67
|
async move { Ok(RespondWorkflowTaskFailedResponse {}) }.boxed()
|
|
68
68
|
});
|
|
69
69
|
|
|
70
|
-
|
|
70
|
+
mg
|
|
71
71
|
}
|
|
@@ -81,7 +81,7 @@ const NANOS_PER_SEC: u32 = 1_000_000_000;
|
|
|
81
81
|
fn try_from_secs_f64(secs: f64) -> Option<Duration> {
|
|
82
82
|
const MAX_NANOS_F64: f64 = ((u64::MAX as u128 + 1) * (NANOS_PER_SEC as u128)) as f64;
|
|
83
83
|
let nanos = secs * (NANOS_PER_SEC as f64);
|
|
84
|
-
if !nanos.is_finite() ||
|
|
84
|
+
if !nanos.is_finite() || !(0.0..MAX_NANOS_F64).contains(&nanos) {
|
|
85
85
|
None
|
|
86
86
|
} else {
|
|
87
87
|
Some(Duration::from_secs_f64(secs))
|
|
@@ -91,13 +91,14 @@ fn try_from_secs_f64(secs: f64) -> Option<Duration> {
|
|
|
91
91
|
#[cfg(test)]
|
|
92
92
|
mod tests {
|
|
93
93
|
use super::*;
|
|
94
|
+
use crate::prost_dur;
|
|
94
95
|
|
|
95
96
|
#[test]
|
|
96
97
|
fn calcs_backoffs_properly() {
|
|
97
98
|
let rp = RetryPolicy {
|
|
98
|
-
initial_interval: Some(
|
|
99
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
|
99
100
|
backoff_coefficient: 2.0,
|
|
100
|
-
maximum_interval: Some(
|
|
101
|
+
maximum_interval: Some(prost_dur!(from_secs(10))),
|
|
101
102
|
maximum_attempts: 10,
|
|
102
103
|
non_retryable_error_types: vec![],
|
|
103
104
|
};
|
|
@@ -132,7 +133,7 @@ mod tests {
|
|
|
132
133
|
#[test]
|
|
133
134
|
fn max_attempts_zero_retry_forever() {
|
|
134
135
|
let rp = RetryPolicy {
|
|
135
|
-
initial_interval: Some(
|
|
136
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
|
136
137
|
backoff_coefficient: 1.2,
|
|
137
138
|
maximum_interval: None,
|
|
138
139
|
maximum_attempts: 0,
|
|
@@ -146,7 +147,7 @@ mod tests {
|
|
|
146
147
|
#[test]
|
|
147
148
|
fn no_overflows() {
|
|
148
149
|
let rp = RetryPolicy {
|
|
149
|
-
initial_interval: Some(
|
|
150
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
|
150
151
|
backoff_coefficient: 10.,
|
|
151
152
|
maximum_interval: None,
|
|
152
153
|
maximum_attempts: 0,
|
|
@@ -160,9 +161,9 @@ mod tests {
|
|
|
160
161
|
#[test]
|
|
161
162
|
fn no_retry_err_str_match() {
|
|
162
163
|
let rp = RetryPolicy {
|
|
163
|
-
initial_interval: Some(
|
|
164
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
|
164
165
|
backoff_coefficient: 2.0,
|
|
165
|
-
maximum_interval: Some(
|
|
166
|
+
maximum_interval: Some(prost_dur!(from_secs(10))),
|
|
166
167
|
maximum_attempts: 10,
|
|
167
168
|
non_retryable_error_types: vec!["no retry".to_string()],
|
|
168
169
|
};
|
|
@@ -181,9 +182,9 @@ mod tests {
|
|
|
181
182
|
#[test]
|
|
182
183
|
fn no_non_retryable_application_failure() {
|
|
183
184
|
let rp = RetryPolicy {
|
|
184
|
-
initial_interval: Some(
|
|
185
|
+
initial_interval: Some(prost_dur!(from_secs(1))),
|
|
185
186
|
backoff_coefficient: 2.0,
|
|
186
|
-
maximum_interval: Some(
|
|
187
|
+
maximum_interval: Some(prost_dur!(from_secs(10))),
|
|
187
188
|
maximum_attempts: 10,
|
|
188
189
|
non_retryable_error_types: vec![],
|
|
189
190
|
};
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
use super::TELEM_SERVICE_NAME;
|
|
2
2
|
use crate::telemetry::GLOBAL_TELEM_DAT;
|
|
3
|
+
use opentelemetry::sdk::metrics::aggregators::Aggregator;
|
|
4
|
+
use opentelemetry::sdk::metrics::sdk_api::{Descriptor, InstrumentKind};
|
|
3
5
|
use opentelemetry::{
|
|
4
6
|
global,
|
|
5
|
-
metrics::{Counter,
|
|
7
|
+
metrics::{Counter, Histogram, Meter},
|
|
6
8
|
sdk::{
|
|
7
|
-
export::metrics::
|
|
9
|
+
export::metrics::AggregatorSelector,
|
|
8
10
|
metrics::aggregators::{histogram, last_value, sum},
|
|
9
11
|
},
|
|
10
|
-
KeyValue,
|
|
12
|
+
Context, KeyValue,
|
|
11
13
|
};
|
|
12
|
-
use std::{
|
|
14
|
+
use std::{sync::Arc, time::Duration};
|
|
13
15
|
|
|
14
16
|
/// Used to track context associated with metrics, and record/update them
|
|
15
17
|
///
|
|
@@ -17,12 +19,16 @@ use std::{borrow::Cow, sync::Arc, time::Duration};
|
|
|
17
19
|
/// appropriate k/vs have already been set.
|
|
18
20
|
#[derive(Default, Clone, Debug)]
|
|
19
21
|
pub(crate) struct MetricsContext {
|
|
22
|
+
ctx: Context,
|
|
20
23
|
kvs: Arc<Vec<KeyValue>>,
|
|
21
24
|
}
|
|
22
25
|
|
|
23
26
|
impl MetricsContext {
|
|
24
27
|
fn new(kvs: Vec<KeyValue>) -> Self {
|
|
25
|
-
Self {
|
|
28
|
+
Self {
|
|
29
|
+
ctx: Context::current(),
|
|
30
|
+
kvs: Arc::new(kvs),
|
|
31
|
+
}
|
|
26
32
|
}
|
|
27
33
|
|
|
28
34
|
pub(crate) fn top_level(namespace: String) -> Self {
|
|
@@ -38,113 +44,116 @@ impl MetricsContext {
|
|
|
38
44
|
pub(crate) fn with_new_attrs(&self, new_kvs: impl IntoIterator<Item = KeyValue>) -> Self {
|
|
39
45
|
let mut kvs = self.kvs.clone();
|
|
40
46
|
Arc::make_mut(&mut kvs).extend(new_kvs);
|
|
41
|
-
Self {
|
|
47
|
+
Self {
|
|
48
|
+
ctx: Context::current(),
|
|
49
|
+
kvs,
|
|
50
|
+
}
|
|
42
51
|
}
|
|
43
52
|
|
|
44
53
|
/// A workflow task queue poll succeeded
|
|
45
54
|
pub(crate) fn wf_tq_poll_ok(&self) {
|
|
46
|
-
WF_TASK_QUEUE_POLL_SUCCEED_COUNTER.add(1, &self.kvs);
|
|
55
|
+
WF_TASK_QUEUE_POLL_SUCCEED_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
47
56
|
}
|
|
48
57
|
|
|
49
58
|
/// A workflow task queue poll timed out / had empty response
|
|
50
59
|
pub(crate) fn wf_tq_poll_empty(&self) {
|
|
51
|
-
WF_TASK_QUEUE_POLL_EMPTY_COUNTER.add(1, &self.kvs);
|
|
60
|
+
WF_TASK_QUEUE_POLL_EMPTY_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
52
61
|
}
|
|
53
62
|
|
|
54
63
|
/// A workflow task execution failed
|
|
55
64
|
pub(crate) fn wf_task_failed(&self) {
|
|
56
|
-
WF_TASK_EXECUTION_FAILURE_COUNTER.add(1, &self.kvs);
|
|
65
|
+
WF_TASK_EXECUTION_FAILURE_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
57
66
|
}
|
|
58
67
|
|
|
59
68
|
/// A workflow completed successfully
|
|
60
69
|
pub(crate) fn wf_completed(&self) {
|
|
61
|
-
WF_COMPLETED_COUNTER.add(1, &self.kvs);
|
|
70
|
+
WF_COMPLETED_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
/// A workflow ended cancelled
|
|
65
74
|
pub(crate) fn wf_canceled(&self) {
|
|
66
|
-
WF_CANCELED_COUNTER.add(1, &self.kvs);
|
|
75
|
+
WF_CANCELED_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
67
76
|
}
|
|
68
77
|
|
|
69
78
|
/// A workflow ended failed
|
|
70
79
|
pub(crate) fn wf_failed(&self) {
|
|
71
|
-
WF_FAILED_COUNTER.add(1, &self.kvs);
|
|
80
|
+
WF_FAILED_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
72
81
|
}
|
|
73
82
|
|
|
74
83
|
/// A workflow continued as new
|
|
75
84
|
pub(crate) fn wf_continued_as_new(&self) {
|
|
76
|
-
WF_CONT_COUNTER.add(1, &self.kvs);
|
|
85
|
+
WF_CONT_COUNTER.add(&self.ctx, 1, &self.kvs);
|
|
77
86
|
}
|
|
78
87
|
|
|
79
88
|
/// Record workflow total execution time in milliseconds
|
|
80
89
|
pub(crate) fn wf_e2e_latency(&self, dur: Duration) {
|
|
81
|
-
WF_E2E_LATENCY.record(dur.as_millis() as u64, &self.kvs);
|
|
90
|
+
WF_E2E_LATENCY.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
82
91
|
}
|
|
83
92
|
|
|
84
93
|
/// Record workflow task schedule to start time in millis
|
|
85
94
|
pub(crate) fn wf_task_sched_to_start_latency(&self, dur: Duration) {
|
|
86
|
-
WF_TASK_SCHED_TO_START_LATENCY.record(dur.as_millis() as u64, &self.kvs);
|
|
95
|
+
WF_TASK_SCHED_TO_START_LATENCY.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
87
96
|
}
|
|
88
97
|
|
|
89
98
|
/// Record workflow task execution time in milliseconds
|
|
90
99
|
pub(crate) fn wf_task_latency(&self, dur: Duration) {
|
|
91
|
-
WF_TASK_EXECUTION_LATENCY.record(dur.as_millis() as u64, &self.kvs);
|
|
100
|
+
WF_TASK_EXECUTION_LATENCY.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
92
101
|
}
|
|
93
102
|
|
|
94
103
|
/// Record time it takes to catch up on replaying a WFT
|
|
95
104
|
pub(crate) fn wf_task_replay_latency(&self, dur: Duration) {
|
|
96
|
-
WF_TASK_REPLAY_LATENCY.record(dur.as_millis() as u64, &self.kvs);
|
|
105
|
+
WF_TASK_REPLAY_LATENCY.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
97
106
|
}
|
|
98
107
|
|
|
99
108
|
/// An activity long poll timed out
|
|
100
109
|
pub(crate) fn act_poll_timeout(&self) {
|
|
101
|
-
ACT_POLL_NO_TASK.add(1, &self.kvs);
|
|
110
|
+
ACT_POLL_NO_TASK.add(&self.ctx, 1, &self.kvs);
|
|
102
111
|
}
|
|
103
112
|
|
|
104
113
|
/// An activity execution failed
|
|
105
114
|
pub(crate) fn act_execution_failed(&self) {
|
|
106
|
-
ACT_EXECUTION_FAILED.add(1, &self.kvs);
|
|
115
|
+
ACT_EXECUTION_FAILED.add(&self.ctx, 1, &self.kvs);
|
|
107
116
|
}
|
|
108
117
|
|
|
109
118
|
/// Record activity task schedule to start time in millis
|
|
110
119
|
pub(crate) fn act_sched_to_start_latency(&self, dur: Duration) {
|
|
111
|
-
ACT_SCHED_TO_START_LATENCY.record(dur.as_millis() as u64, &self.kvs);
|
|
120
|
+
ACT_SCHED_TO_START_LATENCY.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
112
121
|
}
|
|
113
122
|
|
|
114
123
|
/// Record time it took to complete activity execution, from the time core generated the
|
|
115
124
|
/// activity task, to the time lang responded with a completion (failure or success).
|
|
116
125
|
pub(crate) fn act_execution_latency(&self, dur: Duration) {
|
|
117
|
-
ACT_EXEC_LATENCY.record(dur.as_millis() as u64, &self.kvs);
|
|
126
|
+
ACT_EXEC_LATENCY.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
118
127
|
}
|
|
119
128
|
|
|
120
129
|
/// A worker was registered
|
|
121
130
|
pub(crate) fn worker_registered(&self) {
|
|
122
|
-
WORKER_REGISTERED.add(1, &self.kvs);
|
|
131
|
+
WORKER_REGISTERED.add(&self.ctx, 1, &self.kvs);
|
|
123
132
|
}
|
|
124
133
|
|
|
125
134
|
/// Record current number of available task slots. Context should have worker type set.
|
|
126
135
|
pub(crate) fn available_task_slots(&self, num: usize) {
|
|
127
|
-
TASK_SLOTS_AVAILABLE.record(num as u64, &self.kvs)
|
|
136
|
+
TASK_SLOTS_AVAILABLE.record(&self.ctx, num as u64, &self.kvs)
|
|
128
137
|
}
|
|
129
138
|
|
|
130
139
|
/// Record current number of pollers. Context should include poller type / task queue tag.
|
|
131
140
|
pub(crate) fn record_num_pollers(&self, num: usize) {
|
|
132
|
-
NUM_POLLERS.record(num as u64, &self.kvs);
|
|
141
|
+
NUM_POLLERS.record(&self.ctx, num as u64, &self.kvs);
|
|
133
142
|
}
|
|
134
143
|
|
|
135
144
|
/// A workflow task found a cached workflow to run against
|
|
136
145
|
pub(crate) fn sticky_cache_hit(&self) {
|
|
137
|
-
STICKY_CACHE_HIT.add(1, &self.kvs);
|
|
146
|
+
STICKY_CACHE_HIT.add(&self.ctx, 1, &self.kvs);
|
|
138
147
|
}
|
|
139
148
|
|
|
140
149
|
/// A workflow task did not find a cached workflow
|
|
141
150
|
pub(crate) fn sticky_cache_miss(&self) {
|
|
142
|
-
STICKY_CACHE_MISS.add(1, &self.kvs);
|
|
151
|
+
STICKY_CACHE_MISS.add(&self.ctx, 1, &self.kvs);
|
|
143
152
|
}
|
|
144
153
|
|
|
145
154
|
/// Record current cache size (in number of wfs, not bytes)
|
|
146
155
|
pub(crate) fn cache_size(&self, size: u64) {
|
|
147
|
-
STICKY_CACHE_SIZE.record(size, &self.kvs);
|
|
156
|
+
STICKY_CACHE_SIZE.record(&self.ctx, size, &self.kvs);
|
|
148
157
|
}
|
|
149
158
|
}
|
|
150
159
|
|
|
@@ -182,8 +191,8 @@ macro_rules! tm {
|
|
|
182
191
|
};
|
|
183
192
|
(vr_u64, $ident:ident, $name:expr) => {
|
|
184
193
|
lazy_static::lazy_static! {
|
|
185
|
-
static ref $ident:
|
|
186
|
-
METRIC_METER.
|
|
194
|
+
static ref $ident: Histogram<u64> = {
|
|
195
|
+
METRIC_METER.u64_histogram(metric_prefix().to_string() + $name).init()
|
|
187
196
|
};
|
|
188
197
|
}
|
|
189
198
|
};
|
|
@@ -214,22 +223,22 @@ pub(crate) fn activity_type(ty: String) -> KeyValue {
|
|
|
214
223
|
pub(crate) fn workflow_type(ty: String) -> KeyValue {
|
|
215
224
|
KeyValue::new(KEY_WF_TYPE, ty)
|
|
216
225
|
}
|
|
217
|
-
pub(crate)
|
|
226
|
+
pub(crate) fn workflow_worker_type() -> KeyValue {
|
|
218
227
|
KeyValue {
|
|
219
228
|
key: opentelemetry::Key::from_static_str(KEY_WORKER_TYPE),
|
|
220
|
-
value: opentelemetry::Value::String(
|
|
229
|
+
value: opentelemetry::Value::String("WorkflowWorker".into()),
|
|
221
230
|
}
|
|
222
231
|
}
|
|
223
|
-
pub(crate)
|
|
232
|
+
pub(crate) fn activity_worker_type() -> KeyValue {
|
|
224
233
|
KeyValue {
|
|
225
234
|
key: opentelemetry::Key::from_static_str(KEY_WORKER_TYPE),
|
|
226
|
-
value: opentelemetry::Value::String(
|
|
235
|
+
value: opentelemetry::Value::String("ActivityWorker".into()),
|
|
227
236
|
}
|
|
228
237
|
}
|
|
229
|
-
pub(crate)
|
|
238
|
+
pub(crate) fn local_activity_worker_type() -> KeyValue {
|
|
230
239
|
KeyValue {
|
|
231
240
|
key: opentelemetry::Key::from_static_str(KEY_WORKER_TYPE),
|
|
232
|
-
value: opentelemetry::Value::String(
|
|
241
|
+
value: opentelemetry::Value::String("LocalActivityWorker".into()),
|
|
233
242
|
}
|
|
234
243
|
}
|
|
235
244
|
|
|
@@ -338,12 +347,12 @@ pub struct SDKAggSelector;
|
|
|
338
347
|
|
|
339
348
|
impl AggregatorSelector for SDKAggSelector {
|
|
340
349
|
fn aggregator_for(&self, descriptor: &Descriptor) -> Option<Arc<dyn Aggregator + Send + Sync>> {
|
|
341
|
-
//
|
|
342
|
-
if *descriptor.instrument_kind() == InstrumentKind::
|
|
350
|
+
// Gauges are always last value
|
|
351
|
+
if *descriptor.instrument_kind() == InstrumentKind::GaugeObserver {
|
|
343
352
|
return Some(Arc::new(last_value()));
|
|
344
353
|
}
|
|
345
354
|
|
|
346
|
-
if *descriptor.instrument_kind() == InstrumentKind::
|
|
355
|
+
if *descriptor.instrument_kind() == InstrumentKind::Histogram {
|
|
347
356
|
let dname = descriptor
|
|
348
357
|
.name()
|
|
349
358
|
.strip_prefix(metric_prefix())
|
|
@@ -366,7 +375,7 @@ impl AggregatorSelector for SDKAggSelector {
|
|
|
366
375
|
ACT_EXEC_LATENCY_NAME => ACT_EXE_MS_BUCKETS,
|
|
367
376
|
_ => DEFAULT_MS_BUCKETS,
|
|
368
377
|
};
|
|
369
|
-
return Some(Arc::new(histogram(
|
|
378
|
+
return Some(Arc::new(histogram(buckets)));
|
|
370
379
|
}
|
|
371
380
|
|
|
372
381
|
Some(Arc::new(sum()))
|
|
@@ -12,8 +12,12 @@ use once_cell::sync::OnceCell;
|
|
|
12
12
|
use opentelemetry::{
|
|
13
13
|
global,
|
|
14
14
|
metrics::Meter,
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
runtime,
|
|
16
|
+
sdk::{
|
|
17
|
+
export::metrics::aggregation::{self, Temporality, TemporalitySelector},
|
|
18
|
+
trace::Config,
|
|
19
|
+
Resource,
|
|
20
|
+
},
|
|
17
21
|
KeyValue,
|
|
18
22
|
};
|
|
19
23
|
use opentelemetry_otlp::WithExportConfig;
|
|
@@ -69,7 +73,7 @@ pub enum MetricsExporter {
|
|
|
69
73
|
}
|
|
70
74
|
|
|
71
75
|
/// Control where logs go
|
|
72
|
-
#[derive(Debug, Clone)]
|
|
76
|
+
#[derive(Debug, Clone, Copy)]
|
|
73
77
|
pub enum Logger {
|
|
74
78
|
/// Log directly to console.
|
|
75
79
|
Console,
|
|
@@ -101,6 +105,33 @@ pub struct TelemetryOptions {
|
|
|
101
105
|
/// the prefix is consistent with other SDKs.
|
|
102
106
|
#[builder(default)]
|
|
103
107
|
pub no_temporal_prefix_for_metrics: bool,
|
|
108
|
+
|
|
109
|
+
/// Specifies the aggregation temporality for metric export. Defaults to cumulative.
|
|
110
|
+
#[builder(default = "MetricTemporality::Cumulative")]
|
|
111
|
+
pub metric_temporality: MetricTemporality,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/// Types of aggregation temporality for metric export.
|
|
115
|
+
/// See: <https://github.com/open-telemetry/opentelemetry-specification/blob/ce50e4634efcba8da445cc23523243cb893905cb/specification/metrics/datamodel.md#temporality>
|
|
116
|
+
#[derive(Debug, Clone, Copy)]
|
|
117
|
+
pub enum MetricTemporality {
|
|
118
|
+
/// Successive data points repeat the starting timestamp
|
|
119
|
+
Cumulative,
|
|
120
|
+
/// Successive data points advance the starting timestamp
|
|
121
|
+
Delta,
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
impl MetricTemporality {
|
|
125
|
+
fn to_selector(self) -> impl TemporalitySelector + Send + Sync + Clone {
|
|
126
|
+
match self {
|
|
127
|
+
MetricTemporality::Cumulative => {
|
|
128
|
+
aggregation::constant_temporality_selector(Temporality::Cumulative)
|
|
129
|
+
}
|
|
130
|
+
MetricTemporality::Delta => {
|
|
131
|
+
aggregation::constant_temporality_selector(Temporality::Delta)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
104
135
|
}
|
|
105
136
|
|
|
106
137
|
impl TelemetryOptions {
|
|
@@ -123,7 +154,6 @@ impl Default for TelemetryOptions {
|
|
|
123
154
|
/// Things that need to not be dropped while telemetry is ongoing
|
|
124
155
|
#[derive(Default)]
|
|
125
156
|
pub struct GlobalTelemDat {
|
|
126
|
-
metric_push_controller: Option<PushController>,
|
|
127
157
|
core_export_logger: Option<CoreExportLogger>,
|
|
128
158
|
runtime: Option<tokio::runtime::Runtime>,
|
|
129
159
|
prom_srv: Option<PromServer>,
|
|
@@ -202,7 +232,7 @@ pub fn telemetry_init(opts: &TelemetryOptions) -> Result<&'static GlobalTelemDat
|
|
|
202
232
|
.pretty()
|
|
203
233
|
.with_source_location(false);
|
|
204
234
|
let reg = tracing_subscriber::registry()
|
|
205
|
-
.with((
|
|
235
|
+
.with((opts).try_get_env_filter()?)
|
|
206
236
|
.with(
|
|
207
237
|
tracing_subscriber::fmt::layer()
|
|
208
238
|
.with_target(false)
|
|
@@ -221,16 +251,19 @@ pub fn telemetry_init(opts: &TelemetryOptions) -> Result<&'static GlobalTelemDat
|
|
|
221
251
|
if let Some(ref metrics) = opts.metrics {
|
|
222
252
|
match metrics {
|
|
223
253
|
MetricsExporter::Prometheus(addr) => {
|
|
224
|
-
let srv = PromServer::new(*addr)?;
|
|
254
|
+
let srv = PromServer::new(*addr, opts.metric_temporality.to_selector())?;
|
|
225
255
|
globaldat.prom_srv = Some(srv);
|
|
226
256
|
}
|
|
227
257
|
MetricsExporter::Otel(OtelCollectorOptions { url, headers }) => {
|
|
228
258
|
runtime.block_on(async {
|
|
229
259
|
let metrics = opentelemetry_otlp::new_pipeline()
|
|
230
|
-
.metrics(
|
|
231
|
-
|
|
260
|
+
.metrics(
|
|
261
|
+
SDKAggSelector,
|
|
262
|
+
opts.metric_temporality.to_selector(),
|
|
263
|
+
runtime::Tokio,
|
|
264
|
+
)
|
|
232
265
|
.with_period(Duration::from_secs(1))
|
|
233
|
-
.with_resource(
|
|
266
|
+
.with_resource(default_resource())
|
|
234
267
|
.with_exporter(
|
|
235
268
|
// No joke exporter builder literally not cloneable for some insane
|
|
236
269
|
// reason
|
|
@@ -242,8 +275,7 @@ pub fn telemetry_init(opts: &TelemetryOptions) -> Result<&'static GlobalTelemDat
|
|
|
242
275
|
)),
|
|
243
276
|
)
|
|
244
277
|
.build()?;
|
|
245
|
-
global::set_meter_provider(metrics
|
|
246
|
-
globaldat.metric_push_controller = Some(metrics);
|
|
278
|
+
global::set_meter_provider(metrics);
|
|
247
279
|
Result::<(), anyhow::Error>::Ok(())
|
|
248
280
|
})?;
|
|
249
281
|
}
|
|
@@ -266,7 +298,7 @@ pub fn telemetry_init(opts: &TelemetryOptions) -> Result<&'static GlobalTelemDat
|
|
|
266
298
|
)),
|
|
267
299
|
)
|
|
268
300
|
.with_trace_config(tracer_cfg)
|
|
269
|
-
.install_batch(
|
|
301
|
+
.install_batch(runtime::Tokio)?;
|
|
270
302
|
|
|
271
303
|
let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer);
|
|
272
304
|
|
|
@@ -331,6 +363,7 @@ pub(crate) fn test_telem_console() {
|
|
|
331
363
|
tracing: None,
|
|
332
364
|
metrics: None,
|
|
333
365
|
no_temporal_prefix_for_metrics: false,
|
|
366
|
+
metric_temporality: MetricTemporality::Cumulative,
|
|
334
367
|
})
|
|
335
368
|
.unwrap();
|
|
336
369
|
}
|
|
@@ -347,6 +380,7 @@ pub(crate) fn test_telem_collector() {
|
|
|
347
380
|
})),
|
|
348
381
|
metrics: None,
|
|
349
382
|
no_temporal_prefix_for_metrics: false,
|
|
383
|
+
metric_temporality: MetricTemporality::Cumulative,
|
|
350
384
|
})
|
|
351
385
|
.unwrap();
|
|
352
386
|
}
|
|
@@ -1,13 +1,16 @@
|
|
|
1
|
-
use crate::telemetry::{
|
|
2
|
-
default_resource,
|
|
3
|
-
metrics::{SDKAggSelector, DEFAULT_MS_BUCKETS},
|
|
4
|
-
};
|
|
1
|
+
use crate::telemetry::{default_resource, metrics::SDKAggSelector};
|
|
5
2
|
use hyper::{
|
|
6
3
|
header::CONTENT_TYPE,
|
|
7
4
|
service::{make_service_fn, service_fn},
|
|
8
5
|
Body, Method, Request, Response, Server,
|
|
9
6
|
};
|
|
10
|
-
use opentelemetry::
|
|
7
|
+
use opentelemetry::{
|
|
8
|
+
metrics::MetricsError,
|
|
9
|
+
sdk::{
|
|
10
|
+
export::metrics::aggregation::TemporalitySelector,
|
|
11
|
+
metrics::{controllers, processors},
|
|
12
|
+
},
|
|
13
|
+
};
|
|
11
14
|
use opentelemetry_prometheus::{ExporterBuilder, PrometheusExporter};
|
|
12
15
|
use prometheus::{Encoder, TextEncoder};
|
|
13
16
|
use std::{convert::Infallible, net::SocketAddr, sync::Arc};
|
|
@@ -19,14 +22,15 @@ pub(super) struct PromServer {
|
|
|
19
22
|
}
|
|
20
23
|
|
|
21
24
|
impl PromServer {
|
|
22
|
-
pub fn new(
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
pub fn new(
|
|
26
|
+
addr: SocketAddr,
|
|
27
|
+
temporality: impl TemporalitySelector + Send + Sync + 'static,
|
|
28
|
+
) -> Result<Self, MetricsError> {
|
|
29
|
+
let controller =
|
|
30
|
+
controllers::basic(processors::factory(SDKAggSelector, temporality).with_memory(true))
|
|
31
|
+
.with_resource(default_resource())
|
|
32
|
+
.build();
|
|
33
|
+
let exporter = ExporterBuilder::new(controller).try_init()?;
|
|
30
34
|
Ok(Self {
|
|
31
35
|
exporter: Arc::new(exporter),
|
|
32
36
|
addr,
|
|
@@ -9,7 +9,7 @@ use crate::{
|
|
|
9
9
|
client::{mocks::mock_workflow_client, MockWorkerClient, WorkerClient},
|
|
10
10
|
new_wft_poller,
|
|
11
11
|
},
|
|
12
|
-
TaskToken, Worker,
|
|
12
|
+
TaskToken, Worker, WorkerConfig, WorkerConfigBuilder,
|
|
13
13
|
};
|
|
14
14
|
use bimap::BiMap;
|
|
15
15
|
use futures::{future::BoxFuture, stream, stream::BoxStream, FutureExt, Stream, StreamExt};
|
|
@@ -136,7 +136,7 @@ pub(crate) fn mock_worker(mocks: MocksHolder) -> Worker {
|
|
|
136
136
|
Worker::new_with_pollers(
|
|
137
137
|
mocks.inputs.config,
|
|
138
138
|
sticky_q,
|
|
139
|
-
|
|
139
|
+
mocks.client,
|
|
140
140
|
mocks.inputs.wft_stream,
|
|
141
141
|
mocks.inputs.act_poller,
|
|
142
142
|
Default::default(),
|
|
@@ -166,7 +166,7 @@ pub struct FakeWfResponses {
|
|
|
166
166
|
|
|
167
167
|
// TODO: Should be all-internal to this module
|
|
168
168
|
pub struct MocksHolder {
|
|
169
|
-
|
|
169
|
+
client: Arc<dyn WorkerClient>,
|
|
170
170
|
inputs: MockWorkerInputs,
|
|
171
171
|
pub outstanding_task_map: Option<OutstandingWFTMap>,
|
|
172
172
|
}
|
|
@@ -218,11 +218,11 @@ impl MockWorkerInputs {
|
|
|
218
218
|
|
|
219
219
|
impl MocksHolder {
|
|
220
220
|
pub(crate) fn from_mock_worker(
|
|
221
|
-
|
|
221
|
+
client: impl WorkerClient + 'static,
|
|
222
222
|
mock_worker: MockWorkerInputs,
|
|
223
223
|
) -> Self {
|
|
224
224
|
Self {
|
|
225
|
-
|
|
225
|
+
client: Arc::new(client),
|
|
226
226
|
inputs: mock_worker,
|
|
227
227
|
outstanding_task_map: None,
|
|
228
228
|
}
|
|
@@ -245,7 +245,7 @@ impl MocksHolder {
|
|
|
245
245
|
config: test_worker_cfg().build().unwrap(),
|
|
246
246
|
};
|
|
247
247
|
Self {
|
|
248
|
-
|
|
248
|
+
client: Arc::new(client),
|
|
249
249
|
inputs: mock_worker,
|
|
250
250
|
outstanding_task_map: None,
|
|
251
251
|
}
|
|
@@ -267,7 +267,7 @@ impl MocksHolder {
|
|
|
267
267
|
config: test_worker_cfg().build().unwrap(),
|
|
268
268
|
};
|
|
269
269
|
Self {
|
|
270
|
-
|
|
270
|
+
client: Arc::new(client),
|
|
271
271
|
inputs: mock_worker,
|
|
272
272
|
outstanding_task_map: None,
|
|
273
273
|
}
|
|
@@ -356,6 +356,7 @@ pub(crate) fn single_hist_mock_sg(
|
|
|
356
356
|
build_mock_pollers(mh)
|
|
357
357
|
}
|
|
358
358
|
|
|
359
|
+
#[allow(clippy::type_complexity)]
|
|
359
360
|
pub(crate) struct MockPollCfg {
|
|
360
361
|
pub hists: Vec<FakeWfResponses>,
|
|
361
362
|
pub enforce_correct_number_of_polls: bool,
|
|
@@ -604,7 +605,7 @@ pub(crate) fn build_mock_pollers(mut cfg: MockPollCfg) -> MocksHolder {
|
|
|
604
605
|
});
|
|
605
606
|
|
|
606
607
|
MocksHolder {
|
|
607
|
-
|
|
608
|
+
client: Arc::new(cfg.mock_client),
|
|
608
609
|
inputs: mock_worker,
|
|
609
610
|
outstanding_task_map: Some(outstanding_wf_task_tokens),
|
|
610
611
|
}
|
|
@@ -863,3 +864,12 @@ macro_rules! advance_fut {
|
|
|
863
864
|
}
|
|
864
865
|
};
|
|
865
866
|
}
|
|
867
|
+
|
|
868
|
+
#[macro_export]
|
|
869
|
+
macro_rules! prost_dur {
|
|
870
|
+
($dur_call:ident $args:tt) => {
|
|
871
|
+
std::time::Duration::$dur_call$args
|
|
872
|
+
.try_into()
|
|
873
|
+
.expect("test duration fits")
|
|
874
|
+
};
|
|
875
|
+
}
|