@temporalio/core-bridge 1.1.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +765 -128
- package/Cargo.toml +2 -2
- package/common.js +7 -3
- package/index.d.ts +118 -5
- package/index.js +2 -6
- package/package.json +2 -3
- package/releases/aarch64-apple-darwin/index.node +0 -0
- package/releases/aarch64-unknown-linux-gnu/index.node +0 -0
- package/releases/x86_64-apple-darwin/index.node +0 -0
- package/releases/x86_64-pc-windows-msvc/index.node +0 -0
- package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
- package/scripts/build.js +4 -3
- package/sdk-core/.buildkite/docker/Dockerfile +2 -1
- package/sdk-core/.buildkite/pipeline.yml +2 -0
- package/sdk-core/.cargo/config.toml +1 -1
- package/sdk-core/ARCHITECTURE.md +2 -2
- package/sdk-core/README.md +12 -0
- package/sdk-core/bridge-ffi/Cargo.toml +2 -2
- package/sdk-core/bridge-ffi/src/lib.rs +2 -2
- package/sdk-core/client/Cargo.toml +7 -5
- package/sdk-core/client/src/lib.rs +354 -226
- package/sdk-core/client/src/metrics.rs +13 -11
- package/sdk-core/client/src/raw.rs +352 -107
- package/sdk-core/client/src/retry.rs +188 -147
- package/sdk-core/client/src/workflow_handle/mod.rs +1 -1
- package/sdk-core/core/Cargo.toml +28 -15
- package/sdk-core/core/src/core_tests/activity_tasks.rs +98 -33
- package/sdk-core/core/src/core_tests/child_workflows.rs +125 -3
- package/sdk-core/core/src/core_tests/local_activities.rs +6 -6
- package/sdk-core/core/src/core_tests/workers.rs +3 -2
- package/sdk-core/core/src/core_tests/workflow_tasks.rs +70 -2
- package/sdk-core/core/src/ephemeral_server/mod.rs +515 -0
- package/sdk-core/core/src/lib.rs +62 -28
- package/sdk-core/core/src/pollers/mod.rs +2 -0
- package/sdk-core/core/src/pollers/poll_buffer.rs +4 -4
- package/sdk-core/core/src/replay/mod.rs +3 -3
- package/sdk-core/core/src/retry_logic.rs +10 -9
- package/sdk-core/core/src/telemetry/metrics.rs +48 -39
- package/sdk-core/core/src/telemetry/mod.rs +46 -12
- package/sdk-core/core/src/telemetry/prometheus_server.rs +17 -13
- package/sdk-core/core/src/test_help/mod.rs +18 -8
- package/sdk-core/core/src/worker/activities/activity_heartbeat_manager.rs +10 -10
- package/sdk-core/core/src/worker/activities/local_activities.rs +13 -13
- package/sdk-core/core/src/worker/activities.rs +6 -12
- package/sdk-core/core/src/worker/client/mocks.rs +1 -0
- package/sdk-core/core/src/worker/client.rs +193 -64
- package/sdk-core/core/src/worker/mod.rs +14 -19
- package/sdk-core/core/src/worker/workflow/driven_workflow.rs +3 -0
- package/sdk-core/core/src/worker/workflow/history_update.rs +5 -5
- package/sdk-core/core/src/worker/workflow/machines/child_workflow_state_machine.rs +133 -85
- package/sdk-core/core/src/worker/workflow/machines/mod.rs +3 -2
- package/sdk-core/core/src/worker/workflow/machines/workflow_machines.rs +160 -105
- package/sdk-core/core/src/worker/workflow/managed_run.rs +2 -1
- package/sdk-core/core/src/worker/workflow/mod.rs +62 -58
- package/sdk-core/core/src/worker/workflow/run_cache.rs +5 -3
- package/sdk-core/core/src/worker/workflow/workflow_stream.rs +7 -5
- package/sdk-core/core-api/Cargo.toml +3 -3
- package/sdk-core/core-api/src/errors.rs +3 -11
- package/sdk-core/core-api/src/worker.rs +7 -0
- package/sdk-core/protos/api_upstream/.buildkite/Dockerfile +1 -1
- package/sdk-core/protos/api_upstream/.github/CODEOWNERS +1 -1
- package/sdk-core/protos/api_upstream/.github/PULL_REQUEST_TEMPLATE.md +2 -6
- package/sdk-core/protos/api_upstream/.github/workflows/trigger-api-go-update.yml +29 -0
- package/sdk-core/protos/api_upstream/Makefile +2 -2
- package/sdk-core/protos/api_upstream/buf.yaml +1 -0
- package/sdk-core/protos/api_upstream/temporal/api/batch/v1/message.proto +86 -0
- package/sdk-core/protos/api_upstream/temporal/api/command/v1/message.proto +26 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/batch_operation.proto +46 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/command_type.proto +7 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/event_type.proto +14 -0
- package/sdk-core/protos/api_upstream/temporal/api/enums/v1/update.proto +51 -0
- package/sdk-core/protos/api_upstream/temporal/api/failure/v1/message.proto +18 -0
- package/sdk-core/protos/api_upstream/temporal/api/history/v1/message.proto +57 -1
- package/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/request_response.proto +1 -3
- package/sdk-core/protos/api_upstream/temporal/api/operatorservice/v1/service.proto +4 -2
- package/sdk-core/protos/api_upstream/temporal/api/replication/v1/message.proto +11 -0
- package/sdk-core/protos/api_upstream/temporal/api/taskqueue/v1/message.proto +23 -0
- package/sdk-core/protos/api_upstream/temporal/api/update/v1/message.proto +46 -0
- package/sdk-core/protos/api_upstream/temporal/api/workflow/v1/message.proto +1 -0
- package/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/request_response.proto +172 -0
- package/sdk-core/protos/api_upstream/temporal/api/workflowservice/v1/service.proto +30 -0
- package/sdk-core/protos/grpc/health/v1/health.proto +63 -0
- package/sdk-core/protos/local/temporal/sdk/core/workflow_commands/workflow_commands.proto +18 -15
- package/sdk-core/protos/testsrv_upstream/Makefile +80 -0
- package/sdk-core/protos/testsrv_upstream/api-linter.yaml +38 -0
- package/sdk-core/protos/testsrv_upstream/buf.yaml +13 -0
- package/sdk-core/protos/testsrv_upstream/dependencies/gogoproto/gogo.proto +141 -0
- package/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/request_response.proto +63 -0
- package/sdk-core/protos/testsrv_upstream/temporal/api/testservice/v1/service.proto +90 -0
- package/sdk-core/sdk/Cargo.toml +2 -2
- package/sdk-core/sdk/src/lib.rs +2 -2
- package/sdk-core/sdk/src/workflow_context/options.rs +36 -8
- package/sdk-core/sdk/src/workflow_context.rs +30 -6
- package/sdk-core/sdk/src/workflow_future.rs +4 -4
- package/sdk-core/sdk-core-protos/Cargo.toml +5 -5
- package/sdk-core/sdk-core-protos/build.rs +9 -1
- package/sdk-core/sdk-core-protos/src/history_builder.rs +6 -1
- package/sdk-core/sdk-core-protos/src/lib.rs +93 -32
- package/sdk-core/test-utils/Cargo.toml +3 -3
- package/sdk-core/test-utils/src/canned_histories.rs +58 -0
- package/sdk-core/test-utils/src/lib.rs +35 -12
- package/sdk-core/tests/integ_tests/ephemeral_server_tests.rs +128 -0
- package/sdk-core/tests/integ_tests/heartbeat_tests.rs +55 -5
- package/sdk-core/tests/integ_tests/polling_tests.rs +2 -1
- package/sdk-core/tests/integ_tests/queries_tests.rs +5 -5
- package/sdk-core/tests/integ_tests/visibility_tests.rs +93 -0
- package/sdk-core/tests/integ_tests/workflow_tests/activities.rs +93 -10
- package/sdk-core/tests/integ_tests/workflow_tests/cancel_wf.rs +1 -1
- package/sdk-core/tests/integ_tests/workflow_tests/local_activities.rs +14 -14
- package/sdk-core/tests/integ_tests/workflow_tests/replay.rs +2 -6
- package/sdk-core/tests/integ_tests/workflow_tests/resets.rs +12 -12
- package/sdk-core/tests/integ_tests/workflow_tests/signals.rs +12 -1
- package/sdk-core/tests/integ_tests/workflow_tests/timers.rs +3 -3
- package/sdk-core/tests/integ_tests/workflow_tests/upsert_search_attrs.rs +8 -2
- package/sdk-core/tests/integ_tests/workflow_tests.rs +19 -4
- package/sdk-core/tests/load_tests.rs +2 -1
- package/sdk-core/tests/main.rs +17 -0
- package/sdk-core/tests/runner.rs +93 -0
- package/src/conversions.rs +157 -94
- package/src/helpers.rs +190 -0
- package/src/lib.rs +10 -912
- package/src/runtime.rs +436 -0
- package/src/testing.rs +67 -0
- package/src/worker.rs +465 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
use crate::{AttachMetricLabels, LONG_POLL_METHOD_NAMES};
|
|
2
2
|
use futures::{future::BoxFuture, FutureExt};
|
|
3
3
|
use opentelemetry::{
|
|
4
|
-
metrics::{Counter,
|
|
4
|
+
metrics::{Counter, Histogram, Meter},
|
|
5
5
|
KeyValue,
|
|
6
6
|
};
|
|
7
7
|
use std::{
|
|
@@ -17,6 +17,7 @@ use tower::Service;
|
|
|
17
17
|
// appropriate k/vs have already been set.
|
|
18
18
|
#[derive(Clone, Debug)]
|
|
19
19
|
pub struct MetricsContext {
|
|
20
|
+
ctx: opentelemetry::Context,
|
|
20
21
|
kvs: Arc<Vec<KeyValue>>,
|
|
21
22
|
poll_is_long: bool,
|
|
22
23
|
|
|
@@ -25,21 +26,22 @@ pub struct MetricsContext {
|
|
|
25
26
|
long_svc_request: Counter<u64>,
|
|
26
27
|
long_svc_request_failed: Counter<u64>,
|
|
27
28
|
|
|
28
|
-
svc_request_latency:
|
|
29
|
-
long_svc_request_latency:
|
|
29
|
+
svc_request_latency: Histogram<u64>,
|
|
30
|
+
long_svc_request_latency: Histogram<u64>,
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
impl MetricsContext {
|
|
33
34
|
pub(crate) fn new(kvs: Vec<KeyValue>, meter: &Meter) -> Self {
|
|
34
35
|
Self {
|
|
36
|
+
ctx: opentelemetry::Context::current(),
|
|
35
37
|
kvs: Arc::new(kvs),
|
|
36
38
|
poll_is_long: false,
|
|
37
39
|
svc_request: meter.u64_counter("request").init(),
|
|
38
40
|
svc_request_failed: meter.u64_counter("request_failure").init(),
|
|
39
41
|
long_svc_request: meter.u64_counter("long_request").init(),
|
|
40
42
|
long_svc_request_failed: meter.u64_counter("long_request_failure").init(),
|
|
41
|
-
svc_request_latency: meter.
|
|
42
|
-
long_svc_request_latency: meter.
|
|
43
|
+
svc_request_latency: meter.u64_histogram("request_latency").init(),
|
|
44
|
+
long_svc_request_latency: meter.u64_histogram("long_request_latency").init(),
|
|
43
45
|
}
|
|
44
46
|
}
|
|
45
47
|
|
|
@@ -62,18 +64,18 @@ impl MetricsContext {
|
|
|
62
64
|
/// A request to the temporal service was made
|
|
63
65
|
pub(crate) fn svc_request(&self) {
|
|
64
66
|
if self.poll_is_long {
|
|
65
|
-
self.long_svc_request.add(1, &self.kvs);
|
|
67
|
+
self.long_svc_request.add(&self.ctx, 1, &self.kvs);
|
|
66
68
|
} else {
|
|
67
|
-
self.svc_request.add(1, &self.kvs);
|
|
69
|
+
self.svc_request.add(&self.ctx, 1, &self.kvs);
|
|
68
70
|
}
|
|
69
71
|
}
|
|
70
72
|
|
|
71
73
|
/// A request to the temporal service failed
|
|
72
74
|
pub(crate) fn svc_request_failed(&self) {
|
|
73
75
|
if self.poll_is_long {
|
|
74
|
-
self.long_svc_request_failed.add(1, &self.kvs);
|
|
76
|
+
self.long_svc_request_failed.add(&self.ctx, 1, &self.kvs);
|
|
75
77
|
} else {
|
|
76
|
-
self.svc_request_failed.add(1, &self.kvs);
|
|
78
|
+
self.svc_request_failed.add(&self.ctx, 1, &self.kvs);
|
|
77
79
|
}
|
|
78
80
|
}
|
|
79
81
|
|
|
@@ -81,10 +83,10 @@ impl MetricsContext {
|
|
|
81
83
|
pub(crate) fn record_svc_req_latency(&self, dur: Duration) {
|
|
82
84
|
if self.poll_is_long {
|
|
83
85
|
self.long_svc_request_latency
|
|
84
|
-
.record(dur.as_millis() as u64, &self.kvs);
|
|
86
|
+
.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
85
87
|
} else {
|
|
86
88
|
self.svc_request_latency
|
|
87
|
-
.record(dur.as_millis() as u64, &self.kvs);
|
|
89
|
+
.record(&self.ctx, dur.as_millis() as u64, &self.kvs);
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
}
|
|
@@ -5,29 +5,44 @@
|
|
|
5
5
|
use crate::{
|
|
6
6
|
metrics::{namespace_kv, task_queue_kv},
|
|
7
7
|
raw::sealed::RawClientLike,
|
|
8
|
+
Client, ConfiguredClient, InterceptedMetricsSvc, RetryClient, TemporalServiceClient,
|
|
8
9
|
LONG_POLL_TIMEOUT,
|
|
9
10
|
};
|
|
10
|
-
use futures::{future::BoxFuture, FutureExt};
|
|
11
|
-
use temporal_sdk_core_protos::
|
|
12
|
-
|
|
11
|
+
use futures::{future::BoxFuture, FutureExt, TryFutureExt};
|
|
12
|
+
use temporal_sdk_core_protos::{
|
|
13
|
+
grpc::health::v1::{health_client::HealthClient, *},
|
|
14
|
+
temporal::api::{
|
|
15
|
+
operatorservice::v1::{operator_service_client::OperatorServiceClient, *},
|
|
16
|
+
taskqueue::v1::TaskQueue,
|
|
17
|
+
testservice::v1::{test_service_client::TestServiceClient, *},
|
|
18
|
+
workflowservice::v1::{workflow_service_client::WorkflowServiceClient, *},
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
use tonic::{
|
|
22
|
+
body::BoxBody, client::GrpcService, metadata::KeyAndValueRef, Request, Response, Status,
|
|
13
23
|
};
|
|
14
|
-
use tonic::{body::BoxBody, client::GrpcService, metadata::KeyAndValueRef};
|
|
15
24
|
|
|
16
25
|
pub(super) mod sealed {
|
|
17
26
|
use super::*;
|
|
18
|
-
use crate::{Client, ConfiguredClient, InterceptedMetricsSvc, RetryClient};
|
|
19
|
-
use futures::TryFutureExt;
|
|
20
|
-
use tonic::{Request, Response, Status};
|
|
21
27
|
|
|
22
28
|
/// Something that has a workflow service client
|
|
23
29
|
#[async_trait::async_trait]
|
|
24
30
|
pub trait RawClientLike: Send {
|
|
25
31
|
type SvcType: Send + Sync + Clone + 'static;
|
|
26
32
|
|
|
27
|
-
/// Return the
|
|
28
|
-
fn
|
|
33
|
+
/// Return the workflow service client instance
|
|
34
|
+
fn workflow_client(&mut self) -> &mut WorkflowServiceClient<Self::SvcType>;
|
|
35
|
+
|
|
36
|
+
/// Return the operator service client instance
|
|
37
|
+
fn operator_client(&mut self) -> &mut OperatorServiceClient<Self::SvcType>;
|
|
38
|
+
|
|
39
|
+
/// Return the test service client instance
|
|
40
|
+
fn test_client(&mut self) -> &mut TestServiceClient<Self::SvcType>;
|
|
29
41
|
|
|
30
|
-
|
|
42
|
+
/// Return the health service client instance
|
|
43
|
+
fn health_client(&mut self) -> &mut HealthClient<Self::SvcType>;
|
|
44
|
+
|
|
45
|
+
async fn call<F, Req, Resp>(
|
|
31
46
|
&mut self,
|
|
32
47
|
_call_name: &'static str,
|
|
33
48
|
mut callfn: F,
|
|
@@ -35,90 +50,139 @@ pub(super) mod sealed {
|
|
|
35
50
|
) -> Result<Response<Resp>, Status>
|
|
36
51
|
where
|
|
37
52
|
Req: Clone + Unpin + Send + Sync + 'static,
|
|
38
|
-
F: FnMut(
|
|
39
|
-
&mut WorkflowServiceClient<Self::SvcType>,
|
|
40
|
-
Request<Req>,
|
|
41
|
-
) -> BoxFuture<'static, Result<Response<Resp>, Status>>,
|
|
53
|
+
F: FnMut(&mut Self, Request<Req>) -> BoxFuture<'static, Result<Response<Resp>, Status>>,
|
|
42
54
|
F: Send + Sync + Unpin + 'static,
|
|
43
55
|
{
|
|
44
|
-
callfn(self
|
|
56
|
+
callfn(self, req).await
|
|
45
57
|
}
|
|
46
58
|
}
|
|
59
|
+
}
|
|
47
60
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
61
|
+
// Here we implement retry on anything that is already RawClientLike
|
|
62
|
+
#[async_trait::async_trait]
|
|
63
|
+
impl<RC, T> RawClientLike for RetryClient<RC>
|
|
64
|
+
where
|
|
65
|
+
RC: RawClientLike<SvcType = T> + 'static,
|
|
66
|
+
T: Send + Sync + Clone + 'static,
|
|
67
|
+
{
|
|
68
|
+
type SvcType = T;
|
|
56
69
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
70
|
+
fn workflow_client(&mut self) -> &mut WorkflowServiceClient<Self::SvcType> {
|
|
71
|
+
self.get_client_mut().workflow_client()
|
|
72
|
+
}
|
|
60
73
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
call_name: &'static str,
|
|
64
|
-
mut callfn: F,
|
|
65
|
-
req: Request<Req>,
|
|
66
|
-
) -> Result<Response<Resp>, Status>
|
|
67
|
-
where
|
|
68
|
-
Req: Clone + Unpin + Send + Sync + 'static,
|
|
69
|
-
F: FnMut(
|
|
70
|
-
&mut WorkflowServiceClient<Self::SvcType>,
|
|
71
|
-
Request<Req>,
|
|
72
|
-
) -> BoxFuture<'static, Result<Response<Resp>, Status>>,
|
|
73
|
-
F: Send + Sync + Unpin + 'static,
|
|
74
|
-
{
|
|
75
|
-
let rtc = self.get_retry_config(call_name);
|
|
76
|
-
let req = req_cloner(&req);
|
|
77
|
-
let fact = || {
|
|
78
|
-
let req_clone = req_cloner(&req);
|
|
79
|
-
callfn(self.client(), req_clone)
|
|
80
|
-
};
|
|
81
|
-
let res = Self::make_future_retry(rtc, fact, call_name);
|
|
82
|
-
res.map_err(|(e, _attempt)| e).map_ok(|x| x.0).await
|
|
83
|
-
}
|
|
74
|
+
fn operator_client(&mut self) -> &mut OperatorServiceClient<Self::SvcType> {
|
|
75
|
+
self.get_client_mut().operator_client()
|
|
84
76
|
}
|
|
85
77
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
{
|
|
90
|
-
type SvcType = T;
|
|
78
|
+
fn test_client(&mut self) -> &mut TestServiceClient<Self::SvcType> {
|
|
79
|
+
self.get_client_mut().test_client()
|
|
80
|
+
}
|
|
91
81
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
}
|
|
82
|
+
fn health_client(&mut self) -> &mut HealthClient<Self::SvcType> {
|
|
83
|
+
self.get_client_mut().health_client()
|
|
95
84
|
}
|
|
96
85
|
|
|
97
|
-
|
|
86
|
+
async fn call<F, Req, Resp>(
|
|
87
|
+
&mut self,
|
|
88
|
+
call_name: &'static str,
|
|
89
|
+
mut callfn: F,
|
|
90
|
+
req: Request<Req>,
|
|
91
|
+
) -> Result<Response<Resp>, Status>
|
|
98
92
|
where
|
|
99
|
-
|
|
93
|
+
Req: Clone + Unpin + Send + Sync + 'static,
|
|
94
|
+
F: FnMut(&mut Self, Request<Req>) -> BoxFuture<'static, Result<Response<Resp>, Status>>,
|
|
95
|
+
F: Send + Sync + Unpin + 'static,
|
|
100
96
|
{
|
|
101
|
-
|
|
97
|
+
let rtc = self.get_retry_config(call_name);
|
|
98
|
+
let fact = || {
|
|
99
|
+
let req_clone = req_cloner(&req);
|
|
100
|
+
callfn(self, req_clone)
|
|
101
|
+
};
|
|
102
|
+
let res = Self::make_future_retry(rtc, fact, call_name);
|
|
103
|
+
res.map_err(|(e, _attempt)| e).map_ok(|x| x.0).await
|
|
104
|
+
}
|
|
105
|
+
}
|
|
102
106
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
107
|
+
impl<T> RawClientLike for TemporalServiceClient<T>
|
|
108
|
+
where
|
|
109
|
+
T: Send + Sync + Clone + 'static,
|
|
110
|
+
T: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
111
|
+
T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
112
|
+
T::Error: Into<tonic::codegen::StdError>,
|
|
113
|
+
<T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
114
|
+
{
|
|
115
|
+
type SvcType = T;
|
|
116
|
+
|
|
117
|
+
fn workflow_client(&mut self) -> &mut WorkflowServiceClient<Self::SvcType> {
|
|
118
|
+
self.workflow_svc_mut()
|
|
106
119
|
}
|
|
107
120
|
|
|
108
|
-
|
|
109
|
-
|
|
121
|
+
fn operator_client(&mut self) -> &mut OperatorServiceClient<Self::SvcType> {
|
|
122
|
+
self.operator_svc_mut()
|
|
123
|
+
}
|
|
110
124
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
125
|
+
fn test_client(&mut self) -> &mut TestServiceClient<Self::SvcType> {
|
|
126
|
+
self.test_svc_mut()
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
fn health_client(&mut self) -> &mut HealthClient<Self::SvcType> {
|
|
130
|
+
self.health_svc_mut()
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
impl<T> RawClientLike for ConfiguredClient<TemporalServiceClient<T>>
|
|
135
|
+
where
|
|
136
|
+
T: Send + Sync + Clone + 'static,
|
|
137
|
+
T: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
138
|
+
T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
139
|
+
T::Error: Into<tonic::codegen::StdError>,
|
|
140
|
+
<T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
141
|
+
{
|
|
142
|
+
type SvcType = T;
|
|
143
|
+
|
|
144
|
+
fn workflow_client(&mut self) -> &mut WorkflowServiceClient<Self::SvcType> {
|
|
145
|
+
self.client.workflow_client()
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
fn operator_client(&mut self) -> &mut OperatorServiceClient<Self::SvcType> {
|
|
149
|
+
self.client.operator_client()
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
fn test_client(&mut self) -> &mut TestServiceClient<Self::SvcType> {
|
|
153
|
+
self.client.test_client()
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
fn health_client(&mut self) -> &mut HealthClient<Self::SvcType> {
|
|
157
|
+
self.client.health_client()
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
impl RawClientLike for Client {
|
|
162
|
+
type SvcType = InterceptedMetricsSvc;
|
|
163
|
+
|
|
164
|
+
fn workflow_client(&mut self) -> &mut WorkflowServiceClient<Self::SvcType> {
|
|
165
|
+
self.inner.workflow_client()
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
fn operator_client(&mut self) -> &mut OperatorServiceClient<Self::SvcType> {
|
|
169
|
+
self.inner.operator_client()
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
fn test_client(&mut self) -> &mut TestServiceClient<Self::SvcType> {
|
|
173
|
+
self.inner.test_client()
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
fn health_client(&mut self) -> &mut HealthClient<Self::SvcType> {
|
|
177
|
+
self.inner.health_client()
|
|
114
178
|
}
|
|
115
179
|
}
|
|
116
180
|
|
|
117
181
|
/// Helper for cloning a tonic request as long as the inner message may be cloned.
|
|
118
182
|
/// We drop extensions, so, lang bridges can't pass those in :shrug:
|
|
119
|
-
fn req_cloner<T: Clone>(cloneme: &
|
|
183
|
+
fn req_cloner<T: Clone>(cloneme: &Request<T>) -> Request<T> {
|
|
120
184
|
let msg = cloneme.get_ref().clone();
|
|
121
|
-
let mut new_req =
|
|
185
|
+
let mut new_req = Request::new(msg);
|
|
122
186
|
let new_met = new_req.metadata_mut();
|
|
123
187
|
for kv in cloneme.metadata().iter() {
|
|
124
188
|
match kv {
|
|
@@ -146,10 +210,14 @@ impl AttachMetricLabels {
|
|
|
146
210
|
}
|
|
147
211
|
pub fn task_q(&mut self, tq: Option<TaskQueue>) -> &mut Self {
|
|
148
212
|
if let Some(tq) = tq {
|
|
149
|
-
self.
|
|
213
|
+
self.task_q_str(tq.name);
|
|
150
214
|
}
|
|
151
215
|
self
|
|
152
216
|
}
|
|
217
|
+
pub fn task_q_str(&mut self, tq: impl Into<String>) -> &mut Self {
|
|
218
|
+
self.labels.push(task_queue_kv(tq.into()));
|
|
219
|
+
self
|
|
220
|
+
}
|
|
153
221
|
}
|
|
154
222
|
|
|
155
223
|
// Blanket impl the trait for all raw-client-like things. Since the trait default-implements
|
|
@@ -158,7 +226,37 @@ impl<RC, T> WorkflowService for RC
|
|
|
158
226
|
where
|
|
159
227
|
RC: RawClientLike<SvcType = T>,
|
|
160
228
|
T: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
161
|
-
T::ResponseBody: tonic::codegen::Body + Send + 'static,
|
|
229
|
+
T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
230
|
+
T::Error: Into<tonic::codegen::StdError>,
|
|
231
|
+
T::Future: Send,
|
|
232
|
+
<T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
233
|
+
{
|
|
234
|
+
}
|
|
235
|
+
impl<RC, T> OperatorService for RC
|
|
236
|
+
where
|
|
237
|
+
RC: RawClientLike<SvcType = T>,
|
|
238
|
+
T: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
239
|
+
T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
240
|
+
T::Error: Into<tonic::codegen::StdError>,
|
|
241
|
+
T::Future: Send,
|
|
242
|
+
<T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
243
|
+
{
|
|
244
|
+
}
|
|
245
|
+
impl<RC, T> TestService for RC
|
|
246
|
+
where
|
|
247
|
+
RC: RawClientLike<SvcType = T>,
|
|
248
|
+
T: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
249
|
+
T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
250
|
+
T::Error: Into<tonic::codegen::StdError>,
|
|
251
|
+
T::Future: Send,
|
|
252
|
+
<T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
253
|
+
{
|
|
254
|
+
}
|
|
255
|
+
impl<RC, T> HealthService for RC
|
|
256
|
+
where
|
|
257
|
+
RC: RawClientLike<SvcType = T>,
|
|
258
|
+
T: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
259
|
+
T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
162
260
|
T::Error: Into<tonic::codegen::StdError>,
|
|
163
261
|
T::Future: Send,
|
|
164
262
|
<T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
@@ -167,34 +265,35 @@ where
|
|
|
167
265
|
|
|
168
266
|
/// Helps re-declare gRPC client methods
|
|
169
267
|
macro_rules! proxy {
|
|
170
|
-
($method:ident, $req:
|
|
171
|
-
#[doc = concat!("See [
|
|
268
|
+
($client_type:tt, $client_meth:ident, $method:ident, $req:ty, $resp:ty $(, $closure:expr)?) => {
|
|
269
|
+
#[doc = concat!("See [", stringify!($client_type), "::", stringify!($method), "]")]
|
|
172
270
|
fn $method(
|
|
173
271
|
&mut self,
|
|
174
|
-
request: impl tonic::IntoRequest
|
|
175
|
-
) -> BoxFuture<Result<tonic::Response
|
|
272
|
+
request: impl tonic::IntoRequest<$req>,
|
|
273
|
+
) -> BoxFuture<Result<tonic::Response<$resp>, tonic::Status>> {
|
|
176
274
|
#[allow(unused_mut)]
|
|
177
|
-
let fact = |c: &mut
|
|
275
|
+
let fact = |c: &mut Self, mut req: tonic::Request<$req>| {
|
|
178
276
|
$( type_closure_arg(&mut req, $closure); )*
|
|
179
|
-
let mut c = c.clone();
|
|
277
|
+
let mut c = c.$client_meth().clone();
|
|
180
278
|
async move { c.$method(req).await }.boxed()
|
|
181
279
|
};
|
|
182
|
-
self.
|
|
280
|
+
self.call(stringify!($method), fact, request.into_request())
|
|
183
281
|
}
|
|
184
282
|
};
|
|
185
283
|
}
|
|
186
284
|
macro_rules! proxier {
|
|
187
|
-
( $
|
|
285
|
+
( $trait_name:ident; $impl_list_name:ident; $client_type:tt; $client_meth:ident;
|
|
286
|
+
$(($method:ident, $req:ty, $resp:ty $(, $closure:expr)? );)* ) => {
|
|
188
287
|
#[cfg(test)]
|
|
189
|
-
const
|
|
190
|
-
/// Trait version of the generated
|
|
191
|
-
///
|
|
192
|
-
pub trait
|
|
288
|
+
const $impl_list_name: &'static [&'static str] = &[$(stringify!($method)),*];
|
|
289
|
+
/// Trait version of the generated client with modifications to attach appropriate metric
|
|
290
|
+
/// labels or whatever else to requests
|
|
291
|
+
pub trait $trait_name: RawClientLike
|
|
193
292
|
where
|
|
194
293
|
// Yo this is wild
|
|
195
294
|
<Self as RawClientLike>::SvcType: GrpcService<BoxBody> + Send + Clone + 'static,
|
|
196
295
|
<<Self as RawClientLike>::SvcType as GrpcService<BoxBody>>::ResponseBody:
|
|
197
|
-
tonic::codegen::Body + Send + 'static,
|
|
296
|
+
tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
|
|
198
297
|
<<Self as RawClientLike>::SvcType as GrpcService<BoxBody>>::Error:
|
|
199
298
|
Into<tonic::codegen::StdError>,
|
|
200
299
|
<<Self as RawClientLike>::SvcType as GrpcService<BoxBody>>::Future: Send,
|
|
@@ -202,17 +301,19 @@ macro_rules! proxier {
|
|
|
202
301
|
as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
|
|
203
302
|
{
|
|
204
303
|
$(
|
|
205
|
-
proxy!($method, $req, $resp $(,$closure)*);
|
|
304
|
+
proxy!($client_type, $client_meth, $method, $req, $resp $(,$closure)*);
|
|
206
305
|
)*
|
|
207
306
|
}
|
|
208
307
|
};
|
|
209
308
|
}
|
|
309
|
+
|
|
210
310
|
// Nice little trick to avoid the callsite asking to type the closure parameter
|
|
211
311
|
fn type_closure_arg<T, R>(arg: T, f: impl FnOnce(T) -> R) -> R {
|
|
212
312
|
f(arg)
|
|
213
313
|
}
|
|
214
314
|
|
|
215
315
|
proxier! {
|
|
316
|
+
WorkflowService; ALL_IMPLEMENTED_WORKFLOW_SERVICE_RPCS; WorkflowServiceClient; workflow_client;
|
|
216
317
|
(
|
|
217
318
|
register_namespace,
|
|
218
319
|
RegisterNamespaceRequest,
|
|
@@ -629,14 +730,121 @@ proxier! {
|
|
|
629
730
|
r.extensions_mut().insert(labels);
|
|
630
731
|
}
|
|
631
732
|
);
|
|
733
|
+
(
|
|
734
|
+
update_worker_build_id_ordering,
|
|
735
|
+
UpdateWorkerBuildIdOrderingRequest,
|
|
736
|
+
UpdateWorkerBuildIdOrderingResponse,
|
|
737
|
+
|r| {
|
|
738
|
+
let mut labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
739
|
+
labels.task_q_str(r.get_ref().task_queue.clone());
|
|
740
|
+
r.extensions_mut().insert(labels);
|
|
741
|
+
}
|
|
742
|
+
);
|
|
743
|
+
(
|
|
744
|
+
get_worker_build_id_ordering,
|
|
745
|
+
GetWorkerBuildIdOrderingRequest,
|
|
746
|
+
GetWorkerBuildIdOrderingResponse,
|
|
747
|
+
|r| {
|
|
748
|
+
let mut labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
749
|
+
labels.task_q_str(r.get_ref().task_queue.clone());
|
|
750
|
+
r.extensions_mut().insert(labels);
|
|
751
|
+
}
|
|
752
|
+
);
|
|
753
|
+
(
|
|
754
|
+
update_workflow,
|
|
755
|
+
UpdateWorkflowRequest,
|
|
756
|
+
UpdateWorkflowResponse,
|
|
757
|
+
|r| {
|
|
758
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
759
|
+
r.extensions_mut().insert(labels);
|
|
760
|
+
}
|
|
761
|
+
);
|
|
762
|
+
(
|
|
763
|
+
start_batch_operation,
|
|
764
|
+
StartBatchOperationRequest,
|
|
765
|
+
StartBatchOperationResponse,
|
|
766
|
+
|r| {
|
|
767
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
768
|
+
r.extensions_mut().insert(labels);
|
|
769
|
+
}
|
|
770
|
+
);
|
|
771
|
+
(
|
|
772
|
+
stop_batch_operation,
|
|
773
|
+
StopBatchOperationRequest,
|
|
774
|
+
StopBatchOperationResponse,
|
|
775
|
+
|r| {
|
|
776
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
777
|
+
r.extensions_mut().insert(labels);
|
|
778
|
+
}
|
|
779
|
+
);
|
|
780
|
+
(
|
|
781
|
+
describe_batch_operation,
|
|
782
|
+
DescribeBatchOperationRequest,
|
|
783
|
+
DescribeBatchOperationResponse,
|
|
784
|
+
|r| {
|
|
785
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
786
|
+
r.extensions_mut().insert(labels);
|
|
787
|
+
}
|
|
788
|
+
);
|
|
789
|
+
(
|
|
790
|
+
list_batch_operations,
|
|
791
|
+
ListBatchOperationsRequest,
|
|
792
|
+
ListBatchOperationsResponse,
|
|
793
|
+
|r| {
|
|
794
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
795
|
+
r.extensions_mut().insert(labels);
|
|
796
|
+
}
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
proxier! {
|
|
801
|
+
OperatorService; ALL_IMPLEMENTED_OPERATOR_SERVICE_RPCS; OperatorServiceClient; operator_client;
|
|
802
|
+
(add_search_attributes, AddSearchAttributesRequest, AddSearchAttributesResponse);
|
|
803
|
+
(remove_search_attributes, RemoveSearchAttributesRequest, RemoveSearchAttributesResponse);
|
|
804
|
+
(list_search_attributes, ListSearchAttributesRequest, ListSearchAttributesResponse);
|
|
805
|
+
(delete_namespace, DeleteNamespaceRequest, DeleteNamespaceResponse,
|
|
806
|
+
|r| {
|
|
807
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
808
|
+
r.extensions_mut().insert(labels);
|
|
809
|
+
}
|
|
810
|
+
);
|
|
811
|
+
(delete_workflow_execution, DeleteWorkflowExecutionRequest, DeleteWorkflowExecutionResponse,
|
|
812
|
+
|r| {
|
|
813
|
+
let labels = AttachMetricLabels::namespace(r.get_ref().namespace.clone());
|
|
814
|
+
r.extensions_mut().insert(labels);
|
|
815
|
+
}
|
|
816
|
+
);
|
|
817
|
+
(add_or_update_remote_cluster, AddOrUpdateRemoteClusterRequest, AddOrUpdateRemoteClusterResponse);
|
|
818
|
+
(remove_remote_cluster, RemoveRemoteClusterRequest, RemoveRemoteClusterResponse);
|
|
819
|
+
(describe_cluster, DescribeClusterRequest, DescribeClusterResponse);
|
|
820
|
+
(list_clusters, ListClustersRequest, ListClustersResponse);
|
|
821
|
+
(list_cluster_members, ListClusterMembersRequest, ListClusterMembersResponse);
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
proxier! {
|
|
825
|
+
TestService; ALL_IMPLEMENTED_TEST_SERVICE_RPCS; TestServiceClient; test_client;
|
|
826
|
+
(lock_time_skipping, LockTimeSkippingRequest, LockTimeSkippingResponse);
|
|
827
|
+
(unlock_time_skipping, UnlockTimeSkippingRequest, UnlockTimeSkippingResponse);
|
|
828
|
+
(sleep, SleepRequest, SleepResponse);
|
|
829
|
+
(sleep_until, SleepUntilRequest, SleepResponse);
|
|
830
|
+
(unlock_time_skipping_with_sleep, SleepRequest, SleepResponse);
|
|
831
|
+
(get_current_time, (), GetCurrentTimeResponse);
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
proxier! {
|
|
835
|
+
HealthService; ALL_IMPLEMENTED_HEALTH_SERVICE_RPCS; HealthClient; health_client;
|
|
836
|
+
(check, HealthCheckRequest, HealthCheckResponse);
|
|
837
|
+
(watch, HealthCheckRequest, tonic::codec::Streaming<HealthCheckResponse>);
|
|
632
838
|
}
|
|
633
839
|
|
|
634
840
|
#[cfg(test)]
|
|
635
841
|
mod tests {
|
|
636
842
|
use super::*;
|
|
637
|
-
use crate::{ClientOptionsBuilder, RetryClient
|
|
843
|
+
use crate::{ClientOptionsBuilder, RetryClient};
|
|
638
844
|
use std::collections::HashSet;
|
|
639
|
-
use temporal_sdk_core_protos::temporal::api::
|
|
845
|
+
use temporal_sdk_core_protos::temporal::api::{
|
|
846
|
+
operatorservice::v1::DeleteNamespaceRequest, workflowservice::v1::ListNamespacesRequest,
|
|
847
|
+
};
|
|
640
848
|
|
|
641
849
|
// Just to help make sure some stuff compiles. Not run.
|
|
642
850
|
#[allow(dead_code)]
|
|
@@ -645,42 +853,79 @@ mod tests {
|
|
|
645
853
|
let raw_client = opts.connect_no_namespace(None, None).await.unwrap();
|
|
646
854
|
let mut retry_client = RetryClient::new(raw_client, opts.retry_config);
|
|
647
855
|
|
|
648
|
-
let
|
|
649
|
-
let fact = |c: &mut
|
|
650
|
-
let mut c = c.clone();
|
|
856
|
+
let list_ns_req = ListNamespacesRequest::default();
|
|
857
|
+
let fact = |c: &mut RetryClient<_>, req| {
|
|
858
|
+
let mut c = c.workflow_client().clone();
|
|
651
859
|
async move { c.list_namespaces(req).await }.boxed()
|
|
652
860
|
};
|
|
653
861
|
retry_client
|
|
654
|
-
.
|
|
862
|
+
.call("whatever", fact, Request::new(list_ns_req.clone()))
|
|
863
|
+
.await
|
|
864
|
+
.unwrap();
|
|
865
|
+
|
|
866
|
+
// Operator svc method
|
|
867
|
+
let del_ns_req = DeleteNamespaceRequest::default();
|
|
868
|
+
let fact = |c: &mut RetryClient<_>, req| {
|
|
869
|
+
let mut c = c.operator_client().clone();
|
|
870
|
+
async move { c.delete_namespace(req).await }.boxed()
|
|
871
|
+
};
|
|
872
|
+
retry_client
|
|
873
|
+
.call("whatever", fact, Request::new(del_ns_req.clone()))
|
|
874
|
+
.await
|
|
875
|
+
.unwrap();
|
|
876
|
+
|
|
877
|
+
// Verify calling through traits works
|
|
878
|
+
retry_client.list_namespaces(list_ns_req).await.unwrap();
|
|
879
|
+
retry_client.delete_namespace(del_ns_req).await.unwrap();
|
|
880
|
+
retry_client.get_current_time(()).await.unwrap();
|
|
881
|
+
retry_client
|
|
882
|
+
.check(HealthCheckRequest::default())
|
|
655
883
|
.await
|
|
656
884
|
.unwrap();
|
|
657
885
|
}
|
|
658
886
|
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
// This is less work than trying to hook into the codegen process
|
|
662
|
-
let proto_def =
|
|
663
|
-
include_str!("../../protos/api_upstream/temporal/api/workflowservice/v1/service.proto");
|
|
664
|
-
let methods: Vec<_> = proto_def
|
|
887
|
+
fn verify_methods(proto_def_str: &str, impl_list: &[&str]) {
|
|
888
|
+
let methods: Vec<_> = proto_def_str
|
|
665
889
|
.lines()
|
|
666
890
|
.map(|l| l.trim())
|
|
667
891
|
.filter(|l| l.starts_with("rpc"))
|
|
668
892
|
.map(|l| {
|
|
669
893
|
let stripped = l.strip_prefix("rpc ").unwrap();
|
|
670
|
-
(
|
|
894
|
+
(stripped[..stripped.find('(').unwrap()]).trim()
|
|
671
895
|
})
|
|
672
896
|
.collect();
|
|
673
|
-
let no_underscores: HashSet<_> =
|
|
674
|
-
.iter()
|
|
675
|
-
.map(|x| x.replace('_', ""))
|
|
676
|
-
.collect();
|
|
897
|
+
let no_underscores: HashSet<_> = impl_list.iter().map(|x| x.replace('_', "")).collect();
|
|
677
898
|
for method in methods {
|
|
678
899
|
if !no_underscores.contains(&method.to_lowercase()) {
|
|
679
|
-
panic!(
|
|
680
|
-
"WorkflowService RPC method {} is not implemented by raw client",
|
|
681
|
-
method
|
|
682
|
-
)
|
|
900
|
+
panic!("RPC method {} is not implemented by raw client", method)
|
|
683
901
|
}
|
|
684
902
|
}
|
|
685
903
|
}
|
|
904
|
+
#[test]
|
|
905
|
+
fn verify_all_workflow_service_methods_implemented() {
|
|
906
|
+
// This is less work than trying to hook into the codegen process
|
|
907
|
+
let proto_def =
|
|
908
|
+
include_str!("../../protos/api_upstream/temporal/api/workflowservice/v1/service.proto");
|
|
909
|
+
verify_methods(proto_def, ALL_IMPLEMENTED_WORKFLOW_SERVICE_RPCS);
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
#[test]
|
|
913
|
+
fn verify_all_operator_service_methods_implemented() {
|
|
914
|
+
let proto_def =
|
|
915
|
+
include_str!("../../protos/api_upstream/temporal/api/operatorservice/v1/service.proto");
|
|
916
|
+
verify_methods(proto_def, ALL_IMPLEMENTED_OPERATOR_SERVICE_RPCS);
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
#[test]
|
|
920
|
+
fn verify_all_test_service_methods_implemented() {
|
|
921
|
+
let proto_def =
|
|
922
|
+
include_str!("../../protos/testsrv_upstream/temporal/api/testservice/v1/service.proto");
|
|
923
|
+
verify_methods(proto_def, ALL_IMPLEMENTED_TEST_SERVICE_RPCS);
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
#[test]
|
|
927
|
+
fn verify_all_health_service_methods_implemented() {
|
|
928
|
+
let proto_def = include_str!("../../protos/grpc/health/v1/health.proto");
|
|
929
|
+
verify_methods(proto_def, ALL_IMPLEMENTED_HEALTH_SERVICE_RPCS);
|
|
930
|
+
}
|
|
686
931
|
}
|