@proompteng/temporal-bun-sdk 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -10
- package/dist/src/bin/replay-command.d.ts.map +1 -1
- package/dist/src/bin/replay-command.js +6 -2
- package/dist/src/bin/replay-command.js.map +1 -1
- package/dist/src/bin/temporal-bun.d.ts +1 -1
- package/dist/src/bin/temporal-bun.d.ts.map +1 -1
- package/dist/src/bin/temporal-bun.js +74 -0
- package/dist/src/bin/temporal-bun.js.map +1 -1
- package/dist/src/client/layer.d.ts +2 -2
- package/dist/src/client/layer.d.ts.map +1 -1
- package/dist/src/client/retries.d.ts.map +1 -1
- package/dist/src/client/retries.js +27 -3
- package/dist/src/client/retries.js.map +1 -1
- package/dist/src/client/serialization.d.ts +34 -2
- package/dist/src/client/serialization.d.ts.map +1 -1
- package/dist/src/client/serialization.js +78 -5
- package/dist/src/client/serialization.js.map +1 -1
- package/dist/src/client/types.d.ts +26 -0
- package/dist/src/client/types.d.ts.map +1 -1
- package/dist/src/client.d.ts +22 -6
- package/dist/src/client.d.ts.map +1 -1
- package/dist/src/client.js +488 -39
- package/dist/src/client.js.map +1 -1
- package/dist/src/common/payloads/codecs.d.ts +38 -0
- package/dist/src/common/payloads/codecs.d.ts.map +1 -0
- package/dist/src/common/payloads/codecs.js +174 -0
- package/dist/src/common/payloads/codecs.js.map +1 -0
- package/dist/src/common/payloads/converter.d.ts +52 -3
- package/dist/src/common/payloads/converter.d.ts.map +1 -1
- package/dist/src/common/payloads/converter.js +340 -2
- package/dist/src/common/payloads/converter.js.map +1 -1
- package/dist/src/common/payloads/failure.d.ts +5 -6
- package/dist/src/common/payloads/failure.d.ts.map +1 -1
- package/dist/src/common/payloads/failure.js +3 -52
- package/dist/src/common/payloads/failure.js.map +1 -1
- package/dist/src/common/payloads/index.d.ts +1 -0
- package/dist/src/common/payloads/index.d.ts.map +1 -1
- package/dist/src/common/payloads/index.js +1 -0
- package/dist/src/common/payloads/index.js.map +1 -1
- package/dist/src/config.d.ts +9 -0
- package/dist/src/config.d.ts.map +1 -1
- package/dist/src/config.js +62 -1
- package/dist/src/config.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/interceptors/client.d.ts +28 -0
- package/dist/src/interceptors/client.d.ts.map +1 -0
- package/dist/src/interceptors/client.js +169 -0
- package/dist/src/interceptors/client.js.map +1 -0
- package/dist/src/interceptors/types.d.ts +33 -0
- package/dist/src/interceptors/types.d.ts.map +1 -0
- package/dist/src/interceptors/types.js +44 -0
- package/dist/src/interceptors/types.js.map +1 -0
- package/dist/src/interceptors/worker.d.ts +22 -0
- package/dist/src/interceptors/worker.d.ts.map +1 -0
- package/dist/src/interceptors/worker.js +156 -0
- package/dist/src/interceptors/worker.js.map +1 -0
- package/dist/src/runtime/cli-layer.d.ts +4 -3
- package/dist/src/runtime/cli-layer.d.ts.map +1 -1
- package/dist/src/runtime/cli-layer.js +5 -2
- package/dist/src/runtime/cli-layer.js.map +1 -1
- package/dist/src/runtime/effect-layers.d.ts +9 -0
- package/dist/src/runtime/effect-layers.d.ts.map +1 -1
- package/dist/src/runtime/effect-layers.js +15 -0
- package/dist/src/runtime/effect-layers.js.map +1 -1
- package/dist/src/worker/concurrency.d.ts +8 -0
- package/dist/src/worker/concurrency.d.ts.map +1 -1
- package/dist/src/worker/concurrency.js +5 -9
- package/dist/src/worker/concurrency.js.map +1 -1
- package/dist/src/worker/runtime.d.ts +5 -0
- package/dist/src/worker/runtime.d.ts.map +1 -1
- package/dist/src/worker/runtime.js +509 -40
- package/dist/src/worker/runtime.js.map +1 -1
- package/dist/src/worker/sticky-cache.d.ts +5 -0
- package/dist/src/worker/sticky-cache.d.ts.map +1 -1
- package/dist/src/worker/sticky-cache.js +26 -0
- package/dist/src/worker/sticky-cache.js.map +1 -1
- package/dist/src/worker/update-protocol.d.ts +33 -0
- package/dist/src/worker/update-protocol.d.ts.map +1 -0
- package/dist/src/worker/update-protocol.js +243 -0
- package/dist/src/worker/update-protocol.js.map +1 -0
- package/dist/src/worker.js +1 -0
- package/dist/src/worker.js.map +1 -1
- package/dist/src/workflow/commands.d.ts +38 -2
- package/dist/src/workflow/commands.d.ts.map +1 -1
- package/dist/src/workflow/commands.js +153 -1
- package/dist/src/workflow/commands.js.map +1 -1
- package/dist/src/workflow/context.d.ts +111 -3
- package/dist/src/workflow/context.d.ts.map +1 -1
- package/dist/src/workflow/context.js +526 -6
- package/dist/src/workflow/context.js.map +1 -1
- package/dist/src/workflow/definition.d.ts +29 -3
- package/dist/src/workflow/definition.d.ts.map +1 -1
- package/dist/src/workflow/definition.js +30 -4
- package/dist/src/workflow/definition.js.map +1 -1
- package/dist/src/workflow/determinism.d.ts +64 -0
- package/dist/src/workflow/determinism.d.ts.map +1 -1
- package/dist/src/workflow/determinism.js +120 -3
- package/dist/src/workflow/determinism.js.map +1 -1
- package/dist/src/workflow/errors.d.ts +6 -0
- package/dist/src/workflow/errors.d.ts.map +1 -1
- package/dist/src/workflow/errors.js +12 -0
- package/dist/src/workflow/errors.js.map +1 -1
- package/dist/src/workflow/executor.d.ts +56 -0
- package/dist/src/workflow/executor.d.ts.map +1 -1
- package/dist/src/workflow/executor.js +300 -9
- package/dist/src/workflow/executor.js.map +1 -1
- package/dist/src/workflow/inbound.d.ts +84 -0
- package/dist/src/workflow/inbound.d.ts.map +1 -0
- package/dist/src/workflow/inbound.js +65 -0
- package/dist/src/workflow/inbound.js.map +1 -0
- package/dist/src/workflow/index.d.ts +1 -0
- package/dist/src/workflow/index.d.ts.map +1 -1
- package/dist/src/workflow/index.js +1 -0
- package/dist/src/workflow/index.js.map +1 -1
- package/dist/src/workflow/replay.d.ts +24 -2
- package/dist/src/workflow/replay.d.ts.map +1 -1
- package/dist/src/workflow/replay.js +679 -15
- package/dist/src/workflow/replay.js.map +1 -1
- package/dist/src/workflows/index.d.ts +1 -1
- package/dist/src/workflows/index.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -6,10 +6,11 @@ import { Cause, Effect, Exit, Fiber } from 'effect';
|
|
|
6
6
|
import { makeActivityLifecycle, } from '../activities/lifecycle';
|
|
7
7
|
import { buildTransportOptions, normalizeTemporalAddress } from '../client';
|
|
8
8
|
import { durationFromMillis, durationToMillis } from '../common/duration';
|
|
9
|
-
import { createDefaultDataConverter, decodePayloadsToValues, encodeValuesToPayloads, } from '../common/payloads/converter';
|
|
9
|
+
import { buildCodecsFromConfig, createDefaultDataConverter, decodePayloadsToValues, encodeValuesToPayloads, } from '../common/payloads/converter';
|
|
10
10
|
import { encodeErrorToFailure, encodeFailurePayloads, failureToError } from '../common/payloads/failure';
|
|
11
11
|
import { sleep } from '../common/sleep';
|
|
12
12
|
import { loadTemporalConfig } from '../config';
|
|
13
|
+
import { makeDefaultWorkerInterceptors, runWorkerInterceptors, } from '../interceptors/worker';
|
|
13
14
|
import { createObservabilityServices } from '../observability';
|
|
14
15
|
import { CommandSchema, RecordMarkerCommandAttributesSchema, } from '../proto/temporal/api/command/v1/message_pb';
|
|
15
16
|
import { PayloadsSchema, WorkflowExecutionSchema, } from '../proto/temporal/api/common/v1/message_pb';
|
|
@@ -18,9 +19,10 @@ import { CommandType } from '../proto/temporal/api/enums/v1/command_type_pb';
|
|
|
18
19
|
import { WorkerVersioningMode } from '../proto/temporal/api/enums/v1/deployment_pb';
|
|
19
20
|
import { EventType } from '../proto/temporal/api/enums/v1/event_type_pb';
|
|
20
21
|
import { WorkflowTaskFailedCause } from '../proto/temporal/api/enums/v1/failed_cause_pb';
|
|
22
|
+
import { QueryResultType } from '../proto/temporal/api/enums/v1/query_pb';
|
|
21
23
|
import { HistoryEventFilterType, TimeoutType, VersioningBehavior } from '../proto/temporal/api/enums/v1/workflow_pb';
|
|
22
24
|
import { StickyExecutionAttributesSchema, TaskQueueSchema, } from '../proto/temporal/api/taskqueue/v1/message_pb';
|
|
23
|
-
import { GetWorkflowExecutionHistoryRequestSchema, PollActivityTaskQueueRequestSchema, PollWorkflowTaskQueueRequestSchema, RespondActivityTaskCanceledRequestSchema, RespondActivityTaskCompletedRequestSchema, RespondActivityTaskFailedRequestSchema, RespondWorkflowTaskCompletedRequestSchema, RespondWorkflowTaskFailedRequestSchema, } from '../proto/temporal/api/workflowservice/v1/request_response_pb';
|
|
25
|
+
import { GetWorkflowExecutionHistoryRequestSchema, PollActivityTaskQueueRequestSchema, PollWorkflowTaskQueueRequestSchema, RespondActivityTaskCanceledRequestSchema, RespondActivityTaskCompletedRequestSchema, RespondActivityTaskFailedRequestSchema, RespondQueryTaskCompletedRequestSchema, RespondWorkflowTaskCompletedRequestSchema, RespondWorkflowTaskFailedRequestSchema, } from '../proto/temporal/api/workflowservice/v1/request_response_pb';
|
|
24
26
|
import { WorkflowService } from '../proto/temporal/api/workflowservice/v1/service_pb';
|
|
25
27
|
import { WorkflowNondeterminismError } from '../workflow/errors';
|
|
26
28
|
import { WorkflowExecutor } from '../workflow/executor';
|
|
@@ -30,6 +32,24 @@ import { runWithActivityContext } from './activity-context';
|
|
|
30
32
|
import { checkWorkerVersioningCapability, registerWorkerBuildIdCompatibility } from './build-id';
|
|
31
33
|
import { makeWorkerScheduler, } from './concurrency';
|
|
32
34
|
import { makeStickyCache, } from './sticky-cache';
|
|
35
|
+
import { buildUpdateProtocolMessages, collectWorkflowUpdates } from './update-protocol';
|
|
36
|
+
const mergeUpdateInvocations = (historyInvocations, messageInvocations) => {
|
|
37
|
+
const merged = [];
|
|
38
|
+
const seen = new Set();
|
|
39
|
+
for (const invocation of historyInvocations ?? []) {
|
|
40
|
+
if (!seen.has(invocation.updateId)) {
|
|
41
|
+
merged.push(invocation);
|
|
42
|
+
seen.add(invocation.updateId);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
for (const invocation of messageInvocations ?? []) {
|
|
46
|
+
if (!seen.has(invocation.updateId)) {
|
|
47
|
+
merged.push(invocation);
|
|
48
|
+
seen.add(invocation.updateId);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return merged;
|
|
52
|
+
};
|
|
33
53
|
const POLL_TIMEOUT_MS = 60_000;
|
|
34
54
|
const RESPOND_TIMEOUT_MS = 15_000;
|
|
35
55
|
const HISTORY_FETCH_TIMEOUT_MS = 60_000;
|
|
@@ -47,7 +67,6 @@ const COMPLETION_COMMAND_TYPES = new Set([
|
|
|
47
67
|
export class WorkerRuntime {
|
|
48
68
|
static async create(options = {}) {
|
|
49
69
|
const config = options.config ?? (await loadTemporalConfig());
|
|
50
|
-
const dataConverter = options.dataConverter ?? createDefaultDataConverter();
|
|
51
70
|
const namespace = options.namespace ?? config.namespace;
|
|
52
71
|
if (!namespace) {
|
|
53
72
|
throw new Error('Temporal namespace must be provided');
|
|
@@ -61,12 +80,6 @@ export class WorkerRuntime {
|
|
|
61
80
|
if (workflows.length === 0) {
|
|
62
81
|
throw new Error('No workflow definitions were registered; provide workflows or workflowsPath');
|
|
63
82
|
}
|
|
64
|
-
const registry = new WorkflowRegistry();
|
|
65
|
-
registry.registerMany(workflows);
|
|
66
|
-
const executor = new WorkflowExecutor({
|
|
67
|
-
registry,
|
|
68
|
-
dataConverter,
|
|
69
|
-
});
|
|
70
83
|
const activities = options.activities ?? {};
|
|
71
84
|
const observability = await Effect.runPromise(createObservabilityServices({
|
|
72
85
|
logLevel: config.logLevel,
|
|
@@ -78,6 +91,18 @@ export class WorkerRuntime {
|
|
|
78
91
|
metricsExporter: options.metricsExporter,
|
|
79
92
|
}));
|
|
80
93
|
const { logger, metricsRegistry, metricsExporter } = observability;
|
|
94
|
+
const dataConverter = options.dataConverter ??
|
|
95
|
+
createDefaultDataConverter({
|
|
96
|
+
payloadCodecs: buildCodecsFromConfig(config.payloadCodecs),
|
|
97
|
+
logger,
|
|
98
|
+
metricsRegistry,
|
|
99
|
+
});
|
|
100
|
+
const registry = new WorkflowRegistry();
|
|
101
|
+
registry.registerMany(workflows);
|
|
102
|
+
const executor = new WorkflowExecutor({
|
|
103
|
+
registry,
|
|
104
|
+
dataConverter,
|
|
105
|
+
});
|
|
81
106
|
const runtimeMetrics = await WorkerRuntime.#initMetrics(metricsRegistry);
|
|
82
107
|
let workflowService;
|
|
83
108
|
if (options.workflowService) {
|
|
@@ -98,6 +123,12 @@ export class WorkerRuntime {
|
|
|
98
123
|
activityConcurrency,
|
|
99
124
|
hooks: options.schedulerHooks,
|
|
100
125
|
logger,
|
|
126
|
+
metrics: {
|
|
127
|
+
workflowTaskStarted: runtimeMetrics.workflowTaskStarted,
|
|
128
|
+
workflowTaskCompleted: runtimeMetrics.workflowTaskCompleted,
|
|
129
|
+
activityTaskStarted: runtimeMetrics.activityTaskStarted,
|
|
130
|
+
activityTaskCompleted: runtimeMetrics.activityTaskCompleted,
|
|
131
|
+
},
|
|
101
132
|
}));
|
|
102
133
|
const stickyCacheCandidate = options.stickyCache;
|
|
103
134
|
const hasStickyCacheInstance = WorkerRuntime.#isStickyCacheInstance(stickyCacheCandidate);
|
|
@@ -156,6 +187,22 @@ export class WorkerRuntime {
|
|
|
156
187
|
}));
|
|
157
188
|
}
|
|
158
189
|
}
|
|
190
|
+
const tracingEnabled = options.tracingEnabled ?? config.tracingInterceptorsEnabled ?? false;
|
|
191
|
+
const workerInterceptorBuilder = options.interceptorBuilder ?? {
|
|
192
|
+
build: (input) => makeDefaultWorkerInterceptors(input),
|
|
193
|
+
};
|
|
194
|
+
const defaultWorkerInterceptors = await Effect.runPromise(workerInterceptorBuilder.build({
|
|
195
|
+
namespace,
|
|
196
|
+
taskQueue,
|
|
197
|
+
identity,
|
|
198
|
+
buildId,
|
|
199
|
+
logger,
|
|
200
|
+
metricsRegistry,
|
|
201
|
+
metricsExporter,
|
|
202
|
+
dataConverter,
|
|
203
|
+
tracingEnabled,
|
|
204
|
+
}));
|
|
205
|
+
const workerInterceptors = [...defaultWorkerInterceptors, ...(options.interceptors ?? [])];
|
|
159
206
|
const activityLifecycle = await Effect.runPromise(makeActivityLifecycle({
|
|
160
207
|
heartbeatIntervalMs: config.activityHeartbeatIntervalMs,
|
|
161
208
|
heartbeatRpcTimeoutMs: config.activityHeartbeatRpcTimeoutMs,
|
|
@@ -208,6 +255,7 @@ export class WorkerRuntime {
|
|
|
208
255
|
versioningBehavior,
|
|
209
256
|
stickySchedulingEnabled,
|
|
210
257
|
workflowPollerCount,
|
|
258
|
+
interceptors: workerInterceptors,
|
|
211
259
|
});
|
|
212
260
|
}
|
|
213
261
|
#config;
|
|
@@ -223,6 +271,7 @@ export class WorkerRuntime {
|
|
|
223
271
|
#namespace;
|
|
224
272
|
#taskQueue;
|
|
225
273
|
#identity;
|
|
274
|
+
#interceptors;
|
|
226
275
|
#activityLifecycle;
|
|
227
276
|
#scheduler;
|
|
228
277
|
#stickyCache;
|
|
@@ -250,6 +299,7 @@ export class WorkerRuntime {
|
|
|
250
299
|
this.#namespace = params.namespace;
|
|
251
300
|
this.#taskQueue = params.taskQueue;
|
|
252
301
|
this.#identity = params.identity;
|
|
302
|
+
this.#interceptors = params.interceptors;
|
|
253
303
|
this.#activityLifecycle = params.activityLifecycle;
|
|
254
304
|
this.#scheduler = params.scheduler;
|
|
255
305
|
this.#stickyCache = params.stickyCache;
|
|
@@ -267,7 +317,11 @@ export class WorkerRuntime {
|
|
|
267
317
|
if (!state) {
|
|
268
318
|
return false;
|
|
269
319
|
}
|
|
270
|
-
return state.commandHistory.length > 0 ||
|
|
320
|
+
return (state.commandHistory.length > 0 ||
|
|
321
|
+
state.randomValues.length > 0 ||
|
|
322
|
+
state.timeValues.length > 0 ||
|
|
323
|
+
(state.signals?.length ?? 0) > 0 ||
|
|
324
|
+
(state.queries?.length ?? 0) > 0);
|
|
271
325
|
}
|
|
272
326
|
#resolvePreviousHistoryEventId(response) {
|
|
273
327
|
const previous = response.previousStartedEventId;
|
|
@@ -320,6 +374,14 @@ export class WorkerRuntime {
|
|
|
320
374
|
heartbeatFailures: await makeCounter('temporal_worker_heartbeat_failures_total', 'Activity heartbeat failures'),
|
|
321
375
|
activityFailures: await makeCounter('temporal_worker_activity_failures_total', 'Activity failures delivered to Temporal'),
|
|
322
376
|
workflowFailures: await makeCounter('temporal_worker_workflow_failures_total', 'Workflow failure responses sent to Temporal'),
|
|
377
|
+
workflowTaskStarted: await makeCounter('temporal_worker_workflow_tasks_started_total', 'Workflow tasks dispatched to the scheduler'),
|
|
378
|
+
workflowTaskCompleted: await makeCounter('temporal_worker_workflow_tasks_completed_total', 'Workflow tasks completed by the scheduler'),
|
|
379
|
+
activityTaskStarted: await makeCounter('temporal_worker_activity_tasks_started_total', 'Activity tasks dispatched to the scheduler'),
|
|
380
|
+
activityTaskCompleted: await makeCounter('temporal_worker_activity_tasks_completed_total', 'Activity tasks completed by the scheduler'),
|
|
381
|
+
queryTaskStarted: await makeCounter('temporal_worker_query_started_total', 'Query-only workflow tasks dispatched to the scheduler'),
|
|
382
|
+
queryTaskCompleted: await makeCounter('temporal_worker_query_completed_total', 'Query-only workflow tasks completed successfully'),
|
|
383
|
+
queryTaskFailed: await makeCounter('temporal_worker_query_failed_total', 'Query-only workflow tasks responded with failure'),
|
|
384
|
+
queryTaskLatency: await makeHistogram('temporal_worker_query_latency_ms', 'End-to-end workflow query latency (ms)'),
|
|
323
385
|
};
|
|
324
386
|
}
|
|
325
387
|
async run() {
|
|
@@ -481,17 +543,38 @@ export class WorkerRuntime {
|
|
|
481
543
|
#isRpcAbortError(error) {
|
|
482
544
|
return isAbortError(error) || (error instanceof ConnectError && error.code === Code.Canceled);
|
|
483
545
|
}
|
|
546
|
+
#isBenignPollTimeout(error) {
|
|
547
|
+
if (error instanceof ConnectError) {
|
|
548
|
+
return error.code === Code.DeadlineExceeded || error.code === Code.Canceled;
|
|
549
|
+
}
|
|
550
|
+
if (error instanceof Error) {
|
|
551
|
+
const msg = error.message.toLowerCase();
|
|
552
|
+
return msg.includes('deadline') && msg.includes('exceeded');
|
|
553
|
+
}
|
|
554
|
+
if (typeof error === 'string') {
|
|
555
|
+
const msg = error.toLowerCase();
|
|
556
|
+
return msg.includes('deadline') && msg.includes('exceeded');
|
|
557
|
+
}
|
|
558
|
+
return false;
|
|
559
|
+
}
|
|
484
560
|
async #enqueueActivityTask(response) {
|
|
485
561
|
const taskToken = response.taskToken ?? new Uint8Array();
|
|
486
562
|
const envelope = {
|
|
487
563
|
taskToken,
|
|
488
|
-
handler: () => this.#
|
|
564
|
+
handler: () => this.#runActivityTask(response),
|
|
489
565
|
args: [],
|
|
490
566
|
};
|
|
491
567
|
await Effect.runPromise(this.#scheduler.enqueueActivity(envelope));
|
|
492
568
|
}
|
|
493
569
|
#handleWorkflowPollerError(queueName, error) {
|
|
494
570
|
return Effect.promise(async () => {
|
|
571
|
+
if (this.#isBenignPollTimeout(error)) {
|
|
572
|
+
this.#log('debug', 'workflow poll timeout (no tasks)', {
|
|
573
|
+
queueName,
|
|
574
|
+
namespace: this.#namespace,
|
|
575
|
+
});
|
|
576
|
+
return;
|
|
577
|
+
}
|
|
495
578
|
this.#incrementCounter(this.#metrics.workflowPollErrors);
|
|
496
579
|
this.#log('warn', 'workflow polling failed', {
|
|
497
580
|
queueName,
|
|
@@ -503,6 +586,13 @@ export class WorkerRuntime {
|
|
|
503
586
|
}
|
|
504
587
|
#handleActivityPollerError(error) {
|
|
505
588
|
return Effect.promise(async () => {
|
|
589
|
+
if (this.#isBenignPollTimeout(error)) {
|
|
590
|
+
this.#log('debug', 'activity poll timeout (no tasks)', {
|
|
591
|
+
namespace: this.#namespace,
|
|
592
|
+
taskQueue: this.#taskQueue,
|
|
593
|
+
});
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
506
596
|
this.#incrementCounter(this.#metrics.activityPollErrors);
|
|
507
597
|
this.#log('warn', 'activity polling failed', {
|
|
508
598
|
namespace: this.#namespace,
|
|
@@ -514,20 +604,83 @@ export class WorkerRuntime {
|
|
|
514
604
|
}
|
|
515
605
|
async #handleWorkflowTask(response, nondeterminismRetry = 0) {
|
|
516
606
|
const execution = this.#resolveWorkflowExecution(response);
|
|
607
|
+
const queryCount = response.queries && typeof response.queries === 'object' && !Array.isArray(response.queries)
|
|
608
|
+
? Object.keys(response.queries).length
|
|
609
|
+
: Array.isArray(response.queries)
|
|
610
|
+
? response.queries.length
|
|
611
|
+
: 0;
|
|
612
|
+
const hasQueryRequests = Boolean(response.query) || queryCount > 0;
|
|
613
|
+
const hasUpdateMessages = (response.messages?.length ?? 0) > 0;
|
|
614
|
+
const kind = hasUpdateMessages
|
|
615
|
+
? 'worker.updateTask'
|
|
616
|
+
: hasQueryRequests
|
|
617
|
+
? 'worker.queryTask'
|
|
618
|
+
: 'worker.workflowTask';
|
|
619
|
+
const context = {
|
|
620
|
+
kind,
|
|
621
|
+
namespace: this.#namespace,
|
|
622
|
+
taskQueue: this.#taskQueue,
|
|
623
|
+
identity: this.#identity,
|
|
624
|
+
buildId: this.#deploymentOptions.buildId,
|
|
625
|
+
workflowId: execution.workflowId,
|
|
626
|
+
runId: execution.runId,
|
|
627
|
+
attempt: Number(response.attempt ?? 1),
|
|
628
|
+
metadata: { nondeterminismRetry },
|
|
629
|
+
};
|
|
630
|
+
const effect = runWorkerInterceptors(this.#interceptors, context, () => Effect.tryPromise(() => this.#processWorkflowTask(response, nondeterminismRetry, execution)));
|
|
631
|
+
await Effect.runPromise(effect);
|
|
632
|
+
}
|
|
633
|
+
async #processWorkflowTask(response, nondeterminismRetry = 0, executionOverride) {
|
|
634
|
+
const execution = executionOverride ?? this.#resolveWorkflowExecution(response);
|
|
517
635
|
const workflowTaskAttempt = Number(response.attempt ?? 1);
|
|
636
|
+
const isLegacyQueryTask = Boolean(response.query);
|
|
637
|
+
const queryStartTime = isLegacyQueryTask ? Date.now() : null;
|
|
638
|
+
if (isLegacyQueryTask) {
|
|
639
|
+
this.#incrementCounter(this.#metrics.queryTaskStarted);
|
|
640
|
+
}
|
|
518
641
|
const historyEvents = await this.#collectWorkflowHistory(execution, response);
|
|
519
642
|
const workflowType = this.#resolveWorkflowType(response, historyEvents);
|
|
520
643
|
const args = await this.#decodeWorkflowArgs(historyEvents);
|
|
521
644
|
const workflowInfo = this.#buildWorkflowInfo(workflowType, execution);
|
|
522
|
-
const
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
645
|
+
const collectedUpdates = await collectWorkflowUpdates({
|
|
646
|
+
messages: response.messages ?? [],
|
|
647
|
+
dataConverter: this.#dataConverter,
|
|
648
|
+
log: (level, message, fields) => this.#log(level, message, fields),
|
|
649
|
+
});
|
|
526
650
|
const baseLogFields = this.#workflowLogFields(execution, workflowType, {
|
|
527
651
|
workflowTaskAttempt,
|
|
528
652
|
stickyScheduling: this.#stickySchedulingEnabled,
|
|
529
653
|
nondeterminismRetry,
|
|
530
654
|
});
|
|
655
|
+
const signalDeliveries = await this.#extractSignalDeliveries(historyEvents);
|
|
656
|
+
if (signalDeliveries.length > 0) {
|
|
657
|
+
this.#log('debug', 'workflow signal deliveries buffered', {
|
|
658
|
+
...baseLogFields,
|
|
659
|
+
signalCount: signalDeliveries.length,
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
const queryRequests = await this.#extractWorkflowQueryRequests(response);
|
|
663
|
+
const hasQueryRequests = queryRequests.length > 0;
|
|
664
|
+
const hasMultiQueries = queryRequests.some((request) => request.source === 'multi');
|
|
665
|
+
const hasLegacyQueries = queryRequests.some((request) => request.source === 'legacy');
|
|
666
|
+
if (hasQueryRequests) {
|
|
667
|
+
this.#log('debug', 'workflow queries pending evaluation', {
|
|
668
|
+
...baseLogFields,
|
|
669
|
+
queryCount: queryRequests.length,
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
this.#log('info', 'debug: workflow task metadata', {
|
|
673
|
+
...baseLogFields,
|
|
674
|
+
taskTokenBytes: response.taskToken?.length ?? 0,
|
|
675
|
+
queryCount: queryRequests.length,
|
|
676
|
+
historyEventCount: response.history?.events?.length ?? 0,
|
|
677
|
+
});
|
|
678
|
+
const stickyKey = this.#buildStickyKey(execution.workflowId, execution.runId);
|
|
679
|
+
const stickyEntry = stickyKey ? await this.#getStickyEntry(stickyKey) : undefined;
|
|
680
|
+
const historyReplay = await this.#ingestDeterminismState(workflowInfo, historyEvents, {
|
|
681
|
+
queryRequests,
|
|
682
|
+
});
|
|
683
|
+
const hasHistorySnapshot = this.#isValidDeterminismSnapshot(historyReplay?.determinismState);
|
|
531
684
|
let previousState;
|
|
532
685
|
if (stickyEntry && this.#isValidDeterminismSnapshot(stickyEntry.determinismState)) {
|
|
533
686
|
const historyBaselineEventId = this.#resolvePreviousHistoryEventId(response) ?? historyReplay?.lastEventId ?? null;
|
|
@@ -574,7 +727,10 @@ export class WorkerRuntime {
|
|
|
574
727
|
}
|
|
575
728
|
const expectedDeterminismState = previousState;
|
|
576
729
|
try {
|
|
577
|
-
const activityResults = await this.#extractActivityResolutions(historyEvents);
|
|
730
|
+
const { results: activityResults, scheduledEventIds: activityScheduleEventIds } = await this.#extractActivityResolutions(historyEvents);
|
|
731
|
+
const timerResults = await this.#extractTimerResolutions(historyEvents);
|
|
732
|
+
const replayUpdates = historyReplay?.updates ?? [];
|
|
733
|
+
const mergedUpdates = mergeUpdateInvocations(replayUpdates, collectedUpdates.invocations);
|
|
578
734
|
const output = await this.#executor.execute({
|
|
579
735
|
workflowType,
|
|
580
736
|
workflowId: execution.workflowId,
|
|
@@ -584,10 +740,70 @@ export class WorkerRuntime {
|
|
|
584
740
|
arguments: args,
|
|
585
741
|
determinismState: previousState,
|
|
586
742
|
activityResults,
|
|
743
|
+
activityScheduleEventIds,
|
|
744
|
+
signalDeliveries,
|
|
745
|
+
timerResults,
|
|
746
|
+
queryRequests,
|
|
747
|
+
updates: mergedUpdates,
|
|
748
|
+
mode: isLegacyQueryTask ? 'query' : 'workflow',
|
|
749
|
+
});
|
|
750
|
+
this.#log('debug', 'workflow query evaluation summary', {
|
|
751
|
+
...baseLogFields,
|
|
752
|
+
queryResultCount: output.queryResults.length,
|
|
587
753
|
});
|
|
754
|
+
this.#log('debug', 'workflow query results raw', {
|
|
755
|
+
...baseLogFields,
|
|
756
|
+
queryResults: output.queryResults.map((entry) => ({
|
|
757
|
+
name: entry.request.name,
|
|
758
|
+
source: entry.request.source,
|
|
759
|
+
id: entry.request.id ?? null,
|
|
760
|
+
})),
|
|
761
|
+
});
|
|
762
|
+
const multiQueryResults = {};
|
|
763
|
+
let legacyQueryResult;
|
|
764
|
+
for (const entry of output.queryResults) {
|
|
765
|
+
this.#log('debug', 'workflow query evaluation completed', {
|
|
766
|
+
...baseLogFields,
|
|
767
|
+
querySource: entry.request.source,
|
|
768
|
+
queryName: entry.request.name,
|
|
769
|
+
queryId: entry.request.id ?? null,
|
|
770
|
+
});
|
|
771
|
+
if (entry.request.source === 'multi' && entry.request.id) {
|
|
772
|
+
multiQueryResults[entry.request.id] = entry.result;
|
|
773
|
+
}
|
|
774
|
+
else if (entry.request.source === 'legacy') {
|
|
775
|
+
legacyQueryResult = entry;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
if (isLegacyQueryTask) {
|
|
779
|
+
const target = legacyQueryResult ?? output.queryResults.find((entry) => entry.request.source === 'legacy');
|
|
780
|
+
if (!target) {
|
|
781
|
+
throw new Error('Legacy query result missing from workflow execution');
|
|
782
|
+
}
|
|
783
|
+
await this.#respondLegacyQueryTask(response, target);
|
|
784
|
+
if (queryStartTime !== null) {
|
|
785
|
+
this.#observeHistogram(this.#metrics.queryTaskLatency, Date.now() - queryStartTime);
|
|
786
|
+
}
|
|
787
|
+
this.#incrementCounter(this.#metrics.queryTaskCompleted);
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
588
790
|
const cacheBaselineEventId = this.#resolveCurrentStartedEventId(response) ?? historyReplay?.lastEventId ?? null;
|
|
589
791
|
const shouldRecordMarker = output.completion === 'pending';
|
|
590
792
|
let commandsForResponse = output.commands;
|
|
793
|
+
const dispatchesForNewMessages = (output.updateDispatches ?? []).filter((dispatch) => {
|
|
794
|
+
if (dispatch.type === 'acceptance' || dispatch.type === 'rejection') {
|
|
795
|
+
return collectedUpdates.requestsByUpdateId.has(dispatch.updateId);
|
|
796
|
+
}
|
|
797
|
+
// Allow completion messages to be emitted even if the request metadata was seen on a prior task.
|
|
798
|
+
return true;
|
|
799
|
+
});
|
|
800
|
+
const updateProtocolMessages = await buildUpdateProtocolMessages({
|
|
801
|
+
dispatches: dispatchesForNewMessages,
|
|
802
|
+
collected: collectedUpdates,
|
|
803
|
+
dataConverter: this.#dataConverter,
|
|
804
|
+
defaultIdentity: this.#identity,
|
|
805
|
+
log: (level, message, fields) => this.#log(level, message, fields),
|
|
806
|
+
});
|
|
591
807
|
if (stickyKey) {
|
|
592
808
|
if (output.completion === 'pending') {
|
|
593
809
|
await this.#upsertStickyEntry(stickyKey, output.determinismState, cacheBaselineEventId, workflowType);
|
|
@@ -598,6 +814,7 @@ export class WorkerRuntime {
|
|
|
598
814
|
}
|
|
599
815
|
else {
|
|
600
816
|
await this.#removeStickyEntry(stickyKey);
|
|
817
|
+
await this.#removeStickyEntriesForWorkflow(stickyKey.workflowId);
|
|
601
818
|
this.#log('debug', 'sticky cache entry cleared (workflow completed)', baseLogFields);
|
|
602
819
|
}
|
|
603
820
|
}
|
|
@@ -614,36 +831,65 @@ export class WorkerRuntime {
|
|
|
614
831
|
const markerCommand = this.#buildDeterminismMarkerCommand(markerDetails);
|
|
615
832
|
commandsForResponse = this.#injectDeterminismMarker(commandsForResponse, markerCommand);
|
|
616
833
|
}
|
|
617
|
-
const
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
this.#
|
|
632
|
-
|
|
834
|
+
const shouldRespondWorkflowTask = hasMultiQueries || !hasLegacyQueries;
|
|
835
|
+
if (shouldRespondWorkflowTask) {
|
|
836
|
+
const completion = create(RespondWorkflowTaskCompletedRequestSchema, {
|
|
837
|
+
taskToken: response.taskToken,
|
|
838
|
+
commands: commandsForResponse,
|
|
839
|
+
identity: this.#identity,
|
|
840
|
+
namespace: this.#namespace,
|
|
841
|
+
deploymentOptions: this.#deploymentOptions,
|
|
842
|
+
queryResults: multiQueryResults,
|
|
843
|
+
...(this.#stickySchedulingEnabled && !hasLegacyQueries ? { stickyAttributes: this.#stickyAttributes } : {}),
|
|
844
|
+
...(this.#versioningBehavior !== null ? { versioningBehavior: this.#versioningBehavior } : {}),
|
|
845
|
+
...(updateProtocolMessages.length > 0 ? { messages: updateProtocolMessages } : {}),
|
|
846
|
+
});
|
|
847
|
+
try {
|
|
848
|
+
await this.#workflowService.respondWorkflowTaskCompleted(completion, { timeoutMs: RESPOND_TIMEOUT_MS });
|
|
849
|
+
}
|
|
850
|
+
catch (rpcError) {
|
|
851
|
+
this.#log('error', 'debug: respondWorkflowTaskCompleted failed', {
|
|
852
|
+
...baseLogFields,
|
|
853
|
+
error: rpcError instanceof Error ? rpcError.message : String(rpcError),
|
|
854
|
+
});
|
|
855
|
+
if (this.#isTaskNotFoundError(rpcError)) {
|
|
856
|
+
this.#logWorkflowTaskNotFound('respondWorkflowTaskCompleted', execution);
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
throw rpcError;
|
|
633
860
|
}
|
|
634
|
-
|
|
861
|
+
}
|
|
862
|
+
if (legacyQueryResult) {
|
|
863
|
+
await this.#respondLegacyQueryTask(response, legacyQueryResult);
|
|
635
864
|
}
|
|
636
865
|
}
|
|
637
866
|
catch (error) {
|
|
638
867
|
let stickyEntryCleared = false;
|
|
639
868
|
if (stickyKey) {
|
|
640
869
|
await this.#removeStickyEntry(stickyKey);
|
|
870
|
+
await this.#removeStickyEntriesForWorkflow(stickyKey.workflowId);
|
|
641
871
|
stickyEntryCleared = true;
|
|
642
872
|
}
|
|
643
873
|
if (this.#isTaskNotFoundError(error)) {
|
|
644
874
|
this.#logWorkflowTaskNotFound('respondWorkflowTaskCompleted', execution);
|
|
645
875
|
return;
|
|
646
876
|
}
|
|
877
|
+
if (isLegacyQueryTask) {
|
|
878
|
+
this.#incrementCounter(this.#metrics.queryTaskFailed);
|
|
879
|
+
if (queryStartTime !== null) {
|
|
880
|
+
this.#observeHistogram(this.#metrics.queryTaskLatency, Date.now() - queryStartTime);
|
|
881
|
+
}
|
|
882
|
+
if (error instanceof WorkflowNondeterminismError) {
|
|
883
|
+
const mismatches = await this.#computeNondeterminismMismatches(error, expectedDeterminismState);
|
|
884
|
+
this.#incrementCounter(this.#metrics.nondeterminism);
|
|
885
|
+
this.#log('error', 'workflow query nondeterminism detected', {
|
|
886
|
+
...baseLogFields,
|
|
887
|
+
mismatches,
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
await this.#respondLegacyQueryFailure(response, error);
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
647
893
|
if (error instanceof WorkflowNondeterminismError) {
|
|
648
894
|
const mismatches = await this.#computeNondeterminismMismatches(error, expectedDeterminismState);
|
|
649
895
|
if (stickyKey && stickyEntryCleared) {
|
|
@@ -693,7 +939,7 @@ export class WorkerRuntime {
|
|
|
693
939
|
runId,
|
|
694
940
|
};
|
|
695
941
|
}
|
|
696
|
-
async #ingestDeterminismState(workflowInfo, historyEvents) {
|
|
942
|
+
async #ingestDeterminismState(workflowInfo, historyEvents, options) {
|
|
697
943
|
if (historyEvents.length === 0) {
|
|
698
944
|
return undefined;
|
|
699
945
|
}
|
|
@@ -701,6 +947,7 @@ export class WorkerRuntime {
|
|
|
701
947
|
info: workflowInfo,
|
|
702
948
|
history: historyEvents,
|
|
703
949
|
dataConverter: this.#dataConverter,
|
|
950
|
+
queries: options?.queryRequests,
|
|
704
951
|
}));
|
|
705
952
|
}
|
|
706
953
|
async #collectWorkflowHistory(execution, _response) {
|
|
@@ -721,6 +968,7 @@ export class WorkerRuntime {
|
|
|
721
968
|
async #extractActivityResolutions(events) {
|
|
722
969
|
const resolutions = new Map();
|
|
723
970
|
const scheduledActivityIds = new Map();
|
|
971
|
+
const activityScheduleById = new Map();
|
|
724
972
|
const normalizeEventId = (value) => {
|
|
725
973
|
if (value === undefined || value === null) {
|
|
726
974
|
return undefined;
|
|
@@ -751,6 +999,7 @@ export class WorkerRuntime {
|
|
|
751
999
|
const scheduledKey = normalizeEventId(event.eventId);
|
|
752
1000
|
if (activityId && scheduledKey) {
|
|
753
1001
|
scheduledActivityIds.set(scheduledKey, activityId);
|
|
1002
|
+
activityScheduleById.set(activityId, scheduledKey);
|
|
754
1003
|
}
|
|
755
1004
|
break;
|
|
756
1005
|
}
|
|
@@ -815,7 +1064,152 @@ export class WorkerRuntime {
|
|
|
815
1064
|
break;
|
|
816
1065
|
}
|
|
817
1066
|
}
|
|
818
|
-
return resolutions;
|
|
1067
|
+
return { results: resolutions, scheduledEventIds: activityScheduleById };
|
|
1068
|
+
}
|
|
1069
|
+
async #extractSignalDeliveries(events) {
|
|
1070
|
+
const deliveries = [];
|
|
1071
|
+
const normalizeEventId = (value) => {
|
|
1072
|
+
if (value === undefined || value === null) {
|
|
1073
|
+
return null;
|
|
1074
|
+
}
|
|
1075
|
+
if (typeof value === 'string') {
|
|
1076
|
+
return value;
|
|
1077
|
+
}
|
|
1078
|
+
return value.toString();
|
|
1079
|
+
};
|
|
1080
|
+
for (const event of events) {
|
|
1081
|
+
if (event.eventType !== EventType.WORKFLOW_EXECUTION_SIGNALED) {
|
|
1082
|
+
continue;
|
|
1083
|
+
}
|
|
1084
|
+
if (event.attributes?.case !== 'workflowExecutionSignaledEventAttributes') {
|
|
1085
|
+
continue;
|
|
1086
|
+
}
|
|
1087
|
+
const attrs = event.attributes.value;
|
|
1088
|
+
const args = await decodePayloadsToValues(this.#dataConverter, attrs.input?.payloads ?? []);
|
|
1089
|
+
const workflowTaskCompletedEventId = 'workflowTaskCompletedEventId' in attrs
|
|
1090
|
+
? normalizeEventId(attrs
|
|
1091
|
+
.workflowTaskCompletedEventId)
|
|
1092
|
+
: null;
|
|
1093
|
+
deliveries.push({
|
|
1094
|
+
name: attrs.signalName ?? 'unknown',
|
|
1095
|
+
args,
|
|
1096
|
+
metadata: {
|
|
1097
|
+
eventId: normalizeEventId(event.eventId),
|
|
1098
|
+
workflowTaskCompletedEventId,
|
|
1099
|
+
identity: attrs.identity ?? null,
|
|
1100
|
+
},
|
|
1101
|
+
});
|
|
1102
|
+
}
|
|
1103
|
+
return deliveries;
|
|
1104
|
+
}
|
|
1105
|
+
async #extractTimerResolutions(events) {
|
|
1106
|
+
const fired = new Set();
|
|
1107
|
+
for (const event of events) {
|
|
1108
|
+
if (event.eventType !== EventType.TIMER_FIRED) {
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
1111
|
+
if (event.attributes?.case !== 'timerFiredEventAttributes') {
|
|
1112
|
+
continue;
|
|
1113
|
+
}
|
|
1114
|
+
const attrs = event.attributes.value;
|
|
1115
|
+
if (attrs.timerId) {
|
|
1116
|
+
fired.add(attrs.timerId);
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
return fired;
|
|
1120
|
+
}
|
|
1121
|
+
async #extractWorkflowQueryRequests(response) {
|
|
1122
|
+
const requests = [];
|
|
1123
|
+
const map = response.queries ?? {};
|
|
1124
|
+
this.#log('info', 'debug: workflow query payloads detected', {
|
|
1125
|
+
namespace: this.#namespace,
|
|
1126
|
+
taskQueue: this.#taskQueue,
|
|
1127
|
+
workflowId: response.workflowExecution?.workflowId,
|
|
1128
|
+
runId: response.workflowExecution?.runId,
|
|
1129
|
+
queryMapSize: Object.keys(map).length,
|
|
1130
|
+
hasLegacyQuery: Boolean(response.query),
|
|
1131
|
+
});
|
|
1132
|
+
for (const [id, query] of Object.entries(map)) {
|
|
1133
|
+
const args = await decodePayloadsToValues(this.#dataConverter, query.queryArgs?.payloads ?? []);
|
|
1134
|
+
const header = await this.#decodeQueryHeader(query);
|
|
1135
|
+
requests.push({
|
|
1136
|
+
id,
|
|
1137
|
+
name: query.queryType ?? 'query',
|
|
1138
|
+
args,
|
|
1139
|
+
metadata: header ? { header } : undefined,
|
|
1140
|
+
source: 'multi',
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
if (response.query) {
|
|
1144
|
+
const args = await decodePayloadsToValues(this.#dataConverter, response.query.queryArgs?.payloads ?? []);
|
|
1145
|
+
const header = await this.#decodeQueryHeader(response.query);
|
|
1146
|
+
requests.push({
|
|
1147
|
+
name: response.query.queryType ?? 'query',
|
|
1148
|
+
args,
|
|
1149
|
+
metadata: header ? { header } : undefined,
|
|
1150
|
+
source: 'legacy',
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1153
|
+
return requests;
|
|
1154
|
+
}
|
|
1155
|
+
async #decodeQueryHeader(query) {
|
|
1156
|
+
const fields = query?.header?.fields;
|
|
1157
|
+
if (!fields || Object.keys(fields).length === 0) {
|
|
1158
|
+
return undefined;
|
|
1159
|
+
}
|
|
1160
|
+
const decoded = {};
|
|
1161
|
+
for (const [key, payload] of Object.entries(fields)) {
|
|
1162
|
+
const values = await decodePayloadsToValues(this.#dataConverter, payload ? [payload] : []);
|
|
1163
|
+
decoded[key] = values.length === 0 ? undefined : values.length === 1 ? values[0] : Object.freeze([...values]);
|
|
1164
|
+
}
|
|
1165
|
+
return Object.keys(decoded).length > 0 ? decoded : undefined;
|
|
1166
|
+
}
|
|
1167
|
+
async #respondLegacyQueryTask(response, entry) {
|
|
1168
|
+
const queryResult = entry.result;
|
|
1169
|
+
this.#log('debug', 'responding to legacy workflow query', {
|
|
1170
|
+
namespace: this.#namespace,
|
|
1171
|
+
workflowId: response.workflowExecution?.workflowId,
|
|
1172
|
+
runId: response.workflowExecution?.runId,
|
|
1173
|
+
queryName: entry.request.name,
|
|
1174
|
+
});
|
|
1175
|
+
const request = create(RespondQueryTaskCompletedRequestSchema, {
|
|
1176
|
+
taskToken: response.taskToken ?? new Uint8Array(),
|
|
1177
|
+
completedType: queryResult.resultType ?? QueryResultType.ANSWERED,
|
|
1178
|
+
queryResult: queryResult.answer,
|
|
1179
|
+
errorMessage: queryResult.errorMessage ?? '',
|
|
1180
|
+
namespace: this.#namespace,
|
|
1181
|
+
failure: queryResult.failure,
|
|
1182
|
+
cause: WorkflowTaskFailedCause.UNSPECIFIED,
|
|
1183
|
+
});
|
|
1184
|
+
await this.#workflowService.respondQueryTaskCompleted(request, { timeoutMs: RESPOND_TIMEOUT_MS });
|
|
1185
|
+
}
|
|
1186
|
+
async #respondLegacyQueryFailure(response, cause) {
|
|
1187
|
+
const failure = await encodeErrorToFailure(this.#dataConverter, cause);
|
|
1188
|
+
const message = cause instanceof Error ? cause.message : 'Workflow query failed';
|
|
1189
|
+
const request = create(RespondQueryTaskCompletedRequestSchema, {
|
|
1190
|
+
taskToken: response.taskToken ?? new Uint8Array(),
|
|
1191
|
+
completedType: QueryResultType.FAILED,
|
|
1192
|
+
errorMessage: message,
|
|
1193
|
+
namespace: this.#namespace,
|
|
1194
|
+
failure,
|
|
1195
|
+
cause: WorkflowTaskFailedCause.UNSPECIFIED,
|
|
1196
|
+
});
|
|
1197
|
+
try {
|
|
1198
|
+
await this.#workflowService.respondQueryTaskCompleted(request, { timeoutMs: RESPOND_TIMEOUT_MS });
|
|
1199
|
+
}
|
|
1200
|
+
catch (rpcError) {
|
|
1201
|
+
this.#log('error', 'respondQueryTaskCompleted failed for legacy query', {
|
|
1202
|
+
namespace: this.#namespace,
|
|
1203
|
+
workflowId: response.workflowExecution?.workflowId,
|
|
1204
|
+
runId: response.workflowExecution?.runId,
|
|
1205
|
+
error: rpcError instanceof Error ? rpcError.message : String(rpcError),
|
|
1206
|
+
});
|
|
1207
|
+
if (this.#isTaskNotFoundError(rpcError)) {
|
|
1208
|
+
this.#logWorkflowTaskNotFound('respondQueryTaskCompleted', this.#resolveWorkflowExecution(response));
|
|
1209
|
+
return;
|
|
1210
|
+
}
|
|
1211
|
+
throw rpcError;
|
|
1212
|
+
}
|
|
819
1213
|
}
|
|
820
1214
|
async #fetchWorkflowHistoryPage(execution, nextPageToken) {
|
|
821
1215
|
if (!execution.workflowId || !execution.runId) {
|
|
@@ -931,11 +1325,13 @@ export class WorkerRuntime {
|
|
|
931
1325
|
};
|
|
932
1326
|
}
|
|
933
1327
|
async #computeNondeterminismMismatches(error, expectedState) {
|
|
934
|
-
const baseline =
|
|
935
|
-
commandHistory: [],
|
|
936
|
-
randomValues: [],
|
|
937
|
-
timeValues: [],
|
|
938
|
-
failureMetadata:
|
|
1328
|
+
const baseline = {
|
|
1329
|
+
commandHistory: expectedState?.commandHistory ?? [],
|
|
1330
|
+
randomValues: expectedState?.randomValues ?? [],
|
|
1331
|
+
timeValues: expectedState?.timeValues ?? [],
|
|
1332
|
+
failureMetadata: expectedState?.failureMetadata,
|
|
1333
|
+
signals: expectedState?.signals ?? [],
|
|
1334
|
+
queries: expectedState?.queries ?? [],
|
|
939
1335
|
};
|
|
940
1336
|
const hint = error.details?.hint;
|
|
941
1337
|
if (!hint) {
|
|
@@ -944,6 +1340,8 @@ export class WorkerRuntime {
|
|
|
944
1340
|
const commandIndex = this.#parseIndexFromHint(hint, 'commandIndex');
|
|
945
1341
|
const randomIndex = this.#parseIndexFromHint(hint, 'randomIndex');
|
|
946
1342
|
const timeIndex = this.#parseIndexFromHint(hint, 'timeIndex');
|
|
1343
|
+
const signalIndex = this.#parseIndexFromHint(hint, 'signalIndex');
|
|
1344
|
+
const queryIndex = this.#parseIndexFromHint(hint, 'queryIndex');
|
|
947
1345
|
const mutableActual = {
|
|
948
1346
|
commandHistory: baseline.commandHistory.map((entry) => ({
|
|
949
1347
|
intent: entry.intent,
|
|
@@ -952,6 +1350,8 @@ export class WorkerRuntime {
|
|
|
952
1350
|
randomValues: [...baseline.randomValues],
|
|
953
1351
|
timeValues: [...baseline.timeValues],
|
|
954
1352
|
failureMetadata: baseline.failureMetadata ? { ...baseline.failureMetadata } : undefined,
|
|
1353
|
+
signals: baseline.signals.map((record) => ({ ...record })),
|
|
1354
|
+
queries: baseline.queries.map((record) => ({ ...record })),
|
|
955
1355
|
};
|
|
956
1356
|
let mutated = false;
|
|
957
1357
|
if (commandIndex !== null) {
|
|
@@ -980,6 +1380,36 @@ export class WorkerRuntime {
|
|
|
980
1380
|
mutableActual.timeValues[timeIndex] = Number.NaN;
|
|
981
1381
|
mutated = true;
|
|
982
1382
|
}
|
|
1383
|
+
if (signalIndex !== null) {
|
|
1384
|
+
const receivedSignal = this.#asSignalRecord(error.details?.received);
|
|
1385
|
+
if (receivedSignal) {
|
|
1386
|
+
if (signalIndex < mutableActual.signals.length) {
|
|
1387
|
+
mutableActual.signals = mutableActual.signals.map((record, idx) => idx === signalIndex ? receivedSignal : record);
|
|
1388
|
+
}
|
|
1389
|
+
else {
|
|
1390
|
+
mutableActual.signals = [...mutableActual.signals, receivedSignal];
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
else if (signalIndex < mutableActual.signals.length) {
|
|
1394
|
+
mutableActual.signals = mutableActual.signals.filter((_, idx) => idx !== signalIndex);
|
|
1395
|
+
}
|
|
1396
|
+
mutated = true;
|
|
1397
|
+
}
|
|
1398
|
+
if (queryIndex !== null) {
|
|
1399
|
+
const receivedQuery = this.#asQueryRecord(error.details?.received);
|
|
1400
|
+
if (receivedQuery) {
|
|
1401
|
+
if (queryIndex < mutableActual.queries.length) {
|
|
1402
|
+
mutableActual.queries = mutableActual.queries.map((record, idx) => idx === queryIndex ? receivedQuery : record);
|
|
1403
|
+
}
|
|
1404
|
+
else {
|
|
1405
|
+
mutableActual.queries = [...mutableActual.queries, receivedQuery];
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
else if (queryIndex < mutableActual.queries.length) {
|
|
1409
|
+
mutableActual.queries = mutableActual.queries.filter((_, idx) => idx !== queryIndex);
|
|
1410
|
+
}
|
|
1411
|
+
mutated = true;
|
|
1412
|
+
}
|
|
983
1413
|
if (!mutated) {
|
|
984
1414
|
return [];
|
|
985
1415
|
}
|
|
@@ -988,6 +1418,8 @@ export class WorkerRuntime {
|
|
|
988
1418
|
randomValues: mutableActual.randomValues,
|
|
989
1419
|
timeValues: mutableActual.timeValues,
|
|
990
1420
|
failureMetadata: mutableActual.failureMetadata,
|
|
1421
|
+
signals: mutableActual.signals,
|
|
1422
|
+
queries: mutableActual.queries,
|
|
991
1423
|
};
|
|
992
1424
|
const diff = await Effect.runPromise(diffDeterminismState(baseline, actualState));
|
|
993
1425
|
return diff.mismatches;
|
|
@@ -1030,6 +1462,24 @@ export class WorkerRuntime {
|
|
|
1030
1462
|
array.push(Number.NaN);
|
|
1031
1463
|
}
|
|
1032
1464
|
}
|
|
1465
|
+
#asSignalRecord(value) {
|
|
1466
|
+
if (!value || typeof value !== 'object') {
|
|
1467
|
+
return undefined;
|
|
1468
|
+
}
|
|
1469
|
+
if ('signalName' in value && 'payloadHash' in value) {
|
|
1470
|
+
return value;
|
|
1471
|
+
}
|
|
1472
|
+
return undefined;
|
|
1473
|
+
}
|
|
1474
|
+
#asQueryRecord(value) {
|
|
1475
|
+
if (!value || typeof value !== 'object') {
|
|
1476
|
+
return undefined;
|
|
1477
|
+
}
|
|
1478
|
+
if ('queryName' in value && 'requestHash' in value) {
|
|
1479
|
+
return value;
|
|
1480
|
+
}
|
|
1481
|
+
return undefined;
|
|
1482
|
+
}
|
|
1033
1483
|
static #isStickyCacheInstance(value) {
|
|
1034
1484
|
if (!value || typeof value !== 'object') {
|
|
1035
1485
|
return false;
|
|
@@ -1042,6 +1492,11 @@ export class WorkerRuntime {
|
|
|
1042
1492
|
async #removeStickyEntry(key) {
|
|
1043
1493
|
await Effect.runPromise(this.#stickyCache.remove(key));
|
|
1044
1494
|
}
|
|
1495
|
+
async #removeStickyEntriesForWorkflow(workflowId) {
|
|
1496
|
+
if (!workflowId)
|
|
1497
|
+
return;
|
|
1498
|
+
await Effect.runPromise(this.#stickyCache.removeByWorkflow({ namespace: this.#namespace, workflowId }));
|
|
1499
|
+
}
|
|
1045
1500
|
async #failWorkflowTask(response, execution, error, cause = WorkflowTaskFailedCause.UNSPECIFIED) {
|
|
1046
1501
|
const failure = await encodeErrorToFailure(this.#dataConverter, error);
|
|
1047
1502
|
const encoded = await encodeFailurePayloads(this.#dataConverter, failure);
|
|
@@ -1065,6 +1520,20 @@ export class WorkerRuntime {
|
|
|
1065
1520
|
throw rpcError;
|
|
1066
1521
|
}
|
|
1067
1522
|
}
|
|
1523
|
+
async #runActivityTask(response) {
|
|
1524
|
+
const context = {
|
|
1525
|
+
kind: 'worker.activityTask',
|
|
1526
|
+
namespace: this.#namespace,
|
|
1527
|
+
taskQueue: this.#taskQueue,
|
|
1528
|
+
identity: this.#identity,
|
|
1529
|
+
buildId: this.#deploymentOptions.buildId,
|
|
1530
|
+
workflowId: response.workflowExecution?.workflowId ?? undefined,
|
|
1531
|
+
runId: response.workflowExecution?.runId ?? undefined,
|
|
1532
|
+
attempt: Number(response.attempt ?? 1),
|
|
1533
|
+
};
|
|
1534
|
+
const effect = runWorkerInterceptors(this.#interceptors, context, () => Effect.tryPromise(() => this.#processActivityTask(response)));
|
|
1535
|
+
await Effect.runPromise(effect);
|
|
1536
|
+
}
|
|
1068
1537
|
async #processActivityTask(response) {
|
|
1069
1538
|
const cancelRequested = isActivityCancelRequested(response);
|
|
1070
1539
|
if (cancelRequested) {
|