@volter/twin-temporal 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +104 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +36 -0
- package/dist/src/generated/hpack.json +1 -0
- package/dist/src/generated/temporal-api.json +1 -0
- package/dist/src/index.d.ts +18 -0
- package/dist/src/index.js +70 -0
- package/dist/src/temporal-budget.d.ts +31 -0
- package/dist/src/temporal-budget.js +50 -0
- package/dist/src/temporal-capabilities.d.ts +7 -0
- package/dist/src/temporal-capabilities.js +650 -0
- package/dist/src/temporal-conformance.d.ts +8 -0
- package/dist/src/temporal-conformance.js +54 -0
- package/dist/src/temporal-connector.d.ts +32 -0
- package/dist/src/temporal-connector.js +121 -0
- package/dist/src/temporal-engine.d.ts +274 -0
- package/dist/src/temporal-engine.js +2132 -0
- package/dist/src/temporal-grpc.d.ts +51 -0
- package/dist/src/temporal-grpc.js +124 -0
- package/dist/src/temporal-h2.d.ts +40 -0
- package/dist/src/temporal-h2.js +644 -0
- package/dist/src/temporal-history.d.ts +27 -0
- package/dist/src/temporal-history.js +85 -0
- package/dist/src/temporal-http.d.ts +17 -0
- package/dist/src/temporal-http.js +160 -0
- package/dist/src/temporal-json.d.ts +7 -0
- package/dist/src/temporal-json.js +68 -0
- package/dist/src/temporal-proto.d.ts +17 -0
- package/dist/src/temporal-proto.js +46 -0
- package/dist/src/temporal-server.d.ts +25 -0
- package/dist/src/temporal-server.js +56 -0
- package/dist/src/temporal-twin.d.ts +25 -0
- package/dist/src/temporal-twin.js +100 -0
- package/dist/src/temporal-visibility.d.ts +49 -0
- package/dist/src/temporal-visibility.js +293 -0
- package/package.json +59 -0
- package/src/cli.ts +36 -0
- package/src/generated/hpack.json +1 -0
- package/src/generated/temporal-api.json +1 -0
- package/src/index.ts +81 -0
- package/src/temporal-budget.ts +75 -0
- package/src/temporal-capabilities.ts +679 -0
- package/src/temporal-conformance.ts +52 -0
- package/src/temporal-connector.ts +118 -0
- package/src/temporal-engine.ts +2176 -0
- package/src/temporal-grpc.ts +139 -0
- package/src/temporal-h2.ts +614 -0
- package/src/temporal-history.ts +99 -0
- package/src/temporal-http.ts +146 -0
- package/src/temporal-json.ts +77 -0
- package/src/temporal-proto.ts +53 -0
- package/src/temporal-server.ts +67 -0
- package/src/temporal-twin.ts +102 -0
- package/src/temporal-visibility.ts +245 -0
|
@@ -0,0 +1,2176 @@
|
|
|
1
|
+
// THE ENGINE — Temporal's history service, over the World's state.
|
|
2
|
+
//
|
|
3
|
+
// A run is a `workflow-execution` subject (its WorkflowExecutionInfo, the vendor's own shape, plus
|
|
4
|
+
// `namespace` and the server's `_state` bookkeeping) whose history is the chunk list temporal-history.ts
|
|
5
|
+
// keeps beside the tree. Every RPC that moves a run is one transition here: read the run, decide the
|
|
6
|
+
// events it appends and the tasks it makes or retires, write the chunk and the subject. Transitions
|
|
7
|
+
// are serialized in-process (one server owns a World's Temporal), and a poller or a history reader that
|
|
8
|
+
// has nothing to take waits on the engine's notification, never on a sleep loop.
|
|
9
|
+
//
|
|
10
|
+
// Time is the World's (`worldNow()`): an event's time, a timer's fire time, an activity's deadline and
|
|
11
|
+
// a retry's backoff are all read off the World clock, so a World whose operator freezes the clock
|
|
12
|
+
// freezes every timer with it, and `volter-world clock advance` fires what falls due.
|
|
13
|
+
//
|
|
14
|
+
// The rules below are Temporal's server rules as its documentation and its API protos state them;
|
|
15
|
+
// where the twin is narrower it refuses by name (a command it does not model fails the workflow task
|
|
16
|
+
// with WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED and says which), never answers a fabricated success.
|
|
17
|
+
import { createHash } from 'node:crypto';
|
|
18
|
+
import { worldNow } from '@volter/world-core';
|
|
19
|
+
import { GrpcError, GrpcStatus } from './temporal-grpc.ts';
|
|
20
|
+
import { TemporalStore, type HistoryEvent, type Stored } from './temporal-history.ts';
|
|
21
|
+
import { matchesVisibilityQuery, parseVisibilityQuery, visibilityFields, VisibilityQueryError, type VisibilityRow } from './temporal-visibility.ts';
|
|
22
|
+
|
|
23
|
+
// ---- plain-form helpers ----------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
type Obj = Record<string, unknown>;
|
|
26
|
+
type Duration = { seconds?: number; nanos?: number };
|
|
27
|
+
type Timestamp = { seconds: number; nanos?: number };
|
|
28
|
+
|
|
29
|
+
export const DEFAULT_NAMESPACE = 'default';
|
|
30
|
+
const LONG_POLL_MS = 60_000;
|
|
31
|
+
const HISTORY_LONG_POLL_MS = 20_000;
|
|
32
|
+
const DEFAULT_WFT_TIMEOUT_MS = 10_000;
|
|
33
|
+
const MAX_WFT_TIMEOUT_MS = 120_000;
|
|
34
|
+
const DEFAULT_PAGE_SIZE = 1000;
|
|
35
|
+
|
|
36
|
+
export function durationMs(value: unknown): number {
|
|
37
|
+
const d = value as Duration | undefined;
|
|
38
|
+
if (!d) return 0;
|
|
39
|
+
return Number(d.seconds ?? 0) * 1000 + Math.floor(Number(d.nanos ?? 0) / 1e6);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function toDuration(ms: number): Duration | undefined {
|
|
43
|
+
if (!ms || ms <= 0) return undefined;
|
|
44
|
+
const seconds = Math.floor(ms / 1000);
|
|
45
|
+
const nanos = Math.round((ms - seconds * 1000) * 1e6);
|
|
46
|
+
return nanos ? { seconds, nanos } : { seconds };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function toTimestamp(ms: number): Timestamp {
|
|
50
|
+
const seconds = Math.floor(ms / 1000);
|
|
51
|
+
const nanos = Math.round((ms - seconds * 1000) * 1e6);
|
|
52
|
+
return nanos ? { seconds, nanos } : { seconds };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function timestampMs(value: unknown): number | undefined {
|
|
56
|
+
const t = value as Timestamp | undefined;
|
|
57
|
+
if (!t) return undefined;
|
|
58
|
+
return Number(t.seconds ?? 0) * 1000 + Math.floor(Number(t.nanos ?? 0) / 1e6);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The World's now, in epoch milliseconds. */
|
|
62
|
+
export function nowMs(): number {
|
|
63
|
+
return Date.parse(worldNow());
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const b64 = (text: string): string => Buffer.from(text, 'utf8').toString('base64');
|
|
67
|
+
const unb64 = (text: string | undefined): string => (text ? Buffer.from(text, 'base64').toString('utf8') : '');
|
|
68
|
+
|
|
69
|
+
/** A payload's value, when its encoding is one this server can read (json/plain, binary/null). */
|
|
70
|
+
export function payloadValue(payload: unknown): unknown {
|
|
71
|
+
const p = payload as { metadata?: Record<string, string>; data?: string } | undefined;
|
|
72
|
+
if (!p) return undefined;
|
|
73
|
+
const encoding = unb64(p.metadata?.encoding);
|
|
74
|
+
if (encoding === 'binary/null') return null;
|
|
75
|
+
if (encoding === 'json/plain') {
|
|
76
|
+
try { return JSON.parse(unb64(p.data)); } catch { return undefined; }
|
|
77
|
+
}
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function jsonPayload(value: unknown, extra: Record<string, string> = {}): Obj {
|
|
82
|
+
return { metadata: { encoding: b64('json/plain'), ...Object.fromEntries(Object.entries(extra).map(([k, v]) => [k, b64(v)])) }, data: b64(JSON.stringify(value)) };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function uuidFrom(seed: string): string {
|
|
86
|
+
const hex = createHash('sha256').update(seed).digest('hex');
|
|
87
|
+
const variant = ((parseInt(hex[16]!, 16) & 0x3) | 0x8).toString(16);
|
|
88
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-${variant}${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const encodeToken = (token: Obj): string => b64(JSON.stringify(token));
|
|
92
|
+
function decodeToken(token: unknown, what: string): Obj {
|
|
93
|
+
try {
|
|
94
|
+
const parsed = JSON.parse(unb64(token as string)) as Obj;
|
|
95
|
+
if (parsed && typeof parsed === 'object') return parsed;
|
|
96
|
+
} catch { /* below */ }
|
|
97
|
+
throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `invalid ${what}`);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// ---- the run ---------------------------------------------------------------------------------------
|
|
101
|
+
|
|
102
|
+
type RetryPolicy = { initialInterval?: Duration; backoffCoefficient?: number; maximumInterval?: Duration; maximumAttempts?: number; nonRetryableErrorTypes?: string[] };
|
|
103
|
+
|
|
104
|
+
type ActivityState = {
|
|
105
|
+
scheduledEventId: number;
|
|
106
|
+
activityId: string;
|
|
107
|
+
activityType: Obj;
|
|
108
|
+
taskQueue: string;
|
|
109
|
+
input?: Obj;
|
|
110
|
+
header?: Obj;
|
|
111
|
+
priority?: Obj;
|
|
112
|
+
retryPolicy: RetryPolicy;
|
|
113
|
+
scheduleToCloseMs: number;
|
|
114
|
+
scheduleToStartMs: number;
|
|
115
|
+
startToCloseMs: number;
|
|
116
|
+
heartbeatMs: number;
|
|
117
|
+
attempt: number;
|
|
118
|
+
scheduledAt: number;
|
|
119
|
+
attemptScheduledAt: number;
|
|
120
|
+
availableAt: number;
|
|
121
|
+
startedAt?: number;
|
|
122
|
+
identity?: string;
|
|
123
|
+
requestId?: string;
|
|
124
|
+
heartbeatDetails?: Obj;
|
|
125
|
+
lastHeartbeatAt?: number;
|
|
126
|
+
lastFailure?: Obj;
|
|
127
|
+
lastAttemptCompletedAt?: number;
|
|
128
|
+
cancelRequestedEventId?: number;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
type ChildState = {
|
|
132
|
+
initiatedEventId: number;
|
|
133
|
+
namespace: string;
|
|
134
|
+
workflowId: string;
|
|
135
|
+
workflowType: Obj;
|
|
136
|
+
parentClosePolicy: string;
|
|
137
|
+
runId?: string;
|
|
138
|
+
startedEventId?: number;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
type WftState = {
|
|
142
|
+
scheduledEventId: number;
|
|
143
|
+
attempt: number;
|
|
144
|
+
scheduledAt: number;
|
|
145
|
+
availableAt: number;
|
|
146
|
+
startedEventId?: number;
|
|
147
|
+
startedAt?: number;
|
|
148
|
+
timeoutAt?: number;
|
|
149
|
+
identity?: string;
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
/** An event that arrived while a workflow task was in flight, appended once it closes (Temporal's
|
|
153
|
+
* buffered events). `linkPrevious` names an attribute set to the id of the event appended before it
|
|
154
|
+
* (an activity's close names the ActivityTaskStarted written in the same step). */
|
|
155
|
+
type BufferedEvent = { eventType: string; key: string; attributes: Obj; linkPrevious?: string; wakes: boolean };
|
|
156
|
+
|
|
157
|
+
type RunState = {
|
|
158
|
+
/** creation order among the World's runs: the latest run of a workflow id is its current one */
|
|
159
|
+
seq: number;
|
|
160
|
+
nextEventId: number;
|
|
161
|
+
history: string[];
|
|
162
|
+
historyBytes: number;
|
|
163
|
+
requestIds: string[];
|
|
164
|
+
/** the request id that started this run (StartWorkflowExecution / SignalWithStart), apart from signal and cancel ids */
|
|
165
|
+
startRequestIds?: string[];
|
|
166
|
+
workflowTaskTimeoutMs: number;
|
|
167
|
+
runTimeoutMs: number;
|
|
168
|
+
executionTimeoutMs: number;
|
|
169
|
+
runExpiresAt?: number;
|
|
170
|
+
executionExpiresAt?: number;
|
|
171
|
+
retryPolicy?: RetryPolicy;
|
|
172
|
+
attempt: number;
|
|
173
|
+
input?: Obj;
|
|
174
|
+
header?: Obj;
|
|
175
|
+
cronSchedule?: string;
|
|
176
|
+
firstWftAt?: number;
|
|
177
|
+
wft: WftState | null;
|
|
178
|
+
previousStartedEventId: number;
|
|
179
|
+
buffered: BufferedEvent[];
|
|
180
|
+
activities: Record<string, ActivityState>;
|
|
181
|
+
timers: Record<string, { startedEventId: number; fireAt: number }>;
|
|
182
|
+
children: Record<string, ChildState>;
|
|
183
|
+
parent?: { namespace: string; workflowId: string; runId: string; initiatedEventId: number };
|
|
184
|
+
cancelRequested: boolean;
|
|
185
|
+
closeEventId?: number;
|
|
186
|
+
newRunId?: string;
|
|
187
|
+
nextDue?: number;
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
/** A run as the engine handles it: WorkflowExecutionInfo fields at the top, `namespace`, `_state`. */
|
|
191
|
+
export type Run = Obj & {
|
|
192
|
+
id: string;
|
|
193
|
+
namespace: string;
|
|
194
|
+
execution: { workflowId: string; runId: string };
|
|
195
|
+
type: { name: string };
|
|
196
|
+
status: string;
|
|
197
|
+
taskQueue: string;
|
|
198
|
+
firstRunId: string;
|
|
199
|
+
historyLength: number;
|
|
200
|
+
startTime: Timestamp;
|
|
201
|
+
executionTime?: Timestamp;
|
|
202
|
+
closeTime?: Timestamp;
|
|
203
|
+
searchAttributes?: { indexedFields?: Record<string, Obj> };
|
|
204
|
+
memo?: { fields?: Record<string, Obj> };
|
|
205
|
+
parentExecution?: { workflowId: string; runId: string };
|
|
206
|
+
rootExecution?: { workflowId: string; runId: string };
|
|
207
|
+
_state: RunState;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
const RUNNING = 'WORKFLOW_EXECUTION_STATUS_RUNNING';
|
|
211
|
+
const EXECUTION_TYPE = 'workflow-execution';
|
|
212
|
+
const NAMESPACE_TYPE = 'namespace';
|
|
213
|
+
|
|
214
|
+
const STATUS_NAME: Record<string, string> = {
|
|
215
|
+
WORKFLOW_EXECUTION_STATUS_RUNNING: 'Running',
|
|
216
|
+
WORKFLOW_EXECUTION_STATUS_COMPLETED: 'Completed',
|
|
217
|
+
WORKFLOW_EXECUTION_STATUS_FAILED: 'Failed',
|
|
218
|
+
WORKFLOW_EXECUTION_STATUS_CANCELED: 'Canceled',
|
|
219
|
+
WORKFLOW_EXECUTION_STATUS_TERMINATED: 'Terminated',
|
|
220
|
+
WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW: 'ContinuedAsNew',
|
|
221
|
+
WORKFLOW_EXECUTION_STATUS_TIMED_OUT: 'TimedOut',
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
/** Temporal's predefined search attributes (https://docs.temporal.io/search-attribute#default-search-attribute). */
|
|
225
|
+
export const SYSTEM_SEARCH_ATTRIBUTES: Record<string, string> = {
|
|
226
|
+
BatcherUser: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
227
|
+
BinaryChecksums: 'INDEXED_VALUE_TYPE_KEYWORD_LIST',
|
|
228
|
+
BuildIds: 'INDEXED_VALUE_TYPE_KEYWORD_LIST',
|
|
229
|
+
CloseTime: 'INDEXED_VALUE_TYPE_DATETIME',
|
|
230
|
+
ExecutionDuration: 'INDEXED_VALUE_TYPE_INT',
|
|
231
|
+
ExecutionStatus: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
232
|
+
ExecutionTime: 'INDEXED_VALUE_TYPE_DATETIME',
|
|
233
|
+
HistoryLength: 'INDEXED_VALUE_TYPE_INT',
|
|
234
|
+
HistorySizeBytes: 'INDEXED_VALUE_TYPE_INT',
|
|
235
|
+
ParentWorkflowId: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
236
|
+
ParentRunId: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
237
|
+
RootWorkflowId: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
238
|
+
RootRunId: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
239
|
+
RunId: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
240
|
+
StartTime: 'INDEXED_VALUE_TYPE_DATETIME',
|
|
241
|
+
StateTransitionCount: 'INDEXED_VALUE_TYPE_INT',
|
|
242
|
+
TaskQueue: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
243
|
+
TemporalChangeVersion: 'INDEXED_VALUE_TYPE_KEYWORD_LIST',
|
|
244
|
+
WorkflowId: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
245
|
+
WorkflowType: 'INDEXED_VALUE_TYPE_KEYWORD',
|
|
246
|
+
};
|
|
247
|
+
/** the system attributes a worker may set on its own run (the SDK's patch markers upsert this one) */
|
|
248
|
+
const WORKER_SETTABLE_SYSTEM_ATTRIBUTES = new Set(['TemporalChangeVersion', 'BinaryChecksums', 'BuildIds']);
|
|
249
|
+
|
|
250
|
+
/** One run's changes inside a transition: the events it appends, in order. */
|
|
251
|
+
class RunTx {
|
|
252
|
+
readonly run: Run;
|
|
253
|
+
readonly events: HistoryEvent[] = [];
|
|
254
|
+
readonly now: number;
|
|
255
|
+
wakeQueues = false;
|
|
256
|
+
|
|
257
|
+
constructor(run: Run, now: number) {
|
|
258
|
+
// a deep copy: what the store lists is shared with the kernel's memory (TemporalStore.list)
|
|
259
|
+
this.run = structuredClone(run);
|
|
260
|
+
this.now = now;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
get state(): RunState { return this.run._state; }
|
|
264
|
+
|
|
265
|
+
/** Append one event now; answers its id. */
|
|
266
|
+
append(eventType: string, key: string, attributes: Obj): number {
|
|
267
|
+
const eventId = this.state.nextEventId;
|
|
268
|
+
this.state.nextEventId += 1;
|
|
269
|
+
const event: HistoryEvent = { eventId, eventTime: toTimestamp(this.now), eventType, taskId: eventId, [key]: attributes };
|
|
270
|
+
this.events.push(event);
|
|
271
|
+
this.state.historyBytes += JSON.stringify(event).length;
|
|
272
|
+
this.run.historyLength = eventId;
|
|
273
|
+
return eventId;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** Append an event a workflow task must see: buffered while a task is in flight, else appended with a
|
|
277
|
+
* workflow task scheduled for it. */
|
|
278
|
+
deliver(eventType: string, key: string, attributes: Obj, options: { linkPrevious?: string; wakes?: boolean } = {}): void {
|
|
279
|
+
const wakes = options.wakes ?? true;
|
|
280
|
+
if (this.state.wft?.startedEventId !== undefined) {
|
|
281
|
+
this.state.buffered.push({ eventType, key, attributes, wakes, ...(options.linkPrevious ? { linkPrevious: options.linkPrevious } : {}) });
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
this.land({ eventType, key, attributes, wakes, ...(options.linkPrevious ? { linkPrevious: options.linkPrevious } : {}) });
|
|
285
|
+
if (wakes) this.scheduleWorkflowTask();
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** Append a delivered event, resolving its link and recording what later events refer to. */
|
|
289
|
+
private land(event: BufferedEvent): number {
|
|
290
|
+
const attrs = { ...event.attributes };
|
|
291
|
+
if (event.linkPrevious) attrs[event.linkPrevious] = this.state.nextEventId - 1;
|
|
292
|
+
const eventId = this.append(event.eventType, event.key, attrs);
|
|
293
|
+
if (event.eventType === 'EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED') {
|
|
294
|
+
const child = this.state.children[String(attrs.initiatedEventId)];
|
|
295
|
+
if (child) child.startedEventId = eventId;
|
|
296
|
+
}
|
|
297
|
+
return eventId;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Schedule a workflow task unless one is already pending. */
|
|
301
|
+
scheduleWorkflowTask(attempt = 1, delayMs = 0): void {
|
|
302
|
+
if (this.run.status !== RUNNING || this.state.wft || this.state.firstWftAt !== undefined) return;
|
|
303
|
+
const scheduledEventId = this.append('EVENT_TYPE_WORKFLOW_TASK_SCHEDULED', 'workflowTaskScheduledEventAttributes', {
|
|
304
|
+
taskQueue: { name: this.run.taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
305
|
+
startToCloseTimeout: toDuration(this.state.workflowTaskTimeoutMs),
|
|
306
|
+
attempt,
|
|
307
|
+
});
|
|
308
|
+
this.state.wft = { scheduledEventId, attempt, scheduledAt: this.now, availableAt: this.now + delayMs };
|
|
309
|
+
this.wakeQueues = true;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/** Append what arrived while the task was in flight; answers whether any of it needs a decision. */
|
|
313
|
+
flushBuffered(): boolean {
|
|
314
|
+
let wakes = false;
|
|
315
|
+
for (const buffered of this.state.buffered) {
|
|
316
|
+
this.land(buffered);
|
|
317
|
+
wakes ||= buffered.wakes;
|
|
318
|
+
}
|
|
319
|
+
this.state.buffered = [];
|
|
320
|
+
return wakes;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function freshState(): RunState {
|
|
325
|
+
return {
|
|
326
|
+
seq: 0, nextEventId: 1, history: [], historyBytes: 0, requestIds: [], workflowTaskTimeoutMs: DEFAULT_WFT_TIMEOUT_MS,
|
|
327
|
+
runTimeoutMs: 0, executionTimeoutMs: 0, attempt: 1, wft: null, previousStartedEventId: 0, buffered: [],
|
|
328
|
+
activities: {}, timers: {}, children: {}, cancelRequested: false,
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** The server's default activity retry policy (Temporal docs, "Default values for Retry Policy"). */
|
|
333
|
+
function activityRetryPolicy(given: RetryPolicy | undefined): RetryPolicy {
|
|
334
|
+
const initial = durationMs(given?.initialInterval) || 1000;
|
|
335
|
+
return {
|
|
336
|
+
initialInterval: toDuration(initial),
|
|
337
|
+
backoffCoefficient: given?.backoffCoefficient && given.backoffCoefficient >= 1 ? given.backoffCoefficient : 2,
|
|
338
|
+
maximumInterval: toDuration(durationMs(given?.maximumInterval) || initial * 100),
|
|
339
|
+
maximumAttempts: given?.maximumAttempts ?? 0,
|
|
340
|
+
...(given?.nonRetryableErrorTypes?.length ? { nonRetryableErrorTypes: given.nonRetryableErrorTypes } : {}),
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
type RetryDecision = { retry: true; delayMs: number } | { retry: false; state: string };
|
|
345
|
+
|
|
346
|
+
/** Temporal's retry rule for a failed attempt `attempt` (1-based). */
|
|
347
|
+
function decideRetry(policy: RetryPolicy | undefined, attempt: number, failure: Obj | undefined, now: number, deadline?: number): RetryDecision {
|
|
348
|
+
if (!policy) return { retry: false, state: 'RETRY_STATE_RETRY_POLICY_NOT_SET' };
|
|
349
|
+
const app = failure?.applicationFailureInfo as { nonRetryable?: boolean; type?: string; nextRetryDelay?: Duration } | undefined;
|
|
350
|
+
const timeout = failure?.timeoutFailureInfo as { timeoutType?: string } | undefined;
|
|
351
|
+
if (app?.nonRetryable) return { retry: false, state: 'RETRY_STATE_NON_RETRYABLE_FAILURE' };
|
|
352
|
+
if (app?.type && policy.nonRetryableErrorTypes?.includes(app.type)) return { retry: false, state: 'RETRY_STATE_NON_RETRYABLE_FAILURE' };
|
|
353
|
+
if (timeout && timeout.timeoutType !== 'TIMEOUT_TYPE_START_TO_CLOSE' && timeout.timeoutType !== 'TIMEOUT_TYPE_HEARTBEAT') return { retry: false, state: 'RETRY_STATE_TIMEOUT' };
|
|
354
|
+
const maximumAttempts = policy.maximumAttempts ?? 0;
|
|
355
|
+
if (maximumAttempts > 0 && attempt >= maximumAttempts) return { retry: false, state: 'RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED' };
|
|
356
|
+
const initial = durationMs(policy.initialInterval) || 1000;
|
|
357
|
+
const coefficient = policy.backoffCoefficient ?? 2;
|
|
358
|
+
const maximum = durationMs(policy.maximumInterval) || initial * 100;
|
|
359
|
+
const delayMs = app?.nextRetryDelay ? durationMs(app.nextRetryDelay) : Math.min(initial * coefficient ** (attempt - 1), maximum);
|
|
360
|
+
if (deadline !== undefined && now + delayMs >= deadline) return { retry: false, state: 'RETRY_STATE_TIMEOUT' };
|
|
361
|
+
return { retry: true, delayMs };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
const TIMEOUT_NAME: Record<string, string> = {
|
|
365
|
+
TIMEOUT_TYPE_START_TO_CLOSE: 'StartToClose',
|
|
366
|
+
TIMEOUT_TYPE_SCHEDULE_TO_START: 'ScheduleToStart',
|
|
367
|
+
TIMEOUT_TYPE_SCHEDULE_TO_CLOSE: 'ScheduleToClose',
|
|
368
|
+
TIMEOUT_TYPE_HEARTBEAT: 'Heartbeat',
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
function timeoutFailure(kind: string, timeoutType: string, lastHeartbeatDetails?: Obj): Obj {
|
|
372
|
+
return {
|
|
373
|
+
message: `${kind} ${TIMEOUT_NAME[timeoutType] ?? 'unknown'} timeout`,
|
|
374
|
+
source: '',
|
|
375
|
+
timeoutFailureInfo: { timeoutType, ...(lastHeartbeatDetails ? { lastHeartbeatDetails } : {}) },
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
class UnhandledCommand extends Error {
|
|
380
|
+
readonly cause2: string;
|
|
381
|
+
constructor(cause: string, message: string) {
|
|
382
|
+
super(message);
|
|
383
|
+
this.cause2 = cause;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
type PendingQuery = { id: string; namespace: string; runId: string; taskQueue: string; query: Obj; resolve: (value: Obj) => void; reject: (error: Error) => void; dispatched: boolean };
|
|
388
|
+
|
|
389
|
+
export type EngineOptions = { root?: string; readOnly?: boolean };
|
|
390
|
+
|
|
391
|
+
export class TemporalEngine {
|
|
392
|
+
readonly store: TemporalStore;
|
|
393
|
+
readonly readOnly: boolean;
|
|
394
|
+
/** holders from engineForRoot (see releaseEngine) */
|
|
395
|
+
users = 0;
|
|
396
|
+
private lock: Promise<unknown> = Promise.resolve();
|
|
397
|
+
// counter: waiters parked on the next state change (long polls), resolved by `notify`
|
|
398
|
+
private waiters = new Set<() => void>();
|
|
399
|
+
// cache: in-flight queries — Temporal holds these in memory too; they are never state
|
|
400
|
+
private queries = new Map<string, PendingQuery>();
|
|
401
|
+
private querySeq = 0;
|
|
402
|
+
/** runs minted in the current transition and not yet written (see newRun) */
|
|
403
|
+
private pendingRuns = 0;
|
|
404
|
+
private ticker: ReturnType<typeof setInterval> | undefined;
|
|
405
|
+
private ticking = false;
|
|
406
|
+
// cache: sticky queue names a worker announced it is shutting down
|
|
407
|
+
private closedSticky = new Set<string>();
|
|
408
|
+
|
|
409
|
+
constructor(options: EngineOptions = {}) {
|
|
410
|
+
this.store = new TemporalStore(options.root);
|
|
411
|
+
this.readOnly = options.readOnly ?? false;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/** Start the clock-driven work: timers, timeouts, retries and delayed tasks falling due. */
|
|
415
|
+
start(intervalMs = 100): void {
|
|
416
|
+
if (this.ticker) return;
|
|
417
|
+
// a tick that fails (a World clock file caught mid-write, a store error) is reported and retried on
|
|
418
|
+
// the next tick; it never takes the server down
|
|
419
|
+
this.ticker = setInterval(() => {
|
|
420
|
+
this.tick().catch((error: unknown) => { process.stderr.write(`temporal: world-clock tick failed: ${(error as Error)?.message ?? String(error)}\n`); });
|
|
421
|
+
}, intervalMs);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
stop(): void {
|
|
425
|
+
if (this.ticker) clearInterval(this.ticker);
|
|
426
|
+
this.ticker = undefined;
|
|
427
|
+
for (const wake of this.waiters) wake();
|
|
428
|
+
this.waiters.clear();
|
|
429
|
+
for (const query of this.queries.values()) query.reject(new GrpcError(GrpcStatus.UNAVAILABLE, 'server stopping'));
|
|
430
|
+
this.queries.clear();
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
private serial<T>(fn: () => Promise<T>): Promise<T> {
|
|
434
|
+
const run = (): Promise<T> => {
|
|
435
|
+
this.pendingRuns = 0;
|
|
436
|
+
return fn();
|
|
437
|
+
};
|
|
438
|
+
const next = this.lock.then(run, run);
|
|
439
|
+
this.lock = next.catch(() => undefined);
|
|
440
|
+
return next;
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
private notify(): void {
|
|
444
|
+
const waiters = [...this.waiters];
|
|
445
|
+
this.waiters.clear();
|
|
446
|
+
for (const wake of waiters) wake();
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** Park until the next state change, the deadline, or the caller going away. */
|
|
450
|
+
private wait(signal: AbortSignal, until: number): Promise<void> {
|
|
451
|
+
return new Promise((resolve) => {
|
|
452
|
+
const timer = setTimeout(done, Math.max(0, until - Date.now()));
|
|
453
|
+
function done(): void {
|
|
454
|
+
clearTimeout(timer);
|
|
455
|
+
signal.removeEventListener('abort', done);
|
|
456
|
+
resolve();
|
|
457
|
+
}
|
|
458
|
+
this.waiters.add(done);
|
|
459
|
+
signal.addEventListener('abort', done, { once: true });
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
private assertWritable(): void {
|
|
464
|
+
if (this.readOnly) throw new GrpcError(GrpcStatus.PERMISSION_DENIED, 'this Temporal twin is read-only');
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// ---- reads ---------------------------------------------------------------------------------------
|
|
468
|
+
|
|
469
|
+
runs(namespace?: string): Run[] {
|
|
470
|
+
const all = this.store.list(EXECUTION_TYPE) as unknown as Run[];
|
|
471
|
+
return namespace === undefined ? all : all.filter((run) => run.namespace === namespace);
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
run(runId: string): Run | undefined {
|
|
475
|
+
return this.store.get(EXECUTION_TYPE, runId) as unknown as Run | undefined;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/** The run a request names: that run id, or the workflow id's current (latest) run. */
|
|
479
|
+
resolveRun(namespace: string, execution: { workflowId?: string; runId?: string } | undefined, firstExecutionRunId?: string): Run {
|
|
480
|
+
const workflowId = execution?.workflowId ?? '';
|
|
481
|
+
const runId = execution?.runId ?? '';
|
|
482
|
+
if (!workflowId) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'WorkflowId is not set on request.');
|
|
483
|
+
if (runId) {
|
|
484
|
+
const run = this.run(runId);
|
|
485
|
+
if (!run || run.namespace !== namespace || run.execution.workflowId !== workflowId) {
|
|
486
|
+
throw new GrpcError(GrpcStatus.NOT_FOUND, `workflow not found for ID: ${workflowId}`);
|
|
487
|
+
}
|
|
488
|
+
return run;
|
|
489
|
+
}
|
|
490
|
+
const candidates = this.runs(namespace).filter((run) => run.execution.workflowId === workflowId && (!firstExecutionRunId || run.firstRunId === firstExecutionRunId));
|
|
491
|
+
const current = latest(candidates);
|
|
492
|
+
if (!current) throw new GrpcError(GrpcStatus.NOT_FOUND, `workflow not found for ID: ${workflowId}`);
|
|
493
|
+
return current;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
async history(run: Run): Promise<HistoryEvent[]> {
|
|
497
|
+
return this.store.readChunks(run._state.history);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
private async commit(tx: RunTx, operation: string, input?: Obj): Promise<void> {
|
|
501
|
+
const state = tx.state;
|
|
502
|
+
if (tx.events.length > 0) {
|
|
503
|
+
state.history = [...state.history, await this.store.appendChunk(tx.events)];
|
|
504
|
+
}
|
|
505
|
+
tx.run.stateTransitionCount = Number(tx.run.stateTransitionCount ?? 0) + 1;
|
|
506
|
+
tx.run.historySizeBytes = state.historyBytes;
|
|
507
|
+
state.nextDue = nextDue(tx.run, tx.now);
|
|
508
|
+
const { id, ...fields } = tx.run;
|
|
509
|
+
await this.store.put(EXECUTION_TYPE, id, fields, operation, input);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// ---- namespaces and search attributes ------------------------------------------------------------
|
|
513
|
+
//
|
|
514
|
+
// A namespace is one subject: its DescribeNamespaceResponse, plus `_searchAttributes` (its custom
|
|
515
|
+
// search attributes, name → IndexedValueType) — namespace configuration, as the server keeps it.
|
|
516
|
+
// `default` exists from the start, as on Temporal's dev server; it is written to the tree the first
|
|
517
|
+
// time something changes it.
|
|
518
|
+
|
|
519
|
+
/** Every namespace, stored fields included (bookkeeping too). */
|
|
520
|
+
private namespaceRecords(): Obj[] {
|
|
521
|
+
const stored = this.store.list(NAMESPACE_TYPE).map(({ id: _id, ...fields }) => fields);
|
|
522
|
+
const hasDefault = stored.some((ns) => (ns.namespaceInfo as Obj | undefined)?.name === DEFAULT_NAMESPACE);
|
|
523
|
+
return hasDefault ? stored : [defaultNamespace(), ...stored];
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
private namespaceRecord(name: string): Obj {
|
|
527
|
+
const found = this.namespaceRecords().find((ns) => (ns.namespaceInfo as Obj).name === name || (ns.namespaceInfo as Obj).id === name);
|
|
528
|
+
if (!found) throw new GrpcError(GrpcStatus.NOT_FOUND, `Namespace ${name} is not found.`);
|
|
529
|
+
return found;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** The namespaces as DescribeNamespace answers them (bookkeeping never reaches the wire). */
|
|
533
|
+
namespaces(): Obj[] {
|
|
534
|
+
return this.namespaceRecords().map(publicView);
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
namespace(name: string): Obj {
|
|
538
|
+
return publicView(this.namespaceRecord(name));
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
private requireNamespace(name: unknown): string {
|
|
542
|
+
const namespace = String(name ?? '');
|
|
543
|
+
if (!namespace) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'Namespace not set on request.');
|
|
544
|
+
this.namespace(namespace);
|
|
545
|
+
return namespace;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
async registerNamespace(request: Obj): Promise<Obj> {
|
|
549
|
+
this.assertWritable();
|
|
550
|
+
const name = String(request.namespace ?? '');
|
|
551
|
+
if (!name) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'Namespace not set on request.');
|
|
552
|
+
return this.serial(async () => {
|
|
553
|
+
if (this.namespaces().some((ns) => (ns.namespaceInfo as Obj).name === name)) {
|
|
554
|
+
throw new GrpcError(GrpcStatus.ALREADY_EXISTS, 'Namespace already exists.');
|
|
555
|
+
}
|
|
556
|
+
const fields = {
|
|
557
|
+
namespaceInfo: {
|
|
558
|
+
name, state: 'NAMESPACE_STATE_REGISTERED', id: uuidFrom(`namespace\0${name}`),
|
|
559
|
+
...(request.description ? { description: request.description } : {}),
|
|
560
|
+
...(request.ownerEmail ? { ownerEmail: request.ownerEmail } : {}),
|
|
561
|
+
...(request.data ? { data: request.data } : {}),
|
|
562
|
+
capabilities: namespaceCapabilities(), supportsSchedules: false,
|
|
563
|
+
},
|
|
564
|
+
config: {
|
|
565
|
+
workflowExecutionRetentionTtl: request.workflowExecutionRetentionPeriod ?? { seconds: 86_400 },
|
|
566
|
+
historyArchivalState: 'ARCHIVAL_STATE_DISABLED',
|
|
567
|
+
visibilityArchivalState: 'ARCHIVAL_STATE_DISABLED',
|
|
568
|
+
},
|
|
569
|
+
replicationConfig: { activeClusterName: 'active', clusters: [{ clusterName: 'active' }], state: 'REPLICATION_STATE_NORMAL' },
|
|
570
|
+
isGlobalNamespace: false,
|
|
571
|
+
};
|
|
572
|
+
await this.store.put(NAMESPACE_TYPE, name, fields, 'RegisterNamespace', request);
|
|
573
|
+
return {};
|
|
574
|
+
});
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
customSearchAttributes(namespace: string): Record<string, string> {
|
|
578
|
+
return { ...((this.namespaceRecord(namespace)._searchAttributes ?? {}) as Record<string, string>) };
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/** Write a namespace's custom search attributes (the whole namespace subject, `default` included). */
|
|
582
|
+
private async putSearchAttributes(namespace: string, attributes: Record<string, string>, operation: string, request: Obj): Promise<void> {
|
|
583
|
+
const record = this.namespaceRecord(namespace);
|
|
584
|
+
const name = String((record.namespaceInfo as Obj).name);
|
|
585
|
+
await this.store.put(NAMESPACE_TYPE, name, { ...record, _searchAttributes: attributes }, operation, request);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
async addSearchAttributes(request: Obj): Promise<Obj> {
|
|
589
|
+
this.assertWritable();
|
|
590
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
591
|
+
const wanted = (request.searchAttributes ?? {}) as Record<string, string>;
|
|
592
|
+
return this.serial(async () => {
|
|
593
|
+
const existing = this.customSearchAttributes(namespace);
|
|
594
|
+
for (const [name, type] of Object.entries(wanted)) {
|
|
595
|
+
if (SYSTEM_SEARCH_ATTRIBUTES[name]) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `Search attribute ${name} is reserved by system.`);
|
|
596
|
+
if (!type || type === 'INDEXED_VALUE_TYPE_UNSPECIFIED') throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `Unknown search attribute type: ${type}.`);
|
|
597
|
+
if (existing[name] !== undefined && existing[name] !== type) {
|
|
598
|
+
throw new GrpcError(GrpcStatus.ALREADY_EXISTS, `Search attribute ${name} already exists.`);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
const next = { ...existing, ...wanted };
|
|
602
|
+
if (Object.keys(wanted).some((name) => existing[name] === undefined)) await this.putSearchAttributes(namespace, next, 'AddSearchAttributes', request);
|
|
603
|
+
return {};
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
async removeSearchAttributes(request: Obj): Promise<Obj> {
|
|
608
|
+
this.assertWritable();
|
|
609
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
610
|
+
const names = (request.searchAttributes ?? []) as string[];
|
|
611
|
+
return this.serial(async () => {
|
|
612
|
+
const existing = this.customSearchAttributes(namespace);
|
|
613
|
+
for (const name of names) {
|
|
614
|
+
if (existing[name] === undefined) throw new GrpcError(GrpcStatus.NOT_FOUND, `Search attribute ${name} doesn't exist.`);
|
|
615
|
+
}
|
|
616
|
+
for (const name of names) delete existing[name];
|
|
617
|
+
await this.putSearchAttributes(namespace, existing, 'RemoveSearchAttributes', request);
|
|
618
|
+
return {};
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
listSearchAttributes(request: Obj): Obj {
|
|
623
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
624
|
+
return { customAttributes: this.customSearchAttributes(namespace), systemAttributes: SYSTEM_SEARCH_ATTRIBUTES, storageSchema: {} };
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/** INVALID_ARGUMENT for a search attribute the namespace does not define (Temporal refuses these). */
|
|
628
|
+
private validateSearchAttributes(namespace: string, attributes: unknown, workerSet: boolean): void {
|
|
629
|
+
const fields = (attributes as { indexedFields?: Record<string, unknown> } | undefined)?.indexedFields ?? {};
|
|
630
|
+
const custom = this.customSearchAttributes(namespace);
|
|
631
|
+
for (const name of Object.keys(fields)) {
|
|
632
|
+
if (custom[name] !== undefined) continue;
|
|
633
|
+
if (workerSet && WORKER_SETTABLE_SYSTEM_ATTRIBUTES.has(name)) continue;
|
|
634
|
+
throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `Namespace ${namespace} has no mapping defined for search attribute ${name}`);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
private workerError(message: string): never {
|
|
639
|
+
throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_SEARCH_ATTRIBUTES', message);
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
// ---- starting runs -------------------------------------------------------------------------------
|
|
643
|
+
|
|
644
|
+
/** Create a run and its first events (inside a transition). */
|
|
645
|
+
private newRun(now: number, spec: {
|
|
646
|
+
namespace: string; workflowId: string; workflowType: Obj; taskQueue: string; input?: Obj; header?: Obj; memo?: Obj;
|
|
647
|
+
searchAttributes?: Obj; executionTimeoutMs: number; runTimeoutMs: number; workflowTaskTimeoutMs: number; retryPolicy?: RetryPolicy;
|
|
648
|
+
attempt?: number; identity?: string; requestId?: string; parent?: RunState['parent']; rootExecution?: { workflowId: string; runId: string };
|
|
649
|
+
firstRunId?: string; continuedExecutionRunId?: string; initiator?: string; continuedFailure?: Obj; lastCompletionResult?: Obj;
|
|
650
|
+
backoffMs?: number; executionExpiresAt?: number; signal?: { signalName: string; input?: Obj; header?: Obj; identity?: string };
|
|
651
|
+
priority?: Obj; userMetadata?: Obj; seed: string;
|
|
652
|
+
}): RunTx {
|
|
653
|
+
// after every run written so far, and after any this transition minted and has not written yet
|
|
654
|
+
const seq = this.runs().reduce((max, run) => Math.max(max, run._state.seq), 0) + 1 + this.pendingRuns;
|
|
655
|
+
this.pendingRuns += 1;
|
|
656
|
+
const runId = uuidFrom(`run\0${spec.namespace}\0${spec.workflowId}\0${spec.seed}\0${seq}`);
|
|
657
|
+
const state = freshState();
|
|
658
|
+
state.seq = seq;
|
|
659
|
+
state.workflowTaskTimeoutMs = spec.workflowTaskTimeoutMs;
|
|
660
|
+
state.runTimeoutMs = spec.runTimeoutMs;
|
|
661
|
+
state.executionTimeoutMs = spec.executionTimeoutMs;
|
|
662
|
+
state.attempt = spec.attempt ?? 1;
|
|
663
|
+
if (spec.retryPolicy) state.retryPolicy = spec.retryPolicy;
|
|
664
|
+
if (spec.input) state.input = spec.input;
|
|
665
|
+
if (spec.header) state.header = spec.header;
|
|
666
|
+
if (spec.requestId) state.startRequestIds = [spec.requestId];
|
|
667
|
+
if (spec.parent) state.parent = spec.parent;
|
|
668
|
+
const backoff = spec.backoffMs ?? 0;
|
|
669
|
+
const executionExpiresAt = spec.executionExpiresAt ?? (spec.executionTimeoutMs ? now + spec.executionTimeoutMs : undefined);
|
|
670
|
+
if (executionExpiresAt) state.executionExpiresAt = executionExpiresAt;
|
|
671
|
+
if (spec.runTimeoutMs) state.runExpiresAt = now + backoff + spec.runTimeoutMs;
|
|
672
|
+
const firstRunId = spec.firstRunId ?? runId;
|
|
673
|
+
const root = spec.rootExecution ?? { workflowId: spec.workflowId, runId };
|
|
674
|
+
const run: Run = {
|
|
675
|
+
id: runId,
|
|
676
|
+
namespace: spec.namespace,
|
|
677
|
+
execution: { workflowId: spec.workflowId, runId },
|
|
678
|
+
type: spec.workflowType as { name: string },
|
|
679
|
+
startTime: toTimestamp(now),
|
|
680
|
+
executionTime: toTimestamp(now + backoff),
|
|
681
|
+
status: RUNNING,
|
|
682
|
+
historyLength: 0,
|
|
683
|
+
taskQueue: spec.taskQueue,
|
|
684
|
+
firstRunId,
|
|
685
|
+
rootExecution: root,
|
|
686
|
+
stateTransitionCount: 0,
|
|
687
|
+
...(spec.memo ? { memo: spec.memo } : {}),
|
|
688
|
+
...(spec.searchAttributes ? { searchAttributes: spec.searchAttributes } : {}),
|
|
689
|
+
...(spec.parent ? { parentExecution: { workflowId: spec.parent.workflowId, runId: spec.parent.runId }, parentNamespaceId: String((this.namespace(spec.parent.namespace).namespaceInfo as Obj).id) } : {}),
|
|
690
|
+
...(spec.priority ? { priority: spec.priority } : {}),
|
|
691
|
+
_state: state,
|
|
692
|
+
};
|
|
693
|
+
const tx = new RunTx(run, now);
|
|
694
|
+
tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_STARTED', 'workflowExecutionStartedEventAttributes', {
|
|
695
|
+
workflowId: spec.workflowId,
|
|
696
|
+
workflowType: spec.workflowType,
|
|
697
|
+
taskQueue: { name: spec.taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
698
|
+
...(spec.input ? { input: spec.input } : {}),
|
|
699
|
+
...(spec.header ? { header: spec.header } : {}),
|
|
700
|
+
workflowExecutionTimeout: toDuration(spec.executionTimeoutMs),
|
|
701
|
+
workflowRunTimeout: toDuration(spec.runTimeoutMs),
|
|
702
|
+
workflowTaskTimeout: toDuration(spec.workflowTaskTimeoutMs),
|
|
703
|
+
...(spec.continuedExecutionRunId ? { continuedExecutionRunId: spec.continuedExecutionRunId } : {}),
|
|
704
|
+
...(spec.initiator ? { initiator: spec.initiator } : {}),
|
|
705
|
+
...(spec.continuedFailure ? { continuedFailure: spec.continuedFailure } : {}),
|
|
706
|
+
...(spec.lastCompletionResult ? { lastCompletionResult: spec.lastCompletionResult } : {}),
|
|
707
|
+
originalExecutionRunId: runId,
|
|
708
|
+
...(spec.identity ? { identity: spec.identity } : {}),
|
|
709
|
+
firstExecutionRunId: firstRunId,
|
|
710
|
+
...(spec.retryPolicy ? { retryPolicy: spec.retryPolicy } : {}),
|
|
711
|
+
attempt: state.attempt,
|
|
712
|
+
...(executionExpiresAt ? { workflowExecutionExpirationTime: toTimestamp(executionExpiresAt) } : {}),
|
|
713
|
+
...(backoff ? { firstWorkflowTaskBackoff: toDuration(backoff) } : {}),
|
|
714
|
+
...(spec.memo ? { memo: spec.memo } : {}),
|
|
715
|
+
...(spec.searchAttributes ? { searchAttributes: spec.searchAttributes } : {}),
|
|
716
|
+
...(spec.parent ? {
|
|
717
|
+
parentWorkflowNamespace: spec.parent.namespace,
|
|
718
|
+
parentWorkflowNamespaceId: run.parentNamespaceId,
|
|
719
|
+
parentWorkflowExecution: { workflowId: spec.parent.workflowId, runId: spec.parent.runId },
|
|
720
|
+
parentInitiatedEventId: spec.parent.initiatedEventId,
|
|
721
|
+
} : {}),
|
|
722
|
+
rootWorkflowExecution: root,
|
|
723
|
+
...(spec.priority ? { priority: spec.priority } : {}),
|
|
724
|
+
});
|
|
725
|
+
if (spec.signal) {
|
|
726
|
+
tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED', 'workflowExecutionSignaledEventAttributes', {
|
|
727
|
+
signalName: spec.signal.signalName,
|
|
728
|
+
...(spec.signal.input ? { input: spec.signal.input } : {}),
|
|
729
|
+
...(spec.signal.identity ? { identity: spec.signal.identity } : {}),
|
|
730
|
+
...(spec.signal.header ? { header: spec.signal.header } : {}),
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
if (backoff > 0) tx.state.firstWftAt = now + backoff;
|
|
734
|
+
else tx.scheduleWorkflowTask();
|
|
735
|
+
return tx;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
private startSpec(namespace: string, request: Obj, seed: string): Parameters<TemporalEngine['newRun']>[1] {
|
|
739
|
+
const workflowId = String(request.workflowId ?? '');
|
|
740
|
+
if (!workflowId) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'WorkflowId is not set on request.');
|
|
741
|
+
const workflowType = request.workflowType as Obj | undefined;
|
|
742
|
+
if (!workflowType?.name) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'WorkflowType is not set on request.');
|
|
743
|
+
const taskQueue = String((request.taskQueue as Obj | undefined)?.name ?? '');
|
|
744
|
+
if (!taskQueue) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'TaskQueue is not set on request.');
|
|
745
|
+
if (request.cronSchedule) throw new GrpcError(GrpcStatus.UNIMPLEMENTED, 'this Temporal twin does not run cron schedules (CronSchedule)');
|
|
746
|
+
if ((request.completionCallbacks as unknown[] | undefined)?.length) throw new GrpcError(GrpcStatus.UNIMPLEMENTED, 'this Temporal twin does not deliver completion callbacks');
|
|
747
|
+
if (request.versioningOverride) throw new GrpcError(GrpcStatus.UNIMPLEMENTED, 'this Temporal twin does not implement worker versioning (VersioningOverride)');
|
|
748
|
+
this.validateSearchAttributes(namespace, request.searchAttributes, false);
|
|
749
|
+
const executionTimeoutMs = durationMs(request.workflowExecutionTimeout);
|
|
750
|
+
let runTimeoutMs = durationMs(request.workflowRunTimeout);
|
|
751
|
+
if (executionTimeoutMs && (!runTimeoutMs || runTimeoutMs > executionTimeoutMs)) runTimeoutMs = executionTimeoutMs;
|
|
752
|
+
const workflowTaskTimeoutMs = Math.min(durationMs(request.workflowTaskTimeout) || DEFAULT_WFT_TIMEOUT_MS, MAX_WFT_TIMEOUT_MS, runTimeoutMs || MAX_WFT_TIMEOUT_MS);
|
|
753
|
+
return {
|
|
754
|
+
namespace, workflowId, workflowType, taskQueue, executionTimeoutMs, runTimeoutMs, workflowTaskTimeoutMs, seed,
|
|
755
|
+
...(request.input ? { input: request.input as Obj } : {}),
|
|
756
|
+
...(request.header ? { header: request.header as Obj } : {}),
|
|
757
|
+
...(request.memo ? { memo: request.memo as Obj } : {}),
|
|
758
|
+
...(request.searchAttributes ? { searchAttributes: request.searchAttributes as Obj } : {}),
|
|
759
|
+
...(request.retryPolicy ? { retryPolicy: request.retryPolicy as RetryPolicy } : {}),
|
|
760
|
+
...(request.identity ? { identity: String(request.identity) } : {}),
|
|
761
|
+
...(request.requestId ? { requestId: String(request.requestId) } : {}),
|
|
762
|
+
...(request.priority ? { priority: request.priority as Obj } : {}),
|
|
763
|
+
...(durationMs(request.workflowStartDelay) ? { backoffMs: durationMs(request.workflowStartDelay) } : {}),
|
|
764
|
+
...(request.lastCompletionResult ? { lastCompletionResult: request.lastCompletionResult as Obj } : {}),
|
|
765
|
+
...(request.continuedFailure ? { continuedFailure: request.continuedFailure as Obj } : {}),
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/** WorkflowIdConflictPolicy / WorkflowIdReusePolicy against the workflow id's latest run. Answers
|
|
770
|
+
* `existing` when the start resolves to a run that is already running (USE_EXISTING). */
|
|
771
|
+
private async admitStart(namespace: string, workflowId: string, request: Obj, now: number): Promise<{ existing?: Run; duplicate?: boolean }> {
|
|
772
|
+
const runs = this.runs(namespace).filter((run) => run.execution.workflowId === workflowId);
|
|
773
|
+
const current = latest(runs);
|
|
774
|
+
if (!current) return {};
|
|
775
|
+
const requestId = String(request.requestId ?? '');
|
|
776
|
+
// a request id the server has seen is the same request retried: it answers the run that request
|
|
777
|
+
// made, whether or not that run is still open
|
|
778
|
+
// a start retried with its request id answers the run it made while that run is still the
|
|
779
|
+
// current one (open or closed); start request ids are kept apart from signal and cancel ids
|
|
780
|
+
if (requestId && (current._state.startRequestIds ?? []).includes(requestId)) return { existing: current, duplicate: true };
|
|
781
|
+
if (current.status === RUNNING) {
|
|
782
|
+
const conflict = String(request.workflowIdConflictPolicy ?? 'WORKFLOW_ID_CONFLICT_POLICY_UNSPECIFIED');
|
|
783
|
+
const reuse = String(request.workflowIdReusePolicy ?? '');
|
|
784
|
+
if (conflict === 'WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING') return { existing: current };
|
|
785
|
+
if (conflict === 'WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING' || reuse === 'WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING') {
|
|
786
|
+
await this.terminateRun(current, now, { reason: 'terminated by new runID', identity: String(request.identity ?? '') }, 'StartWorkflowExecution');
|
|
787
|
+
return {};
|
|
788
|
+
}
|
|
789
|
+
throw new GrpcError(GrpcStatus.ALREADY_EXISTS, `Workflow execution is already running. WorkflowId: ${workflowId}, RunId: ${current.execution.runId}.`);
|
|
790
|
+
}
|
|
791
|
+
const reuse = String(request.workflowIdReusePolicy ?? 'WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE');
|
|
792
|
+
if (reuse === 'WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE') {
|
|
793
|
+
throw new GrpcError(GrpcStatus.ALREADY_EXISTS, `Workflow execution already finished and WorkflowIdReusePolicy is REJECT_DUPLICATE. WorkflowId: ${workflowId}, RunId: ${current.execution.runId}.`);
|
|
794
|
+
}
|
|
795
|
+
if (reuse === 'WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE_FAILED_ONLY' && current.status === 'WORKFLOW_EXECUTION_STATUS_COMPLETED') {
|
|
796
|
+
throw new GrpcError(GrpcStatus.ALREADY_EXISTS, `Workflow execution already finished successfully. WorkflowId: ${workflowId}, RunId: ${current.execution.runId}. Workflow Id reuse policy: allow duplicate workflow Id if last run failed.`);
|
|
797
|
+
}
|
|
798
|
+
return {};
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
async startWorkflowExecution(request: Obj): Promise<Obj> {
|
|
802
|
+
this.assertWritable();
|
|
803
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
804
|
+
if (request.requestEagerExecution) { /* eager start is a hint a server may decline; this one does */ }
|
|
805
|
+
return this.serial(async () => {
|
|
806
|
+
const now = nowMs();
|
|
807
|
+
const spec = this.startSpec(namespace, request, String(request.requestId ?? ''));
|
|
808
|
+
const { existing, duplicate } = await this.admitStart(namespace, spec.workflowId, request, now);
|
|
809
|
+
if (existing) return { runId: existing.execution.runId, started: Boolean(duplicate), status: existing.status };
|
|
810
|
+
const tx = this.newRun(now, spec);
|
|
811
|
+
await this.commit(tx, 'StartWorkflowExecution', request);
|
|
812
|
+
this.notify();
|
|
813
|
+
return { runId: tx.run.execution.runId, started: true, status: RUNNING };
|
|
814
|
+
});
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
async signalWithStartWorkflowExecution(request: Obj): Promise<Obj> {
|
|
818
|
+
this.assertWritable();
|
|
819
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
820
|
+
const signalName = String(request.signalName ?? '');
|
|
821
|
+
if (!signalName) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'SignalName is not set on request.');
|
|
822
|
+
return this.serial(async () => {
|
|
823
|
+
const now = nowMs();
|
|
824
|
+
const spec = this.startSpec(namespace, request, `signal\0${String(request.requestId ?? '')}`);
|
|
825
|
+
// signal-with-start signals a running run unless the conflict policy says otherwise
|
|
826
|
+
const current = latest(this.runs(namespace).filter((run) => run.execution.workflowId === spec.workflowId));
|
|
827
|
+
const conflict = String(request.workflowIdConflictPolicy ?? 'WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING');
|
|
828
|
+
if (current?.status === RUNNING && conflict !== 'WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING') {
|
|
829
|
+
if (conflict === 'WORKFLOW_ID_CONFLICT_POLICY_FAIL') {
|
|
830
|
+
throw new GrpcError(GrpcStatus.ALREADY_EXISTS, `Workflow execution is already running. WorkflowId: ${spec.workflowId}, RunId: ${current.execution.runId}.`);
|
|
831
|
+
}
|
|
832
|
+
const requestId = String(request.requestId ?? '');
|
|
833
|
+
// the same request retried signals once
|
|
834
|
+
if (requestId && (current._state.requestIds.includes(requestId) || (current._state.startRequestIds ?? []).includes(requestId))) return { runId: current.execution.runId, started: false };
|
|
835
|
+
const tx = new RunTx(current, now);
|
|
836
|
+
if (requestId) tx.state.requestIds = [...tx.state.requestIds, requestId].slice(-100);
|
|
837
|
+
this.signalInto(tx, { signalName, input: request.signalInput as Obj | undefined, header: request.header as Obj | undefined, identity: String(request.identity ?? '') });
|
|
838
|
+
await this.commit(tx, 'SignalWithStartWorkflowExecution', request);
|
|
839
|
+
this.notify();
|
|
840
|
+
return { runId: current.execution.runId, started: false };
|
|
841
|
+
}
|
|
842
|
+
const admitted = await this.admitStart(namespace, spec.workflowId, { ...request, workflowIdConflictPolicy: conflict }, now);
|
|
843
|
+
if (admitted.existing) return { runId: admitted.existing.execution.runId, started: false };
|
|
844
|
+
const tx = this.newRun(now, { ...spec, signal: { signalName, ...(request.signalInput ? { input: request.signalInput as Obj } : {}), ...(request.header ? { header: request.header as Obj } : {}), ...(request.identity ? { identity: String(request.identity) } : {}) } });
|
|
845
|
+
await this.commit(tx, 'SignalWithStartWorkflowExecution', request);
|
|
846
|
+
this.notify();
|
|
847
|
+
return { runId: tx.run.execution.runId, started: true };
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
// ---- signals, cancellation, termination ----------------------------------------------------------
|
|
852
|
+
|
|
853
|
+
private signalInto(tx: RunTx, signal: { signalName: string; input?: Obj; header?: Obj; identity?: string; external?: { workflowId: string; runId: string } }): void {
|
|
854
|
+
tx.deliver('EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED', 'workflowExecutionSignaledEventAttributes', {
|
|
855
|
+
signalName: signal.signalName,
|
|
856
|
+
...(signal.input ? { input: signal.input } : {}),
|
|
857
|
+
...(signal.identity ? { identity: signal.identity } : {}),
|
|
858
|
+
...(signal.header ? { header: signal.header } : {}),
|
|
859
|
+
...(signal.external ? { externalWorkflowExecution: signal.external } : {}),
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
async signalWorkflowExecution(request: Obj): Promise<Obj> {
|
|
864
|
+
this.assertWritable();
|
|
865
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
866
|
+
const signalName = String(request.signalName ?? '');
|
|
867
|
+
if (!signalName) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'SignalName is not set on request.');
|
|
868
|
+
return this.serial(async () => {
|
|
869
|
+
const run = this.resolveRun(namespace, request.workflowExecution as Obj);
|
|
870
|
+
if (run.status !== RUNNING) throw new GrpcError(GrpcStatus.NOT_FOUND, 'workflow execution already completed');
|
|
871
|
+
const requestId = String(request.requestId ?? '');
|
|
872
|
+
if (requestId && run._state.requestIds.includes(requestId)) return {};
|
|
873
|
+
const tx = new RunTx(run, nowMs());
|
|
874
|
+
if (requestId) tx.state.requestIds = [...tx.state.requestIds, requestId].slice(-100);
|
|
875
|
+
this.signalInto(tx, { signalName, input: request.input as Obj | undefined, header: request.header as Obj | undefined, identity: String(request.identity ?? '') });
|
|
876
|
+
await this.commit(tx, 'SignalWorkflowExecution', request);
|
|
877
|
+
this.notify();
|
|
878
|
+
return {};
|
|
879
|
+
});
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
async requestCancelWorkflowExecution(request: Obj): Promise<Obj> {
|
|
883
|
+
this.assertWritable();
|
|
884
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
885
|
+
return this.serial(async () => {
|
|
886
|
+
const run = this.resolveRun(namespace, request.workflowExecution as Obj, request.firstExecutionRunId as string | undefined);
|
|
887
|
+
if (run.status !== RUNNING) throw new GrpcError(GrpcStatus.NOT_FOUND, 'workflow execution already completed');
|
|
888
|
+
const requestId = String(request.requestId ?? '');
|
|
889
|
+
if (run._state.cancelRequested) return {};
|
|
890
|
+
const tx = new RunTx(run, nowMs());
|
|
891
|
+
this.cancelRequestInto(tx, { cause: String(request.reason ?? ''), identity: String(request.identity ?? '') });
|
|
892
|
+
if (requestId) tx.state.requestIds = [...tx.state.requestIds, requestId].slice(-100);
|
|
893
|
+
await this.commit(tx, 'RequestCancelWorkflowExecution', request);
|
|
894
|
+
this.notify();
|
|
895
|
+
return {};
|
|
896
|
+
});
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
private cancelRequestInto(tx: RunTx, request: { cause?: string; identity?: string; externalInitiatedEventId?: number; externalWorkflowExecution?: Obj }): void {
|
|
900
|
+
tx.state.cancelRequested = true;
|
|
901
|
+
tx.deliver('EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED', 'workflowExecutionCancelRequestedEventAttributes', {
|
|
902
|
+
...(request.cause ? { cause: request.cause } : {}),
|
|
903
|
+
...(request.identity ? { identity: request.identity } : {}),
|
|
904
|
+
...(request.externalInitiatedEventId ? { externalInitiatedEventId: request.externalInitiatedEventId } : {}),
|
|
905
|
+
...(request.externalWorkflowExecution ? { externalWorkflowExecution: request.externalWorkflowExecution } : {}),
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
async terminateWorkflowExecution(request: Obj): Promise<Obj> {
|
|
910
|
+
this.assertWritable();
|
|
911
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
912
|
+
return this.serial(async () => {
|
|
913
|
+
const run = this.resolveRun(namespace, request.workflowExecution as Obj, request.firstExecutionRunId as string | undefined);
|
|
914
|
+
if (run.status !== RUNNING) throw new GrpcError(GrpcStatus.NOT_FOUND, 'workflow execution already completed');
|
|
915
|
+
await this.terminateRun(run, nowMs(), { reason: String(request.reason ?? ''), identity: String(request.identity ?? ''), details: request.details as Obj | undefined }, 'TerminateWorkflowExecution', request);
|
|
916
|
+
this.notify();
|
|
917
|
+
return {};
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
private async terminateRun(run: Run, now: number, why: { reason: string; identity: string; details?: Obj }, operation: string, input?: Obj): Promise<void> {
|
|
922
|
+
const tx = new RunTx(run, now);
|
|
923
|
+
if (tx.state.wft?.startedEventId !== undefined) {
|
|
924
|
+
// a task in flight fails with the termination, as the server records it
|
|
925
|
+
tx.append('EVENT_TYPE_WORKFLOW_TASK_FAILED', 'workflowTaskFailedEventAttributes', {
|
|
926
|
+
scheduledEventId: tx.state.wft.scheduledEventId, startedEventId: tx.state.wft.startedEventId,
|
|
927
|
+
cause: 'WORKFLOW_TASK_FAILED_CAUSE_FORCE_CLOSE_COMMAND', identity: why.identity,
|
|
928
|
+
});
|
|
929
|
+
}
|
|
930
|
+
tx.state.wft = null;
|
|
931
|
+
tx.flushBuffered();
|
|
932
|
+
const closeEventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED', 'workflowExecutionTerminatedEventAttributes', {
|
|
933
|
+
reason: why.reason, ...(why.details ? { details: why.details } : {}), identity: why.identity,
|
|
934
|
+
});
|
|
935
|
+
await this.closeRun(tx, 'WORKFLOW_EXECUTION_STATUS_TERMINATED', closeEventId, operation, input, { terminated: true });
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/** Close a run: retire its tasks, apply its children's parent-close policy, tell its parent. */
|
|
939
|
+
private async closeRun(tx: RunTx, status: string, closeEventId: number, operation: string, input?: Obj, how: { terminated?: boolean; newRunId?: string; continued?: boolean } = {}): Promise<void> {
|
|
940
|
+
const run = tx.run;
|
|
941
|
+
const state = tx.state;
|
|
942
|
+
run.status = status;
|
|
943
|
+
run.closeTime = toTimestamp(tx.now);
|
|
944
|
+
run.executionDuration = toDuration(Math.max(0, tx.now - (timestampMs(run.executionTime) ?? tx.now)));
|
|
945
|
+
state.closeEventId = closeEventId;
|
|
946
|
+
state.wft = null;
|
|
947
|
+
state.firstWftAt = undefined;
|
|
948
|
+
state.buffered = [];
|
|
949
|
+
state.activities = {};
|
|
950
|
+
state.timers = {};
|
|
951
|
+
if (how.newRunId) state.newRunId = how.newRunId;
|
|
952
|
+
const children = Object.values(state.children);
|
|
953
|
+
state.children = {};
|
|
954
|
+
await this.commit(tx, operation, input);
|
|
955
|
+
// children by their parent-close policy: the run closing is what applies it, a continue-as-new included
|
|
956
|
+
{
|
|
957
|
+
for (const child of children) {
|
|
958
|
+
if (!child.runId) continue;
|
|
959
|
+
const childRun = this.run(child.runId);
|
|
960
|
+
const current = childRun ? latest(this.runs(childRun.namespace).filter((r) => r.execution.workflowId === childRun.execution.workflowId && r.firstRunId === childRun.firstRunId)) : undefined;
|
|
961
|
+
if (!current || current.status !== RUNNING) continue;
|
|
962
|
+
if (child.parentClosePolicy === 'PARENT_CLOSE_POLICY_ABANDON') continue;
|
|
963
|
+
if (child.parentClosePolicy === 'PARENT_CLOSE_POLICY_REQUEST_CANCEL') {
|
|
964
|
+
const childTx = new RunTx(current, tx.now);
|
|
965
|
+
this.cancelRequestInto(childTx, { cause: 'by parent close policy', identity: 'history-service' });
|
|
966
|
+
await this.commit(childTx, 'ParentClosePolicy');
|
|
967
|
+
} else {
|
|
968
|
+
await this.terminateRun(current, tx.now, { reason: 'by parent close policy', identity: 'history-service' }, 'ParentClosePolicy');
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
// the parent hears of its child's close when the chain ends (not on continue-as-new or a retry)
|
|
973
|
+
if (state.parent && !how.newRunId) await this.notifyParent(run, status, closeEventId, tx.now);
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
private async notifyParent(child: Run, status: string, closeEventId: number, now: number): Promise<void> {
|
|
977
|
+
const parentInfo = child._state.parent!;
|
|
978
|
+
const parent = this.run(parentInfo.runId);
|
|
979
|
+
if (!parent || parent.status !== RUNNING) return;
|
|
980
|
+
const pending = parent._state.children[String(parentInfo.initiatedEventId)];
|
|
981
|
+
if (!pending) return;
|
|
982
|
+
const events = await this.history(child);
|
|
983
|
+
const close: Record<string, unknown> = events.find((event) => event.eventId === closeEventId) ?? {};
|
|
984
|
+
const common = {
|
|
985
|
+
namespace: child.namespace,
|
|
986
|
+
namespaceId: String((this.namespace(child.namespace).namespaceInfo as Obj).id),
|
|
987
|
+
workflowExecution: { workflowId: child.execution.workflowId, runId: child.execution.runId },
|
|
988
|
+
workflowType: child.type,
|
|
989
|
+
initiatedEventId: parentInfo.initiatedEventId,
|
|
990
|
+
startedEventId: pending.startedEventId,
|
|
991
|
+
};
|
|
992
|
+
const tx = new RunTx(parent, now);
|
|
993
|
+
delete tx.state.children[String(parentInfo.initiatedEventId)];
|
|
994
|
+
switch (status) {
|
|
995
|
+
case 'WORKFLOW_EXECUTION_STATUS_COMPLETED': {
|
|
996
|
+
const attrs = close.workflowExecutionCompletedEventAttributes as Obj | undefined;
|
|
997
|
+
tx.deliver('EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED', 'childWorkflowExecutionCompletedEventAttributes', { ...common, ...(attrs?.result ? { result: attrs.result } : {}) });
|
|
998
|
+
break;
|
|
999
|
+
}
|
|
1000
|
+
case 'WORKFLOW_EXECUTION_STATUS_FAILED': {
|
|
1001
|
+
const attrs = close.workflowExecutionFailedEventAttributes as Obj | undefined;
|
|
1002
|
+
tx.deliver('EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_FAILED', 'childWorkflowExecutionFailedEventAttributes', { ...common, failure: attrs?.failure, retryState: attrs?.retryState ?? 'RETRY_STATE_RETRY_POLICY_NOT_SET' });
|
|
1003
|
+
break;
|
|
1004
|
+
}
|
|
1005
|
+
case 'WORKFLOW_EXECUTION_STATUS_CANCELED': {
|
|
1006
|
+
const attrs = close.workflowExecutionCanceledEventAttributes as Obj | undefined;
|
|
1007
|
+
tx.deliver('EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED', 'childWorkflowExecutionCanceledEventAttributes', { ...common, ...(attrs?.details ? { details: attrs.details } : {}) });
|
|
1008
|
+
break;
|
|
1009
|
+
}
|
|
1010
|
+
case 'WORKFLOW_EXECUTION_STATUS_TIMED_OUT':
|
|
1011
|
+
tx.deliver('EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TIMED_OUT', 'childWorkflowExecutionTimedOutEventAttributes', { ...common, retryState: 'RETRY_STATE_TIMEOUT' });
|
|
1012
|
+
break;
|
|
1013
|
+
default:
|
|
1014
|
+
tx.deliver('EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TERMINATED', 'childWorkflowExecutionTerminatedEventAttributes', common);
|
|
1015
|
+
}
|
|
1016
|
+
await this.commit(tx, 'ChildWorkflowExecutionClosed');
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
// ---- workflow tasks ------------------------------------------------------------------------------
|
|
1020
|
+
|
|
1021
|
+
async pollWorkflowTaskQueue(request: Obj, signal: AbortSignal, deadline?: number): Promise<Obj> {
|
|
1022
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
1023
|
+
const queue = request.taskQueue as { name?: string; kind?: string } | undefined;
|
|
1024
|
+
const name = String(queue?.name ?? '');
|
|
1025
|
+
if (!name) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'TaskQueue is not set on request.');
|
|
1026
|
+
const until = Math.min(Date.now() + LONG_POLL_MS, (deadline ?? Infinity) - 1000);
|
|
1027
|
+
// this server routes every workflow task through the normal queue with its full history (a sticky
|
|
1028
|
+
// queue is an optimization a server may skip); a sticky poll waits out its deadline
|
|
1029
|
+
if (queue?.kind === 'TASK_QUEUE_KIND_STICKY') {
|
|
1030
|
+
while (!signal.aborted && Date.now() < until && !this.closedSticky.has(name)) await this.wait(signal, until);
|
|
1031
|
+
return {};
|
|
1032
|
+
}
|
|
1033
|
+
const identity = String(request.identity ?? '');
|
|
1034
|
+
while (!signal.aborted) {
|
|
1035
|
+
const task = await this.serial(() => this.takeWorkflowTask(namespace, name, identity));
|
|
1036
|
+
if (task) return task;
|
|
1037
|
+
if (Date.now() >= until) return {};
|
|
1038
|
+
await this.wait(signal, until);
|
|
1039
|
+
}
|
|
1040
|
+
return {};
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
private async takeWorkflowTask(namespace: string, queue: string, identity: string): Promise<Obj | undefined> {
|
|
1044
|
+
for (const query of this.queries.values()) {
|
|
1045
|
+
if (query.dispatched || query.namespace !== namespace || query.taskQueue !== queue) continue;
|
|
1046
|
+
const run = this.run(query.runId);
|
|
1047
|
+
if (!run) continue;
|
|
1048
|
+
query.dispatched = true;
|
|
1049
|
+
const events = await this.history(run);
|
|
1050
|
+
return {
|
|
1051
|
+
taskToken: encodeToken({ kind: 'query', id: query.id }),
|
|
1052
|
+
workflowExecution: run.execution,
|
|
1053
|
+
workflowType: run.type,
|
|
1054
|
+
previousStartedEventId: run._state.previousStartedEventId,
|
|
1055
|
+
startedEventId: run._state.previousStartedEventId,
|
|
1056
|
+
attempt: 1,
|
|
1057
|
+
history: { events },
|
|
1058
|
+
query: query.query,
|
|
1059
|
+
workflowExecutionTaskQueue: { name: run.taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
1060
|
+
scheduledTime: toTimestamp(nowMs()),
|
|
1061
|
+
startedTime: toTimestamp(nowMs()),
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
const now = nowMs();
|
|
1065
|
+
const ready = this.runs(namespace)
|
|
1066
|
+
.filter((run) => run.status === RUNNING && run.taskQueue === queue && run._state.wft && run._state.wft.startedEventId === undefined && run._state.wft.availableAt <= now)
|
|
1067
|
+
.sort((a, b) => a._state.wft!.scheduledAt - b._state.wft!.scheduledAt);
|
|
1068
|
+
const run = ready[0];
|
|
1069
|
+
if (!run) return undefined;
|
|
1070
|
+
const tx = new RunTx(run, now);
|
|
1071
|
+
const wft = tx.state.wft!;
|
|
1072
|
+
const requestId = uuidFrom(`wft\0${run.id}\0${wft.scheduledEventId}\0${wft.attempt}`);
|
|
1073
|
+
const startedEventId = tx.append('EVENT_TYPE_WORKFLOW_TASK_STARTED', 'workflowTaskStartedEventAttributes', {
|
|
1074
|
+
scheduledEventId: wft.scheduledEventId, identity, requestId, historySizeBytes: tx.state.historyBytes,
|
|
1075
|
+
});
|
|
1076
|
+
wft.startedEventId = startedEventId;
|
|
1077
|
+
wft.startedAt = now;
|
|
1078
|
+
wft.timeoutAt = now + tx.state.workflowTaskTimeoutMs;
|
|
1079
|
+
wft.identity = identity;
|
|
1080
|
+
await this.commit(tx, 'PollWorkflowTaskQueue');
|
|
1081
|
+
const events = await this.history(tx.run);
|
|
1082
|
+
return {
|
|
1083
|
+
taskToken: encodeToken({ kind: 'workflow', runId: run.id, scheduledEventId: wft.scheduledEventId, startedEventId, attempt: wft.attempt }),
|
|
1084
|
+
workflowExecution: run.execution,
|
|
1085
|
+
workflowType: run.type,
|
|
1086
|
+
previousStartedEventId: tx.state.previousStartedEventId,
|
|
1087
|
+
startedEventId,
|
|
1088
|
+
attempt: wft.attempt,
|
|
1089
|
+
history: { events },
|
|
1090
|
+
workflowExecutionTaskQueue: { name: run.taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
1091
|
+
scheduledTime: toTimestamp(wft.scheduledAt),
|
|
1092
|
+
startedTime: toTimestamp(now),
|
|
1093
|
+
queries: {},
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
/** The run and in-flight task a workflow task token names, or NOT_FOUND. */
|
|
1098
|
+
private workflowTaskOf(token: Obj): Run {
|
|
1099
|
+
const run = this.run(String(token.runId ?? ''));
|
|
1100
|
+
const wft = run?._state.wft;
|
|
1101
|
+
if (!run || run.status !== RUNNING || !wft || wft.startedEventId === undefined || wft.scheduledEventId !== token.scheduledEventId || wft.startedEventId !== token.startedEventId) {
|
|
1102
|
+
throw new GrpcError(GrpcStatus.NOT_FOUND, 'Workflow task not found.');
|
|
1103
|
+
}
|
|
1104
|
+
return run;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
async respondWorkflowTaskCompleted(request: Obj): Promise<Obj> {
|
|
1108
|
+
this.assertWritable();
|
|
1109
|
+
const token = decodeToken(request.taskToken, 'TaskToken');
|
|
1110
|
+
if (token.kind !== 'workflow') throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'invalid TaskToken');
|
|
1111
|
+
return this.serial(async () => {
|
|
1112
|
+
const run = this.workflowTaskOf(token);
|
|
1113
|
+
const now = nowMs();
|
|
1114
|
+
const tx = new RunTx(run, now);
|
|
1115
|
+
const wft = tx.state.wft!;
|
|
1116
|
+
const commands = (request.commands ?? []) as Obj[];
|
|
1117
|
+
const closing = commands.find((command) => CLOSING_COMMANDS.has(String(command.commandType)));
|
|
1118
|
+
const messages = (request.messages ?? []) as unknown[];
|
|
1119
|
+
if (messages.length > 0) {
|
|
1120
|
+
await this.failWorkflowTask(tx, 'WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED', 'this Temporal twin does not implement workflow updates (protocol messages)', String(request.identity ?? ''));
|
|
1121
|
+
return {};
|
|
1122
|
+
}
|
|
1123
|
+
if (closing && tx.state.buffered.some((event) => event.wakes)) {
|
|
1124
|
+
// new events arrived while the task ran: the workflow may not close over them
|
|
1125
|
+
await this.failWorkflowTask(tx, 'WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND', 'UnhandledCommand', String(request.identity ?? ''), { sameAttempt: true });
|
|
1126
|
+
return {};
|
|
1127
|
+
}
|
|
1128
|
+
const checkpoint = JSON.stringify(tx.run);
|
|
1129
|
+
const eventsBefore = tx.events.length;
|
|
1130
|
+
const completedEventId = tx.append('EVENT_TYPE_WORKFLOW_TASK_COMPLETED', 'workflowTaskCompletedEventAttributes', {
|
|
1131
|
+
scheduledEventId: wft.scheduledEventId,
|
|
1132
|
+
startedEventId: wft.startedEventId,
|
|
1133
|
+
identity: String(request.identity ?? ''),
|
|
1134
|
+
...(request.binaryChecksum ? { binaryChecksum: request.binaryChecksum } : {}),
|
|
1135
|
+
...(request.workerVersionStamp ? { workerVersion: request.workerVersionStamp } : {}),
|
|
1136
|
+
...(request.sdkMetadata ? { sdkMetadata: request.sdkMetadata } : {}),
|
|
1137
|
+
...(request.meteringMetadata ? { meteringMetadata: request.meteringMetadata } : {}),
|
|
1138
|
+
...(request.deployment ? { deployment: request.deployment } : {}),
|
|
1139
|
+
...(request.versioningBehavior ? { versioningBehavior: request.versioningBehavior } : {}),
|
|
1140
|
+
});
|
|
1141
|
+
tx.state.previousStartedEventId = wft.startedEventId!;
|
|
1142
|
+
tx.state.wft = null;
|
|
1143
|
+
const after: Array<() => Promise<void>> = [];
|
|
1144
|
+
let closed: { status: string; eventId: number; newRun?: RunTx; continued?: boolean } | undefined;
|
|
1145
|
+
let needsTask = Boolean(request.forceCreateNewWorkflowTask);
|
|
1146
|
+
try {
|
|
1147
|
+
for (const command of commands) {
|
|
1148
|
+
if (closed) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND', `${String(command.commandType)} follows a command that closed the workflow`);
|
|
1149
|
+
const result = this.applyCommand(tx, command, completedEventId, String(request.identity ?? ''), after);
|
|
1150
|
+
if (result?.closed) closed = result.closed;
|
|
1151
|
+
if (result?.wakes) needsTask = true;
|
|
1152
|
+
}
|
|
1153
|
+
} catch (error) {
|
|
1154
|
+
if (!(error instanceof UnhandledCommand)) throw error;
|
|
1155
|
+
// the commands are refused whole: restore the run as it stood and fail the task instead
|
|
1156
|
+
const restored = JSON.parse(checkpoint) as Run;
|
|
1157
|
+
const retry = new RunTx(restored, now);
|
|
1158
|
+
tx.events.splice(eventsBefore);
|
|
1159
|
+
await this.failWorkflowTask(retry, error.cause2, error.message, String(request.identity ?? ''));
|
|
1160
|
+
return {};
|
|
1161
|
+
}
|
|
1162
|
+
if (closed) {
|
|
1163
|
+
await this.closeRun(tx, closed.status, closed.eventId, 'RespondWorkflowTaskCompleted', undefined, { ...(closed.newRun ? { newRunId: closed.newRun.run.id } : {}), ...(closed.continued ? { continued: true } : {}) });
|
|
1164
|
+
if (closed.newRun) await this.commit(closed.newRun, 'RespondWorkflowTaskCompleted');
|
|
1165
|
+
} else {
|
|
1166
|
+
if (tx.flushBuffered()) needsTask = true;
|
|
1167
|
+
if (needsTask) tx.scheduleWorkflowTask();
|
|
1168
|
+
await this.commit(tx, 'RespondWorkflowTaskCompleted');
|
|
1169
|
+
}
|
|
1170
|
+
for (const step of after) await step();
|
|
1171
|
+
this.notify();
|
|
1172
|
+
return {};
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/** One command → its events (and, for cross-run effects, a step run after this run is written). */
|
|
1177
|
+
private applyCommand(tx: RunTx, command: Obj, completedEventId: number, identity: string, after: Array<() => Promise<void>>): { closed?: { status: string; eventId: number; newRun?: RunTx; continued?: boolean }; wakes?: boolean } | undefined {
|
|
1178
|
+
const type = String(command.commandType ?? '');
|
|
1179
|
+
const run = tx.run;
|
|
1180
|
+
const state = tx.state;
|
|
1181
|
+
const now = tx.now;
|
|
1182
|
+
switch (type) {
|
|
1183
|
+
case 'COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK': {
|
|
1184
|
+
const a = (command.scheduleActivityTaskCommandAttributes ?? {}) as Obj;
|
|
1185
|
+
const activityId = String(a.activityId ?? '');
|
|
1186
|
+
const activityType = a.activityType as Obj | undefined;
|
|
1187
|
+
if (!activityId || !activityType?.name) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_SCHEDULE_ACTIVITY_ATTRIBUTES', 'ActivityId and ActivityType are required');
|
|
1188
|
+
if (Object.values(state.activities).some((activity) => activity.activityId === activityId)) {
|
|
1189
|
+
throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_SCHEDULE_ACTIVITY_DUPLICATE_ID', `Duplicate activity ID: ${activityId}`);
|
|
1190
|
+
}
|
|
1191
|
+
const taskQueue = String((a.taskQueue as Obj | undefined)?.name || run.taskQueue);
|
|
1192
|
+
let scheduleToClose = durationMs(a.scheduleToCloseTimeout);
|
|
1193
|
+
let scheduleToStart = durationMs(a.scheduleToStartTimeout);
|
|
1194
|
+
let startToClose = durationMs(a.startToCloseTimeout);
|
|
1195
|
+
let heartbeat = durationMs(a.heartbeatTimeout);
|
|
1196
|
+
if (!scheduleToClose && !startToClose) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_SCHEDULE_ACTIVITY_ATTRIBUTES', 'A valid StartToClose or ScheduleToCloseTimeout is not set on ScheduleActivityTaskCommand.');
|
|
1197
|
+
const runLeft = state.runExpiresAt ? state.runExpiresAt - now : 0;
|
|
1198
|
+
if (!scheduleToClose && runLeft > 0) scheduleToClose = runLeft;
|
|
1199
|
+
if (runLeft > 0 && scheduleToClose > runLeft) scheduleToClose = runLeft;
|
|
1200
|
+
if (!scheduleToStart || (scheduleToClose && scheduleToStart > scheduleToClose)) scheduleToStart = scheduleToClose;
|
|
1201
|
+
if (!startToClose || (scheduleToClose && startToClose > scheduleToClose)) startToClose = scheduleToClose;
|
|
1202
|
+
if (heartbeat > startToClose) heartbeat = startToClose;
|
|
1203
|
+
const given = a.retryPolicy as RetryPolicy | undefined;
|
|
1204
|
+
if (given?.backoffCoefficient !== undefined && given.backoffCoefficient !== 0 && given.backoffCoefficient < 1) {
|
|
1205
|
+
throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_SCHEDULE_ACTIVITY_ATTRIBUTES', 'BackoffCoefficient cannot be less than 1.');
|
|
1206
|
+
}
|
|
1207
|
+
const retryPolicy = activityRetryPolicy(given);
|
|
1208
|
+
const scheduledEventId = tx.append('EVENT_TYPE_ACTIVITY_TASK_SCHEDULED', 'activityTaskScheduledEventAttributes', {
|
|
1209
|
+
activityId, activityType, taskQueue: { name: taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
1210
|
+
...(a.header ? { header: a.header } : {}), ...(a.input ? { input: a.input } : {}),
|
|
1211
|
+
scheduleToCloseTimeout: toDuration(scheduleToClose), scheduleToStartTimeout: toDuration(scheduleToStart),
|
|
1212
|
+
startToCloseTimeout: toDuration(startToClose), heartbeatTimeout: toDuration(heartbeat),
|
|
1213
|
+
workflowTaskCompletedEventId: completedEventId, retryPolicy,
|
|
1214
|
+
...(a.useWorkflowBuildId ? { useWorkflowBuildId: a.useWorkflowBuildId } : {}),
|
|
1215
|
+
...(a.priority ? { priority: a.priority } : {}),
|
|
1216
|
+
});
|
|
1217
|
+
state.activities[String(scheduledEventId)] = {
|
|
1218
|
+
scheduledEventId, activityId, activityType, taskQueue, retryPolicy,
|
|
1219
|
+
...(a.input ? { input: a.input as Obj } : {}), ...(a.header ? { header: a.header as Obj } : {}), ...(a.priority ? { priority: a.priority as Obj } : {}),
|
|
1220
|
+
scheduleToCloseMs: scheduleToClose, scheduleToStartMs: scheduleToStart, startToCloseMs: startToClose, heartbeatMs: heartbeat,
|
|
1221
|
+
attempt: 1, scheduledAt: now, attemptScheduledAt: now, availableAt: now,
|
|
1222
|
+
};
|
|
1223
|
+
tx.wakeQueues = true;
|
|
1224
|
+
return undefined;
|
|
1225
|
+
}
|
|
1226
|
+
case 'COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK': {
|
|
1227
|
+
const a = (command.requestCancelActivityTaskCommandAttributes ?? {}) as Obj;
|
|
1228
|
+
const activity = state.activities[String(a.scheduledEventId)];
|
|
1229
|
+
if (!activity) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_ACTIVITY_ATTRIBUTES', `Unable to find activity with scheduled event ID ${String(a.scheduledEventId)}`);
|
|
1230
|
+
const requestedId = tx.append('EVENT_TYPE_ACTIVITY_TASK_CANCEL_REQUESTED', 'activityTaskCancelRequestedEventAttributes', { scheduledEventId: activity.scheduledEventId, workflowTaskCompletedEventId: completedEventId });
|
|
1231
|
+
if (activity.startedAt === undefined) {
|
|
1232
|
+
tx.append('EVENT_TYPE_ACTIVITY_TASK_CANCELED', 'activityTaskCanceledEventAttributes', { scheduledEventId: activity.scheduledEventId, latestCancelRequestedEventId: requestedId, identity });
|
|
1233
|
+
delete state.activities[String(activity.scheduledEventId)];
|
|
1234
|
+
return { wakes: true };
|
|
1235
|
+
}
|
|
1236
|
+
activity.cancelRequestedEventId = requestedId;
|
|
1237
|
+
return undefined;
|
|
1238
|
+
}
|
|
1239
|
+
case 'COMMAND_TYPE_START_TIMER': {
|
|
1240
|
+
const a = (command.startTimerCommandAttributes ?? {}) as Obj;
|
|
1241
|
+
const timerId = String(a.timerId ?? '');
|
|
1242
|
+
const fireMs = durationMs(a.startToFireTimeout);
|
|
1243
|
+
if (!timerId || fireMs <= 0) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_START_TIMER_ATTRIBUTES', 'A valid TimerId and StartToFireTimeout are required');
|
|
1244
|
+
if (state.timers[timerId]) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_START_TIMER_DUPLICATE_ID', `TimerId ${timerId} is already in use`);
|
|
1245
|
+
const startedEventId = tx.append('EVENT_TYPE_TIMER_STARTED', 'timerStartedEventAttributes', { timerId, startToFireTimeout: a.startToFireTimeout, workflowTaskCompletedEventId: completedEventId });
|
|
1246
|
+
state.timers[timerId] = { startedEventId, fireAt: now + fireMs };
|
|
1247
|
+
return undefined;
|
|
1248
|
+
}
|
|
1249
|
+
case 'COMMAND_TYPE_CANCEL_TIMER': {
|
|
1250
|
+
const a = (command.cancelTimerCommandAttributes ?? {}) as Obj;
|
|
1251
|
+
const timerId = String(a.timerId ?? '');
|
|
1252
|
+
const timer = state.timers[timerId];
|
|
1253
|
+
if (!timer) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_CANCEL_TIMER_ATTRIBUTES', `TimerId ${timerId} is not open`);
|
|
1254
|
+
tx.append('EVENT_TYPE_TIMER_CANCELED', 'timerCanceledEventAttributes', { timerId, startedEventId: timer.startedEventId, workflowTaskCompletedEventId: completedEventId, identity });
|
|
1255
|
+
delete state.timers[timerId];
|
|
1256
|
+
return undefined;
|
|
1257
|
+
}
|
|
1258
|
+
case 'COMMAND_TYPE_RECORD_MARKER': {
|
|
1259
|
+
const a = (command.recordMarkerCommandAttributes ?? {}) as Obj;
|
|
1260
|
+
if (!a.markerName) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_RECORD_MARKER_ATTRIBUTES', 'MarkerName is not set on RecordMarkerCommand.');
|
|
1261
|
+
tx.append('EVENT_TYPE_MARKER_RECORDED', 'markerRecordedEventAttributes', { ...a, workflowTaskCompletedEventId: completedEventId });
|
|
1262
|
+
return undefined;
|
|
1263
|
+
}
|
|
1264
|
+
case 'COMMAND_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES': {
|
|
1265
|
+
const a = (command.upsertWorkflowSearchAttributesCommandAttributes ?? {}) as Obj;
|
|
1266
|
+
try {
|
|
1267
|
+
this.validateSearchAttributes(run.namespace, a.searchAttributes, true);
|
|
1268
|
+
} catch (error) {
|
|
1269
|
+
this.workerError((error as Error).message);
|
|
1270
|
+
}
|
|
1271
|
+
tx.append('EVENT_TYPE_UPSERT_WORKFLOW_SEARCH_ATTRIBUTES', 'upsertWorkflowSearchAttributesEventAttributes', { searchAttributes: a.searchAttributes, workflowTaskCompletedEventId: completedEventId });
|
|
1272
|
+
run.searchAttributes = { indexedFields: mergeFields(run.searchAttributes?.indexedFields, (a.searchAttributes as Obj | undefined)?.indexedFields as Record<string, Obj> | undefined) };
|
|
1273
|
+
return undefined;
|
|
1274
|
+
}
|
|
1275
|
+
case 'COMMAND_TYPE_MODIFY_WORKFLOW_PROPERTIES': {
|
|
1276
|
+
const a = (command.modifyWorkflowPropertiesCommandAttributes ?? {}) as Obj;
|
|
1277
|
+
tx.append('EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED', 'workflowPropertiesModifiedEventAttributes', { upsertedMemo: a.upsertedMemo, workflowTaskCompletedEventId: completedEventId });
|
|
1278
|
+
run.memo = { fields: mergeFields(run.memo?.fields, (a.upsertedMemo as Obj | undefined)?.fields as Record<string, Obj> | undefined) };
|
|
1279
|
+
return undefined;
|
|
1280
|
+
}
|
|
1281
|
+
case 'COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION': {
|
|
1282
|
+
const a = (command.completeWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1283
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED', 'workflowExecutionCompletedEventAttributes', { ...(a.result ? { result: a.result } : {}), workflowTaskCompletedEventId: completedEventId });
|
|
1284
|
+
return { closed: { status: 'WORKFLOW_EXECUTION_STATUS_COMPLETED', eventId } };
|
|
1285
|
+
}
|
|
1286
|
+
case 'COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION': {
|
|
1287
|
+
const a = (command.failWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1288
|
+
const decision = decideRetry(state.retryPolicy, state.attempt, a.failure as Obj | undefined, now, state.executionExpiresAt);
|
|
1289
|
+
if (decision.retry) {
|
|
1290
|
+
const next = this.nextRunOf(tx, { backoffMs: decision.delayMs, attempt: state.attempt + 1, initiator: 'CONTINUE_AS_NEW_INITIATOR_RETRY', continuedFailure: a.failure as Obj | undefined });
|
|
1291
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_FAILED', 'workflowExecutionFailedEventAttributes', { failure: a.failure, retryState: 'RETRY_STATE_IN_PROGRESS', workflowTaskCompletedEventId: completedEventId, newExecutionRunId: next.run.id });
|
|
1292
|
+
return { closed: { status: 'WORKFLOW_EXECUTION_STATUS_FAILED', eventId, newRun: next } };
|
|
1293
|
+
}
|
|
1294
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_FAILED', 'workflowExecutionFailedEventAttributes', { failure: a.failure, retryState: decision.state, workflowTaskCompletedEventId: completedEventId });
|
|
1295
|
+
return { closed: { status: 'WORKFLOW_EXECUTION_STATUS_FAILED', eventId } };
|
|
1296
|
+
}
|
|
1297
|
+
case 'COMMAND_TYPE_CANCEL_WORKFLOW_EXECUTION': {
|
|
1298
|
+
const a = (command.cancelWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1299
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED', 'workflowExecutionCanceledEventAttributes', { ...(a.details ? { details: a.details } : {}), workflowTaskCompletedEventId: completedEventId });
|
|
1300
|
+
return { closed: { status: 'WORKFLOW_EXECUTION_STATUS_CANCELED', eventId } };
|
|
1301
|
+
}
|
|
1302
|
+
case 'COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION': {
|
|
1303
|
+
const a = (command.continueAsNewWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1304
|
+
if (a.cronSchedule) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED', 'this Temporal twin does not run cron schedules (CronSchedule on ContinueAsNew)');
|
|
1305
|
+
try {
|
|
1306
|
+
this.validateSearchAttributes(run.namespace, a.searchAttributes, true);
|
|
1307
|
+
} catch (error) {
|
|
1308
|
+
this.workerError((error as Error).message);
|
|
1309
|
+
}
|
|
1310
|
+
const backoffMs = durationMs(a.backoffStartInterval);
|
|
1311
|
+
const next = this.nextRunOf(tx, {
|
|
1312
|
+
backoffMs, attempt: 1, initiator: 'CONTINUE_AS_NEW_INITIATOR_WORKFLOW',
|
|
1313
|
+
...(a.workflowType && (a.workflowType as Obj).name ? { workflowType: a.workflowType as Obj } : {}),
|
|
1314
|
+
...((a.taskQueue as Obj | undefined)?.name ? { taskQueue: String((a.taskQueue as Obj).name) } : {}),
|
|
1315
|
+
input: a.input as Obj | undefined,
|
|
1316
|
+
...(a.header ? { header: a.header as Obj } : {}),
|
|
1317
|
+
...(durationMs(a.workflowRunTimeout) ? { runTimeoutMs: durationMs(a.workflowRunTimeout) } : {}),
|
|
1318
|
+
...(durationMs(a.workflowTaskTimeout) ? { workflowTaskTimeoutMs: durationMs(a.workflowTaskTimeout) } : {}),
|
|
1319
|
+
...(a.memo ? { memo: a.memo as Obj } : {}),
|
|
1320
|
+
...(a.searchAttributes ? { searchAttributes: a.searchAttributes as Obj } : {}),
|
|
1321
|
+
...(a.retryPolicy ? { retryPolicy: a.retryPolicy as RetryPolicy } : {}),
|
|
1322
|
+
...(a.lastCompletionResult ? { lastCompletionResult: a.lastCompletionResult as Obj } : {}),
|
|
1323
|
+
});
|
|
1324
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW', 'workflowExecutionContinuedAsNewEventAttributes', {
|
|
1325
|
+
newExecutionRunId: next.run.id,
|
|
1326
|
+
workflowType: next.run.type,
|
|
1327
|
+
taskQueue: { name: next.run.taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
1328
|
+
...(a.input ? { input: a.input } : {}),
|
|
1329
|
+
workflowRunTimeout: toDuration(next.state.runTimeoutMs),
|
|
1330
|
+
workflowTaskTimeout: toDuration(next.state.workflowTaskTimeoutMs),
|
|
1331
|
+
workflowTaskCompletedEventId: completedEventId,
|
|
1332
|
+
...(backoffMs ? { backoffStartInterval: toDuration(backoffMs) } : {}),
|
|
1333
|
+
initiator: 'CONTINUE_AS_NEW_INITIATOR_WORKFLOW',
|
|
1334
|
+
...(a.header ? { header: a.header } : {}),
|
|
1335
|
+
...(next.run.memo ? { memo: next.run.memo } : {}),
|
|
1336
|
+
...(next.run.searchAttributes ? { searchAttributes: next.run.searchAttributes } : {}),
|
|
1337
|
+
});
|
|
1338
|
+
return { closed: { status: 'WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW', eventId, newRun: next, continued: true } };
|
|
1339
|
+
}
|
|
1340
|
+
case 'COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION': {
|
|
1341
|
+
const a = (command.startChildWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1342
|
+
const childNamespace = String(a.namespace || run.namespace);
|
|
1343
|
+
const workflowId = String(a.workflowId ?? '');
|
|
1344
|
+
if (!workflowId || !(a.workflowType as Obj | undefined)?.name) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_START_CHILD_EXECUTION_ATTRIBUTES', 'WorkflowId and WorkflowType are required');
|
|
1345
|
+
if (a.cronSchedule) throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED', 'this Temporal twin does not run cron schedules (CronSchedule on a child)');
|
|
1346
|
+
try {
|
|
1347
|
+
this.namespace(childNamespace);
|
|
1348
|
+
this.validateSearchAttributes(childNamespace, a.searchAttributes, false);
|
|
1349
|
+
} catch (error) {
|
|
1350
|
+
throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_BAD_START_CHILD_EXECUTION_ATTRIBUTES', (error as Error).message);
|
|
1351
|
+
}
|
|
1352
|
+
const parentClosePolicy = String(a.parentClosePolicy && a.parentClosePolicy !== 'PARENT_CLOSE_POLICY_UNSPECIFIED' ? a.parentClosePolicy : 'PARENT_CLOSE_POLICY_TERMINATE');
|
|
1353
|
+
const taskQueue = String((a.taskQueue as Obj | undefined)?.name || run.taskQueue);
|
|
1354
|
+
const initiatedEventId = tx.append('EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED', 'startChildWorkflowExecutionInitiatedEventAttributes', {
|
|
1355
|
+
...a, namespace: childNamespace, namespaceId: String((this.namespace(childNamespace).namespaceInfo as Obj).id), taskQueue: { name: taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
1356
|
+
parentClosePolicy, workflowTaskCompletedEventId: completedEventId,
|
|
1357
|
+
});
|
|
1358
|
+
state.children[String(initiatedEventId)] = { initiatedEventId, namespace: childNamespace, workflowId, workflowType: a.workflowType as Obj, parentClosePolicy };
|
|
1359
|
+
after.push(() => this.startChild(run.id, initiatedEventId, { ...a, taskQueue: { name: taskQueue } }, childNamespace, completedEventId));
|
|
1360
|
+
return undefined;
|
|
1361
|
+
}
|
|
1362
|
+
case 'COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION': {
|
|
1363
|
+
const a = (command.signalExternalWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1364
|
+
const targetNamespace = String(a.namespace || run.namespace);
|
|
1365
|
+
const initiatedEventId = tx.append('EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED', 'signalExternalWorkflowExecutionInitiatedEventAttributes', {
|
|
1366
|
+
namespace: targetNamespace, workflowExecution: a.execution, signalName: a.signalName, ...(a.input ? { input: a.input } : {}),
|
|
1367
|
+
...(a.control ? { control: a.control } : {}), childWorkflowOnly: Boolean(a.childWorkflowOnly), ...(a.header ? { header: a.header } : {}),
|
|
1368
|
+
workflowTaskCompletedEventId: completedEventId,
|
|
1369
|
+
});
|
|
1370
|
+
after.push(() => this.signalExternal(run.id, initiatedEventId, targetNamespace, a, completedEventId));
|
|
1371
|
+
return undefined;
|
|
1372
|
+
}
|
|
1373
|
+
case 'COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION': {
|
|
1374
|
+
const a = (command.requestCancelExternalWorkflowExecutionCommandAttributes ?? {}) as Obj;
|
|
1375
|
+
const targetNamespace = String(a.namespace || run.namespace);
|
|
1376
|
+
const execution = { workflowId: String(a.workflowId ?? ''), ...(a.runId ? { runId: String(a.runId) } : {}) };
|
|
1377
|
+
const initiatedEventId = tx.append('EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED', 'requestCancelExternalWorkflowExecutionInitiatedEventAttributes', {
|
|
1378
|
+
namespace: targetNamespace, workflowExecution: execution, ...(a.control ? { control: a.control } : {}), childWorkflowOnly: Boolean(a.childWorkflowOnly),
|
|
1379
|
+
...(a.reason ? { reason: a.reason } : {}), workflowTaskCompletedEventId: completedEventId,
|
|
1380
|
+
});
|
|
1381
|
+
after.push(() => this.cancelExternal(run.id, initiatedEventId, targetNamespace, execution, a, completedEventId));
|
|
1382
|
+
return undefined;
|
|
1383
|
+
}
|
|
1384
|
+
default:
|
|
1385
|
+
throw new UnhandledCommand('WORKFLOW_TASK_FAILED_CAUSE_FEATURE_DISABLED', `this Temporal twin does not implement ${type || 'an unspecified command'}`);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
/** The run a continue-as-new or a retry hands the workflow to (built now, written after this run). */
|
|
1390
|
+
private nextRunOf(tx: RunTx, next: { backoffMs: number; attempt: number; initiator: string; continuedFailure?: Obj; workflowType?: Obj; taskQueue?: string; input?: Obj; header?: Obj; runTimeoutMs?: number; workflowTaskTimeoutMs?: number; memo?: Obj; searchAttributes?: Obj; retryPolicy?: RetryPolicy; lastCompletionResult?: Obj }): RunTx {
|
|
1391
|
+
const run = tx.run;
|
|
1392
|
+
const state = tx.state;
|
|
1393
|
+
const input = next.initiator === 'CONTINUE_AS_NEW_INITIATOR_RETRY' ? state.input : next.input;
|
|
1394
|
+
const header = next.header ?? state.header;
|
|
1395
|
+
const memo = next.memo ?? run.memo;
|
|
1396
|
+
const searchAttributes = next.searchAttributes ?? run.searchAttributes;
|
|
1397
|
+
const retryPolicy = next.retryPolicy ?? state.retryPolicy;
|
|
1398
|
+
return this.newRun(tx.now, {
|
|
1399
|
+
namespace: run.namespace,
|
|
1400
|
+
workflowId: run.execution.workflowId,
|
|
1401
|
+
workflowType: next.workflowType ?? run.type,
|
|
1402
|
+
taskQueue: next.taskQueue ?? run.taskQueue,
|
|
1403
|
+
...(input ? { input } : {}),
|
|
1404
|
+
...(header ? { header } : {}),
|
|
1405
|
+
...(memo ? { memo } : {}),
|
|
1406
|
+
...(searchAttributes ? { searchAttributes } : {}),
|
|
1407
|
+
executionTimeoutMs: state.executionTimeoutMs,
|
|
1408
|
+
runTimeoutMs: next.runTimeoutMs ?? state.runTimeoutMs,
|
|
1409
|
+
workflowTaskTimeoutMs: next.workflowTaskTimeoutMs ?? state.workflowTaskTimeoutMs,
|
|
1410
|
+
...(retryPolicy ? { retryPolicy } : {}),
|
|
1411
|
+
attempt: next.attempt,
|
|
1412
|
+
...(state.parent ? { parent: state.parent } : {}),
|
|
1413
|
+
...(run.rootExecution ? { rootExecution: run.rootExecution } : {}),
|
|
1414
|
+
firstRunId: run.firstRunId,
|
|
1415
|
+
continuedExecutionRunId: run.id,
|
|
1416
|
+
initiator: next.initiator,
|
|
1417
|
+
...(next.continuedFailure ? { continuedFailure: next.continuedFailure } : {}),
|
|
1418
|
+
...(next.lastCompletionResult ? { lastCompletionResult: next.lastCompletionResult } : {}),
|
|
1419
|
+
backoffMs: next.backoffMs,
|
|
1420
|
+
...(state.executionExpiresAt ? { executionExpiresAt: state.executionExpiresAt } : {}),
|
|
1421
|
+
seed: `continue\0${run.id}`,
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
private async startChild(parentRunId: string, initiatedEventId: number, a: Obj, namespace: string, completedEventId: number): Promise<void> {
|
|
1426
|
+
const parent = this.run(parentRunId);
|
|
1427
|
+
if (!parent) return;
|
|
1428
|
+
// a parent that closed in the same task (a close command after the start) starts no child, whatever
|
|
1429
|
+
// its parent-close policy: the server's transfer task refuses a start for a closed parent
|
|
1430
|
+
// (history service, processStartChildExecution), and the parent's history ends at its close event
|
|
1431
|
+
if (parent.status !== RUNNING) return;
|
|
1432
|
+
const parentOpen = true;
|
|
1433
|
+
const now = nowMs();
|
|
1434
|
+
const workflowId = String(a.workflowId);
|
|
1435
|
+
const current = latest(this.runs(namespace).filter((run) => run.execution.workflowId === workflowId));
|
|
1436
|
+
const reuse = String(a.workflowIdReusePolicy ?? 'WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE');
|
|
1437
|
+
const refused = current && (current.status === RUNNING
|
|
1438
|
+
|| reuse === 'WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE'
|
|
1439
|
+
|| (reuse === 'WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE_FAILED_ONLY' && current.status === 'WORKFLOW_EXECUTION_STATUS_COMPLETED'));
|
|
1440
|
+
const parentTx = new RunTx(parent, now);
|
|
1441
|
+
if (refused) {
|
|
1442
|
+
if (!parentOpen) return;
|
|
1443
|
+
delete parentTx.state.children[String(initiatedEventId)];
|
|
1444
|
+
parentTx.deliver('EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED', 'startChildWorkflowExecutionFailedEventAttributes', {
|
|
1445
|
+
namespace, namespaceId: String((this.namespace(namespace).namespaceInfo as Obj).id), workflowId, workflowType: a.workflowType,
|
|
1446
|
+
cause: 'START_CHILD_WORKFLOW_EXECUTION_FAILED_CAUSE_WORKFLOW_ALREADY_EXISTS', ...(a.control ? { control: a.control } : {}),
|
|
1447
|
+
initiatedEventId, workflowTaskCompletedEventId: completedEventId,
|
|
1448
|
+
});
|
|
1449
|
+
await this.commit(parentTx, 'StartChildWorkflowExecution');
|
|
1450
|
+
return;
|
|
1451
|
+
}
|
|
1452
|
+
const executionTimeoutMs = durationMs(a.workflowExecutionTimeout);
|
|
1453
|
+
let runTimeoutMs = durationMs(a.workflowRunTimeout);
|
|
1454
|
+
if (executionTimeoutMs && (!runTimeoutMs || runTimeoutMs > executionTimeoutMs)) runTimeoutMs = executionTimeoutMs;
|
|
1455
|
+
const childTx = this.newRun(now, {
|
|
1456
|
+
namespace, workflowId, workflowType: a.workflowType as Obj, taskQueue: String((a.taskQueue as Obj).name),
|
|
1457
|
+
...(a.input ? { input: a.input as Obj } : {}), ...(a.header ? { header: a.header as Obj } : {}), ...(a.memo ? { memo: a.memo as Obj } : {}),
|
|
1458
|
+
...(a.searchAttributes ? { searchAttributes: a.searchAttributes as Obj } : {}), ...(a.retryPolicy ? { retryPolicy: a.retryPolicy as RetryPolicy } : {}),
|
|
1459
|
+
...(a.priority ? { priority: a.priority as Obj } : {}),
|
|
1460
|
+
executionTimeoutMs, runTimeoutMs,
|
|
1461
|
+
workflowTaskTimeoutMs: Math.min(durationMs(a.workflowTaskTimeout) || DEFAULT_WFT_TIMEOUT_MS, MAX_WFT_TIMEOUT_MS, runTimeoutMs || MAX_WFT_TIMEOUT_MS),
|
|
1462
|
+
parent: { namespace: parent.namespace, workflowId: parent.execution.workflowId, runId: parent.id, initiatedEventId },
|
|
1463
|
+
rootExecution: parent.rootExecution ?? parent.execution,
|
|
1464
|
+
seed: `child\0${parent.id}\0${initiatedEventId}`,
|
|
1465
|
+
});
|
|
1466
|
+
await this.commit(childTx, 'StartChildWorkflowExecution');
|
|
1467
|
+
if (!parentOpen) return;
|
|
1468
|
+
const startedAttrs = {
|
|
1469
|
+
namespace, namespaceId: String((this.namespace(namespace).namespaceInfo as Obj).id), initiatedEventId,
|
|
1470
|
+
workflowExecution: childTx.run.execution, workflowType: childTx.run.type, ...(a.header ? { header: a.header } : {}),
|
|
1471
|
+
};
|
|
1472
|
+
const pending = parentTx.state.children[String(initiatedEventId)];
|
|
1473
|
+
if (pending) pending.runId = childTx.run.id;
|
|
1474
|
+
// the started event's id lands on the pending child when the event itself lands (RunTx.land)
|
|
1475
|
+
parentTx.deliver('EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED', 'childWorkflowExecutionStartedEventAttributes', startedAttrs);
|
|
1476
|
+
await this.commit(parentTx, 'StartChildWorkflowExecution');
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
private async signalExternal(runId: string, initiatedEventId: number, namespace: string, a: Obj, completedEventId: number): Promise<void> {
|
|
1480
|
+
const source = this.run(runId);
|
|
1481
|
+
// a source that closed in the same task (a close command after this one) sends nothing: its
|
|
1482
|
+
// history ends at the close event, and the server drops the pending transfer
|
|
1483
|
+
if (!source || source.status !== RUNNING) return;
|
|
1484
|
+
const now = nowMs();
|
|
1485
|
+
const execution = (a.execution ?? {}) as { workflowId?: string; runId?: string };
|
|
1486
|
+
let target: Run | undefined;
|
|
1487
|
+
try {
|
|
1488
|
+
target = this.resolveRun(namespace, execution);
|
|
1489
|
+
} catch { target = undefined; }
|
|
1490
|
+
if (a.childWorkflowOnly && target && target._state.parent?.runId !== source.id) target = undefined;
|
|
1491
|
+
const sourceTx = new RunTx(source, now);
|
|
1492
|
+
if (!target || target.status !== RUNNING) {
|
|
1493
|
+
sourceTx.deliver('EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED', 'signalExternalWorkflowExecutionFailedEventAttributes', {
|
|
1494
|
+
cause: 'SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND', namespace,
|
|
1495
|
+
namespaceId: String((this.namespace(namespace).namespaceInfo as Obj).id), workflowExecution: execution, initiatedEventId,
|
|
1496
|
+
...(a.control ? { control: a.control } : {}), workflowTaskCompletedEventId: completedEventId,
|
|
1497
|
+
});
|
|
1498
|
+
await this.commit(sourceTx, 'SignalExternalWorkflowExecution');
|
|
1499
|
+
return;
|
|
1500
|
+
}
|
|
1501
|
+
const targetTx = new RunTx(target, now);
|
|
1502
|
+
this.signalInto(targetTx, { signalName: String(a.signalName), input: a.input as Obj | undefined, header: a.header as Obj | undefined, identity: '', external: source.execution });
|
|
1503
|
+
await this.commit(targetTx, 'SignalExternalWorkflowExecution');
|
|
1504
|
+
// the source re-read after the target's write: a workflow that signals itself is both
|
|
1505
|
+
const signaledTx = target.id === source.id ? new RunTx(this.run(source.id)!, now) : sourceTx;
|
|
1506
|
+
signaledTx.deliver('EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_SIGNALED', 'externalWorkflowExecutionSignaledEventAttributes', {
|
|
1507
|
+
initiatedEventId, namespace, namespaceId: String((this.namespace(namespace).namespaceInfo as Obj).id), workflowExecution: target.execution, ...(a.control ? { control: a.control } : {}),
|
|
1508
|
+
});
|
|
1509
|
+
await this.commit(signaledTx, 'SignalExternalWorkflowExecution');
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
private async cancelExternal(runId: string, initiatedEventId: number, namespace: string, execution: { workflowId: string; runId?: string }, a: Obj, completedEventId: number): Promise<void> {
|
|
1513
|
+
const source = this.run(runId);
|
|
1514
|
+
// a source that closed in the same task (a close command after this one) sends nothing: its
|
|
1515
|
+
// history ends at the close event, and the server drops the pending transfer
|
|
1516
|
+
if (!source || source.status !== RUNNING) return;
|
|
1517
|
+
const now = nowMs();
|
|
1518
|
+
let target: Run | undefined;
|
|
1519
|
+
try {
|
|
1520
|
+
target = this.resolveRun(namespace, execution);
|
|
1521
|
+
} catch { target = undefined; }
|
|
1522
|
+
if (a.childWorkflowOnly && target && target._state.parent?.runId !== source.id) target = undefined;
|
|
1523
|
+
const sourceTx = new RunTx(source, now);
|
|
1524
|
+
if (!target || target.status !== RUNNING) {
|
|
1525
|
+
sourceTx.deliver('EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED', 'requestCancelExternalWorkflowExecutionFailedEventAttributes', {
|
|
1526
|
+
cause: 'CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND', namespace,
|
|
1527
|
+
namespaceId: String((this.namespace(namespace).namespaceInfo as Obj).id), workflowExecution: execution, initiatedEventId,
|
|
1528
|
+
...(a.control ? { control: a.control } : {}), workflowTaskCompletedEventId: completedEventId,
|
|
1529
|
+
});
|
|
1530
|
+
await this.commit(sourceTx, 'RequestCancelExternalWorkflowExecution');
|
|
1531
|
+
return;
|
|
1532
|
+
}
|
|
1533
|
+
if (!target._state.cancelRequested) {
|
|
1534
|
+
const targetTx = new RunTx(target, now);
|
|
1535
|
+
this.cancelRequestInto(targetTx, { ...(a.reason ? { cause: String(a.reason) } : {}), externalInitiatedEventId: initiatedEventId, externalWorkflowExecution: source.execution });
|
|
1536
|
+
await this.commit(targetTx, 'RequestCancelExternalWorkflowExecution');
|
|
1537
|
+
}
|
|
1538
|
+
// the source re-read after the target's write: a workflow that cancels itself is both
|
|
1539
|
+
const requestedTx = target.id === source.id ? new RunTx(this.run(source.id)!, now) : sourceTx;
|
|
1540
|
+
requestedTx.deliver('EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_CANCEL_REQUESTED', 'externalWorkflowExecutionCancelRequestedEventAttributes', {
|
|
1541
|
+
initiatedEventId, namespace, namespaceId: String((this.namespace(namespace).namespaceInfo as Obj).id), workflowExecution: target.execution,
|
|
1542
|
+
});
|
|
1543
|
+
await this.commit(requestedTx, 'RequestCancelExternalWorkflowExecution');
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/** Fail the in-flight workflow task and schedule the next attempt. */
|
|
1547
|
+
private async failWorkflowTask(tx: RunTx, cause: string, message: string, identity: string, options: { sameAttempt?: boolean; failure?: Obj; operation?: string } = {}): Promise<void> {
|
|
1548
|
+
const wft = tx.state.wft!;
|
|
1549
|
+
tx.append('EVENT_TYPE_WORKFLOW_TASK_FAILED', 'workflowTaskFailedEventAttributes', {
|
|
1550
|
+
scheduledEventId: wft.scheduledEventId, startedEventId: wft.startedEventId, cause,
|
|
1551
|
+
failure: options.failure ?? { message, source: 'temporal-twin', serverFailureInfo: { nonRetryable: false } },
|
|
1552
|
+
identity,
|
|
1553
|
+
});
|
|
1554
|
+
tx.state.wft = null;
|
|
1555
|
+
tx.flushBuffered();
|
|
1556
|
+
const attempt = options.sameAttempt ? 1 : wft.attempt + 1;
|
|
1557
|
+
// a task that keeps failing is retried with a growing delay, as the server backs off its retries
|
|
1558
|
+
tx.scheduleWorkflowTask(attempt, attempt <= 2 ? 0 : Math.min(1000 * 2 ** (attempt - 3), 10 * 60_000));
|
|
1559
|
+
await this.commit(tx, options.operation ?? 'RespondWorkflowTaskFailed');
|
|
1560
|
+
this.notify();
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
async respondWorkflowTaskFailed(request: Obj): Promise<Obj> {
|
|
1564
|
+
this.assertWritable();
|
|
1565
|
+
const token = decodeToken(request.taskToken, 'TaskToken');
|
|
1566
|
+
if (token.kind !== 'workflow') throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'invalid TaskToken');
|
|
1567
|
+
return this.serial(async () => {
|
|
1568
|
+
const run = this.workflowTaskOf(token);
|
|
1569
|
+
const tx = new RunTx(run, nowMs());
|
|
1570
|
+
await this.failWorkflowTask(tx, String(request.cause ?? 'WORKFLOW_TASK_FAILED_CAUSE_UNSPECIFIED'), '', String(request.identity ?? ''), request.failure ? { failure: request.failure as Obj } : {});
|
|
1571
|
+
return {};
|
|
1572
|
+
});
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
// ---- queries -------------------------------------------------------------------------------------
|
|
1576
|
+
|
|
1577
|
+
async queryWorkflow(request: Obj, signal: AbortSignal, deadline?: number): Promise<Obj> {
|
|
1578
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
1579
|
+
const run = this.resolveRun(namespace, request.execution as Obj);
|
|
1580
|
+
const reject = String(request.queryRejectCondition ?? '');
|
|
1581
|
+
if (reject === 'QUERY_REJECT_CONDITION_NOT_OPEN' && run.status !== RUNNING) return { queryRejected: { status: run.status } };
|
|
1582
|
+
if (reject === 'QUERY_REJECT_CONDITION_NOT_COMPLETED_CLEANLY' && run.status !== RUNNING && run.status !== 'WORKFLOW_EXECUTION_STATUS_COMPLETED') return { queryRejected: { status: run.status } };
|
|
1583
|
+
if (run._state.previousStartedEventId === 0) {
|
|
1584
|
+
throw new GrpcError(GrpcStatus.FAILED_PRECONDITION, 'Cannot query workflow before the first workflow task completes.');
|
|
1585
|
+
}
|
|
1586
|
+
this.querySeq += 1;
|
|
1587
|
+
const id = `q${this.querySeq}`;
|
|
1588
|
+
const answer = new Promise<Obj>((resolve, rejectQuery) => {
|
|
1589
|
+
this.queries.set(id, { id, namespace, runId: run.id, taskQueue: run.taskQueue, query: request.query as Obj, resolve, reject: rejectQuery, dispatched: false });
|
|
1590
|
+
});
|
|
1591
|
+
this.notify();
|
|
1592
|
+
const until = Math.min(Date.now() + LONG_POLL_MS, (deadline ?? Infinity) - 500);
|
|
1593
|
+
const timeout = new Promise<never>((_, rejectTimeout) => {
|
|
1594
|
+
const timer = setTimeout(() => rejectTimeout(new GrpcError(GrpcStatus.DEADLINE_EXCEEDED, 'query timed out before a worker answered it')), Math.max(0, until - Date.now()));
|
|
1595
|
+
signal.addEventListener('abort', () => { clearTimeout(timer); rejectTimeout(new GrpcError(GrpcStatus.CANCELLED, 'query canceled')); }, { once: true });
|
|
1596
|
+
void answer.finally(() => clearTimeout(timer)).catch(() => undefined);
|
|
1597
|
+
});
|
|
1598
|
+
try {
|
|
1599
|
+
return await Promise.race([answer, timeout]);
|
|
1600
|
+
} finally {
|
|
1601
|
+
this.queries.delete(id);
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
|
|
1605
|
+
async respondQueryTaskCompleted(request: Obj): Promise<Obj> {
|
|
1606
|
+
const token = decodeToken(request.taskToken, 'TaskToken');
|
|
1607
|
+
const query = token.kind === 'query' ? this.queries.get(String(token.id)) : undefined;
|
|
1608
|
+
if (!query) throw new GrpcError(GrpcStatus.NOT_FOUND, 'query task not found, or already expired');
|
|
1609
|
+
this.queries.delete(query.id);
|
|
1610
|
+
if (request.completedType === 'QUERY_RESULT_TYPE_ANSWERED') query.resolve({ queryResult: request.queryResult ?? { payloads: [] } });
|
|
1611
|
+
else query.reject(new GrpcError(GrpcStatus.INVALID_ARGUMENT, String(request.errorMessage || (request.failure as Obj | undefined)?.message || 'query failed')));
|
|
1612
|
+
return {};
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
// ---- activity tasks ------------------------------------------------------------------------------
|
|
1616
|
+
|
|
1617
|
+
async pollActivityTaskQueue(request: Obj, signal: AbortSignal, deadline?: number): Promise<Obj> {
|
|
1618
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
1619
|
+
const name = String((request.taskQueue as Obj | undefined)?.name ?? '');
|
|
1620
|
+
if (!name) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'TaskQueue is not set on request.');
|
|
1621
|
+
const identity = String(request.identity ?? '');
|
|
1622
|
+
const until = Math.min(Date.now() + LONG_POLL_MS, (deadline ?? Infinity) - 1000);
|
|
1623
|
+
while (!signal.aborted) {
|
|
1624
|
+
const task = await this.serial(() => this.takeActivityTask(namespace, name, identity));
|
|
1625
|
+
if (task) return task;
|
|
1626
|
+
if (Date.now() >= until) return {};
|
|
1627
|
+
await this.wait(signal, until);
|
|
1628
|
+
}
|
|
1629
|
+
return {};
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
private async takeActivityTask(namespace: string, queue: string, identity: string): Promise<Obj | undefined> {
|
|
1633
|
+
const now = nowMs();
|
|
1634
|
+
let best: { run: Run; activity: ActivityState } | undefined;
|
|
1635
|
+
for (const run of this.runs(namespace)) {
|
|
1636
|
+
if (run.status !== RUNNING) continue;
|
|
1637
|
+
for (const activity of Object.values(run._state.activities)) {
|
|
1638
|
+
if (activity.taskQueue !== queue || activity.startedAt !== undefined || activity.availableAt > now) continue;
|
|
1639
|
+
if (!best || activity.availableAt < best.activity.availableAt) best = { run, activity };
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
if (!best) return undefined;
|
|
1643
|
+
const tx = new RunTx(best.run, now);
|
|
1644
|
+
const activity = tx.state.activities[String(best.activity.scheduledEventId)]!;
|
|
1645
|
+
activity.startedAt = now;
|
|
1646
|
+
activity.identity = identity;
|
|
1647
|
+
activity.requestId = uuidFrom(`activity\0${best.run.id}\0${activity.scheduledEventId}\0${activity.attempt}`);
|
|
1648
|
+
activity.lastHeartbeatAt = undefined;
|
|
1649
|
+
await this.commit(tx, 'PollActivityTaskQueue');
|
|
1650
|
+
return {
|
|
1651
|
+
taskToken: encodeToken({ kind: 'activity', runId: best.run.id, scheduledEventId: activity.scheduledEventId, attempt: activity.attempt }),
|
|
1652
|
+
workflowNamespace: namespace,
|
|
1653
|
+
workflowType: best.run.type,
|
|
1654
|
+
workflowExecution: best.run.execution,
|
|
1655
|
+
activityType: activity.activityType,
|
|
1656
|
+
activityId: activity.activityId,
|
|
1657
|
+
...(activity.header ? { header: activity.header } : {}),
|
|
1658
|
+
...(activity.input ? { input: activity.input } : {}),
|
|
1659
|
+
...(activity.heartbeatDetails ? { heartbeatDetails: activity.heartbeatDetails } : {}),
|
|
1660
|
+
scheduledTime: toTimestamp(activity.scheduledAt),
|
|
1661
|
+
currentAttemptScheduledTime: toTimestamp(activity.attemptScheduledAt),
|
|
1662
|
+
startedTime: toTimestamp(now),
|
|
1663
|
+
attempt: activity.attempt,
|
|
1664
|
+
scheduleToCloseTimeout: toDuration(activity.scheduleToCloseMs) ?? {},
|
|
1665
|
+
startToCloseTimeout: toDuration(activity.startToCloseMs) ?? {},
|
|
1666
|
+
heartbeatTimeout: toDuration(activity.heartbeatMs) ?? {},
|
|
1667
|
+
retryPolicy: activity.retryPolicy,
|
|
1668
|
+
...(activity.priority ? { priority: activity.priority } : {}),
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/** The run and started attempt an activity token (or an id triple) names, or NOT_FOUND. */
|
|
1673
|
+
private activityOf(target: { runId?: string; scheduledEventId?: number; attempt?: number; namespace?: string; workflowId?: string; activityId?: string }): { run: Run; activity: ActivityState } {
|
|
1674
|
+
let run: Run | undefined;
|
|
1675
|
+
let activity: ActivityState | undefined;
|
|
1676
|
+
if (target.activityId !== undefined) {
|
|
1677
|
+
try {
|
|
1678
|
+
run = this.resolveRun(String(target.namespace), { workflowId: target.workflowId, ...(target.runId ? { runId: target.runId } : {}) });
|
|
1679
|
+
} catch { run = undefined; }
|
|
1680
|
+
activity = run ? Object.values(run._state.activities).find((candidate) => candidate.activityId === target.activityId) : undefined;
|
|
1681
|
+
} else {
|
|
1682
|
+
run = this.run(String(target.runId ?? ''));
|
|
1683
|
+
activity = run?._state.activities[String(target.scheduledEventId)];
|
|
1684
|
+
if (activity && target.attempt !== undefined && activity.attempt !== target.attempt) activity = undefined;
|
|
1685
|
+
}
|
|
1686
|
+
if (!run || run.status !== RUNNING) throw new GrpcError(GrpcStatus.NOT_FOUND, 'workflow execution already completed');
|
|
1687
|
+
if (!activity || activity.startedAt === undefined) throw new GrpcError(GrpcStatus.NOT_FOUND, 'activity task not found.');
|
|
1688
|
+
return { run, activity };
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
private activityTarget(request: Obj, byId: boolean): Parameters<TemporalEngine['activityOf']>[0] {
|
|
1692
|
+
if (byId) {
|
|
1693
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
1694
|
+
return { namespace, workflowId: String(request.workflowId ?? ''), ...(request.runId ? { runId: String(request.runId) } : {}), activityId: String(request.activityId ?? '') };
|
|
1695
|
+
}
|
|
1696
|
+
const token = decodeToken(request.taskToken, 'TaskToken');
|
|
1697
|
+
if (token.kind !== 'activity') throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'invalid TaskToken');
|
|
1698
|
+
return { runId: String(token.runId), scheduledEventId: Number(token.scheduledEventId), attempt: Number(token.attempt) };
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
/** Close an activity with its final event (Started is written now, as Temporal writes it at the close). */
|
|
1702
|
+
private closeActivity(tx: RunTx, activity: ActivityState, eventType: string, key: string, attributes: Obj): void {
|
|
1703
|
+
const started = activity.startedAt !== undefined;
|
|
1704
|
+
if (started) {
|
|
1705
|
+
tx.deliver('EVENT_TYPE_ACTIVITY_TASK_STARTED', 'activityTaskStartedEventAttributes', {
|
|
1706
|
+
scheduledEventId: activity.scheduledEventId, identity: activity.identity ?? '', requestId: activity.requestId ?? '', attempt: activity.attempt,
|
|
1707
|
+
...(activity.lastFailure ? { lastFailure: activity.lastFailure } : {}),
|
|
1708
|
+
}, { wakes: false });
|
|
1709
|
+
}
|
|
1710
|
+
tx.deliver(eventType, key, { scheduledEventId: activity.scheduledEventId, ...attributes, ...(started ? {} : { startedEventId: 0 }) }, started ? { linkPrevious: 'startedEventId' } : {});
|
|
1711
|
+
delete tx.state.activities[String(activity.scheduledEventId)];
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
async respondActivityTaskCompleted(request: Obj, byId = false): Promise<Obj> {
|
|
1715
|
+
this.assertWritable();
|
|
1716
|
+
const target = this.activityTarget(request, byId);
|
|
1717
|
+
return this.serial(async () => {
|
|
1718
|
+
const { run, activity } = this.activityOf(target);
|
|
1719
|
+
const tx = new RunTx(run, nowMs());
|
|
1720
|
+
this.closeActivity(tx, tx.state.activities[String(activity.scheduledEventId)]!, 'EVENT_TYPE_ACTIVITY_TASK_COMPLETED', 'activityTaskCompletedEventAttributes', {
|
|
1721
|
+
...(request.result ? { result: request.result } : {}), identity: String(request.identity ?? ''),
|
|
1722
|
+
});
|
|
1723
|
+
await this.commit(tx, byId ? 'RespondActivityTaskCompletedById' : 'RespondActivityTaskCompleted');
|
|
1724
|
+
this.notify();
|
|
1725
|
+
return {};
|
|
1726
|
+
});
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
async respondActivityTaskFailed(request: Obj, byId = false): Promise<Obj> {
|
|
1730
|
+
this.assertWritable();
|
|
1731
|
+
const target = this.activityTarget(request, byId);
|
|
1732
|
+
return this.serial(async () => {
|
|
1733
|
+
const { run, activity: found } = this.activityOf(target);
|
|
1734
|
+
const now = nowMs();
|
|
1735
|
+
const tx = new RunTx(run, now);
|
|
1736
|
+
const activity = tx.state.activities[String(found.scheduledEventId)]!;
|
|
1737
|
+
const failure = (request.failure ?? { message: 'activity failed' }) as Obj;
|
|
1738
|
+
if (request.lastHeartbeatDetails) activity.heartbeatDetails = request.lastHeartbeatDetails as Obj;
|
|
1739
|
+
const deadline = activity.scheduleToCloseMs ? activity.scheduledAt + activity.scheduleToCloseMs : undefined;
|
|
1740
|
+
const decision = activity.cancelRequestedEventId !== undefined ? { retry: false as const, state: 'RETRY_STATE_CANCEL_REQUESTED' } : decideRetry(activity.retryPolicy, activity.attempt, failure, now, deadline);
|
|
1741
|
+
if (decision.retry) {
|
|
1742
|
+
this.retryActivity(activity, failure, now, decision.delayMs);
|
|
1743
|
+
tx.wakeQueues = true;
|
|
1744
|
+
} else {
|
|
1745
|
+
this.closeActivity(tx, activity, 'EVENT_TYPE_ACTIVITY_TASK_FAILED', 'activityTaskFailedEventAttributes', { failure, retryState: decision.state, identity: String(request.identity ?? '') });
|
|
1746
|
+
}
|
|
1747
|
+
await this.commit(tx, byId ? 'RespondActivityTaskFailedById' : 'RespondActivityTaskFailed');
|
|
1748
|
+
this.notify();
|
|
1749
|
+
return {};
|
|
1750
|
+
});
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
private retryActivity(activity: ActivityState, failure: Obj, now: number, delayMs: number): void {
|
|
1754
|
+
activity.attempt += 1;
|
|
1755
|
+
activity.lastFailure = failure;
|
|
1756
|
+
activity.lastAttemptCompletedAt = now;
|
|
1757
|
+
activity.startedAt = undefined;
|
|
1758
|
+
activity.identity = undefined;
|
|
1759
|
+
activity.requestId = undefined;
|
|
1760
|
+
activity.lastHeartbeatAt = undefined;
|
|
1761
|
+
activity.attemptScheduledAt = now + delayMs;
|
|
1762
|
+
activity.availableAt = now + delayMs;
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
async respondActivityTaskCanceled(request: Obj, byId = false): Promise<Obj> {
|
|
1766
|
+
this.assertWritable();
|
|
1767
|
+
const target = this.activityTarget(request, byId);
|
|
1768
|
+
return this.serial(async () => {
|
|
1769
|
+
const { run, activity } = this.activityOf(target);
|
|
1770
|
+
if (activity.cancelRequestedEventId === undefined) {
|
|
1771
|
+
throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'Activity task cannot be canceled: no cancellation was requested.');
|
|
1772
|
+
}
|
|
1773
|
+
const tx = new RunTx(run, nowMs());
|
|
1774
|
+
this.closeActivity(tx, tx.state.activities[String(activity.scheduledEventId)]!, 'EVENT_TYPE_ACTIVITY_TASK_CANCELED', 'activityTaskCanceledEventAttributes', {
|
|
1775
|
+
...(request.details ? { details: request.details } : {}), latestCancelRequestedEventId: activity.cancelRequestedEventId, identity: String(request.identity ?? ''),
|
|
1776
|
+
});
|
|
1777
|
+
await this.commit(tx, byId ? 'RespondActivityTaskCanceledById' : 'RespondActivityTaskCanceled');
|
|
1778
|
+
this.notify();
|
|
1779
|
+
return {};
|
|
1780
|
+
});
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
async recordActivityTaskHeartbeat(request: Obj, byId = false): Promise<Obj> {
|
|
1784
|
+
this.assertWritable();
|
|
1785
|
+
const target = this.activityTarget(request, byId);
|
|
1786
|
+
return this.serial(async () => {
|
|
1787
|
+
const { run, activity } = this.activityOf(target);
|
|
1788
|
+
const tx = new RunTx(run, nowMs());
|
|
1789
|
+
const held = tx.state.activities[String(activity.scheduledEventId)]!;
|
|
1790
|
+
held.lastHeartbeatAt = tx.now;
|
|
1791
|
+
if (request.details) held.heartbeatDetails = request.details as Obj;
|
|
1792
|
+
await this.commit(tx, byId ? 'RecordActivityTaskHeartbeatById' : 'RecordActivityTaskHeartbeat');
|
|
1793
|
+
return { cancelRequested: held.cancelRequestedEventId !== undefined, activityPaused: false, activityReset: false };
|
|
1794
|
+
});
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
// ---- the clock -----------------------------------------------------------------------------------
|
|
1798
|
+
|
|
1799
|
+
/** Fire what the World clock says is due: timers, timeouts, delayed first tasks, expirations. */
|
|
1800
|
+
async tick(): Promise<void> {
|
|
1801
|
+
if (this.ticking || this.readOnly) return;
|
|
1802
|
+
this.ticking = true;
|
|
1803
|
+
try {
|
|
1804
|
+
const now = nowMs();
|
|
1805
|
+
const due = this.runs().filter((run) => run.status === RUNNING && run._state.nextDue !== undefined && run._state.nextDue <= now);
|
|
1806
|
+
let woke = false;
|
|
1807
|
+
for (const candidate of due) {
|
|
1808
|
+
await this.serial(async () => {
|
|
1809
|
+
const run = this.run(candidate.id);
|
|
1810
|
+
if (!run || run.status !== RUNNING) return;
|
|
1811
|
+
try {
|
|
1812
|
+
if (await this.fireDue(run, nowMs())) woke = true;
|
|
1813
|
+
} catch (error) {
|
|
1814
|
+
process.stderr.write(`temporal: world-clock work for run ${run.id} failed: ${(error as Error)?.message ?? String(error)}\n`);
|
|
1815
|
+
}
|
|
1816
|
+
});
|
|
1817
|
+
}
|
|
1818
|
+
if (woke) this.notify();
|
|
1819
|
+
} finally {
|
|
1820
|
+
this.ticking = false;
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
private async fireDue(run: Run, now: number): Promise<boolean> {
|
|
1825
|
+
const tx = new RunTx(run, now);
|
|
1826
|
+
const state = tx.state;
|
|
1827
|
+
// the run's own deadlines close it
|
|
1828
|
+
const expiry = [state.runExpiresAt, state.executionExpiresAt].filter((value): value is number => value !== undefined && value <= now);
|
|
1829
|
+
if (expiry.length > 0) {
|
|
1830
|
+
if (state.wft?.startedEventId !== undefined) {
|
|
1831
|
+
state.wft = null;
|
|
1832
|
+
}
|
|
1833
|
+
state.wft = null;
|
|
1834
|
+
tx.flushBuffered();
|
|
1835
|
+
const decision = state.executionExpiresAt !== undefined && state.executionExpiresAt <= now
|
|
1836
|
+
? { retry: false as const, state: 'RETRY_STATE_TIMEOUT' }
|
|
1837
|
+
: decideRetry(state.retryPolicy, state.attempt, timeoutFailure('workflow', 'TIMEOUT_TYPE_START_TO_CLOSE'), now, state.executionExpiresAt);
|
|
1838
|
+
if (decision.retry) {
|
|
1839
|
+
const next = this.nextRunOf(tx, { backoffMs: decision.delayMs, attempt: state.attempt + 1, initiator: 'CONTINUE_AS_NEW_INITIATOR_RETRY' });
|
|
1840
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT', 'workflowExecutionTimedOutEventAttributes', { retryState: 'RETRY_STATE_IN_PROGRESS', newExecutionRunId: next.run.id });
|
|
1841
|
+
await this.closeRun(tx, 'WORKFLOW_EXECUTION_STATUS_TIMED_OUT', eventId, 'WorkflowExecutionTimedOut', undefined, { newRunId: next.run.id });
|
|
1842
|
+
await this.commit(next, 'WorkflowExecutionTimedOut');
|
|
1843
|
+
} else {
|
|
1844
|
+
const eventId = tx.append('EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT', 'workflowExecutionTimedOutEventAttributes', { retryState: decision.state });
|
|
1845
|
+
await this.closeRun(tx, 'WORKFLOW_EXECUTION_STATUS_TIMED_OUT', eventId, 'WorkflowExecutionTimedOut');
|
|
1846
|
+
}
|
|
1847
|
+
return true;
|
|
1848
|
+
}
|
|
1849
|
+
let woke = false;
|
|
1850
|
+
if (state.firstWftAt !== undefined && state.firstWftAt <= now) {
|
|
1851
|
+
state.firstWftAt = undefined;
|
|
1852
|
+
tx.scheduleWorkflowTask();
|
|
1853
|
+
woke = true;
|
|
1854
|
+
}
|
|
1855
|
+
// an in-flight workflow task past its start-to-close timeout
|
|
1856
|
+
if (state.wft?.timeoutAt !== undefined && state.wft.timeoutAt <= now) {
|
|
1857
|
+
const wft = state.wft;
|
|
1858
|
+
tx.append('EVENT_TYPE_WORKFLOW_TASK_TIMED_OUT', 'workflowTaskTimedOutEventAttributes', { scheduledEventId: wft.scheduledEventId, startedEventId: wft.startedEventId, timeoutType: 'TIMEOUT_TYPE_START_TO_CLOSE' });
|
|
1859
|
+
state.wft = null;
|
|
1860
|
+
tx.flushBuffered();
|
|
1861
|
+
tx.scheduleWorkflowTask(wft.attempt + 1);
|
|
1862
|
+
woke = true;
|
|
1863
|
+
}
|
|
1864
|
+
if (state.wft && state.wft.startedEventId === undefined && state.wft.availableAt <= now && state.wft.availableAt > (state.wft.scheduledAt)) woke = true;
|
|
1865
|
+
for (const [timerId, timer] of Object.entries(state.timers)) {
|
|
1866
|
+
if (timer.fireAt > now) continue;
|
|
1867
|
+
delete state.timers[timerId];
|
|
1868
|
+
tx.deliver('EVENT_TYPE_TIMER_FIRED', 'timerFiredEventAttributes', { timerId, startedEventId: timer.startedEventId });
|
|
1869
|
+
woke = true;
|
|
1870
|
+
}
|
|
1871
|
+
for (const activity of Object.values(state.activities)) {
|
|
1872
|
+
const timedOut = activityTimeout(activity, now);
|
|
1873
|
+
if (!timedOut) {
|
|
1874
|
+
if (activity.startedAt === undefined && activity.availableAt <= now && activity.availableAt > activity.scheduledAt) woke = true;
|
|
1875
|
+
continue;
|
|
1876
|
+
}
|
|
1877
|
+
const failure = timeoutFailure('activity', timedOut, activity.heartbeatDetails);
|
|
1878
|
+
const deadline = activity.scheduleToCloseMs ? activity.scheduledAt + activity.scheduleToCloseMs : undefined;
|
|
1879
|
+
const decision = activity.cancelRequestedEventId !== undefined ? { retry: false as const, state: 'RETRY_STATE_CANCEL_REQUESTED' } : decideRetry(activity.retryPolicy, activity.attempt, failure, now, deadline);
|
|
1880
|
+
if (decision.retry) {
|
|
1881
|
+
this.retryActivity(activity, failure, now, decision.delayMs);
|
|
1882
|
+
} else {
|
|
1883
|
+
this.closeActivity(tx, activity, 'EVENT_TYPE_ACTIVITY_TASK_TIMED_OUT', 'activityTaskTimedOutEventAttributes', { failure, retryState: decision.state });
|
|
1884
|
+
}
|
|
1885
|
+
woke = true;
|
|
1886
|
+
}
|
|
1887
|
+
if (tx.events.length > 0 || woke) await this.commit(tx, 'WorldClock');
|
|
1888
|
+
return woke;
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// ---- reading runs back ---------------------------------------------------------------------------
|
|
1892
|
+
|
|
1893
|
+
async getWorkflowExecutionHistory(request: Obj, signal: AbortSignal, deadline?: number): Promise<Obj> {
|
|
1894
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
1895
|
+
const token = request.nextPageToken ? decodeToken(request.nextPageToken, 'NextPageToken') : undefined;
|
|
1896
|
+
const execution = request.execution as Obj | undefined;
|
|
1897
|
+
let run = token?.runId ? this.run(String(token.runId)) : this.resolveRun(namespace, execution);
|
|
1898
|
+
if (run && run.namespace !== namespace) run = undefined;
|
|
1899
|
+
if (!run) throw new GrpcError(GrpcStatus.NOT_FOUND, 'workflow not found');
|
|
1900
|
+
const closeOnly = request.historyEventFilterType === 'HISTORY_EVENT_FILTER_TYPE_CLOSE_EVENT';
|
|
1901
|
+
const wait = Boolean(request.waitNewEvent);
|
|
1902
|
+
const until = Math.min(Date.now() + HISTORY_LONG_POLL_MS, (deadline ?? Infinity) - 500);
|
|
1903
|
+
const pageSize = Number(request.maximumPageSize) > 0 ? Number(request.maximumPageSize) : DEFAULT_PAGE_SIZE;
|
|
1904
|
+
const from = Number(token?.from ?? 1);
|
|
1905
|
+
for (;;) {
|
|
1906
|
+
run = this.run(run.id)!;
|
|
1907
|
+
if (closeOnly) {
|
|
1908
|
+
if (run.status !== RUNNING) {
|
|
1909
|
+
const events = await this.history(run);
|
|
1910
|
+
const close = events.find((event) => event.eventId === run!._state.closeEventId);
|
|
1911
|
+
return { history: { events: close ? [close] : [] } };
|
|
1912
|
+
}
|
|
1913
|
+
} else {
|
|
1914
|
+
const events = await this.history(run);
|
|
1915
|
+
if (events.length >= from || !wait || run.status !== RUNNING) {
|
|
1916
|
+
const page = events.slice(from - 1, from - 1 + pageSize);
|
|
1917
|
+
const last = page.length ? page[page.length - 1]!.eventId : from - 1;
|
|
1918
|
+
const more = last < events.length || (wait && run.status === RUNNING);
|
|
1919
|
+
return { history: { events: page }, ...(more ? { nextPageToken: encodeToken({ runId: run.id, from: last + 1 }) } : {}) };
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1922
|
+
if (!wait || signal.aborted || Date.now() >= until) {
|
|
1923
|
+
return { history: { events: [] }, ...(closeOnly ? {} : { nextPageToken: encodeToken({ runId: run.id, from }) }) };
|
|
1924
|
+
}
|
|
1925
|
+
await this.wait(signal, until);
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
async describeWorkflowExecution(request: Obj): Promise<Obj> {
|
|
1930
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
1931
|
+
const run = this.resolveRun(namespace, request.execution as Obj);
|
|
1932
|
+
const state = run._state;
|
|
1933
|
+
const pendingActivities = Object.values(state.activities).map((activity) => ({
|
|
1934
|
+
activityId: activity.activityId,
|
|
1935
|
+
activityType: activity.activityType,
|
|
1936
|
+
state: activity.cancelRequestedEventId !== undefined ? 'PENDING_ACTIVITY_STATE_CANCEL_REQUESTED' : activity.startedAt !== undefined ? 'PENDING_ACTIVITY_STATE_STARTED' : 'PENDING_ACTIVITY_STATE_SCHEDULED',
|
|
1937
|
+
...(activity.heartbeatDetails ? { heartbeatDetails: activity.heartbeatDetails } : {}),
|
|
1938
|
+
...(activity.lastHeartbeatAt ? { lastHeartbeatTime: toTimestamp(activity.lastHeartbeatAt) } : {}),
|
|
1939
|
+
...(activity.startedAt ? { lastStartedTime: toTimestamp(activity.startedAt) } : {}),
|
|
1940
|
+
attempt: activity.attempt,
|
|
1941
|
+
maximumAttempts: activity.retryPolicy.maximumAttempts ?? 0,
|
|
1942
|
+
scheduledTime: toTimestamp(activity.attemptScheduledAt),
|
|
1943
|
+
...(activity.scheduleToCloseMs ? { expirationTime: toTimestamp(activity.scheduledAt + activity.scheduleToCloseMs) } : {}),
|
|
1944
|
+
...(activity.lastFailure ? { lastFailure: activity.lastFailure } : {}),
|
|
1945
|
+
...(activity.identity ? { lastWorkerIdentity: activity.identity } : {}),
|
|
1946
|
+
...(activity.lastAttemptCompletedAt ? { lastAttemptCompleteTime: toTimestamp(activity.lastAttemptCompletedAt) } : {}),
|
|
1947
|
+
...(activity.startedAt === undefined && activity.attempt > 1 ? { nextAttemptScheduleTime: toTimestamp(activity.availableAt) } : {}),
|
|
1948
|
+
}));
|
|
1949
|
+
const pendingChildren = Object.values(state.children).map((child) => ({
|
|
1950
|
+
workflowId: child.workflowId, ...(child.runId ? { runId: child.runId } : {}), workflowTypeName: String(child.workflowType.name),
|
|
1951
|
+
initiatedId: child.initiatedEventId, parentClosePolicy: child.parentClosePolicy,
|
|
1952
|
+
}));
|
|
1953
|
+
const wft = state.wft;
|
|
1954
|
+
return {
|
|
1955
|
+
executionConfig: {
|
|
1956
|
+
taskQueue: { name: run.taskQueue, kind: 'TASK_QUEUE_KIND_NORMAL' },
|
|
1957
|
+
workflowExecutionTimeout: toDuration(state.executionTimeoutMs),
|
|
1958
|
+
workflowRunTimeout: toDuration(state.runTimeoutMs),
|
|
1959
|
+
defaultWorkflowTaskTimeout: toDuration(state.workflowTaskTimeoutMs),
|
|
1960
|
+
},
|
|
1961
|
+
workflowExecutionInfo: executionInfo(run),
|
|
1962
|
+
pendingActivities,
|
|
1963
|
+
pendingChildren,
|
|
1964
|
+
...(wft ? {
|
|
1965
|
+
pendingWorkflowTask: {
|
|
1966
|
+
state: wft.startedEventId !== undefined ? 'PENDING_WORKFLOW_TASK_STATE_STARTED' : 'PENDING_WORKFLOW_TASK_STATE_SCHEDULED',
|
|
1967
|
+
scheduledTime: toTimestamp(wft.scheduledAt), originalScheduledTime: toTimestamp(wft.scheduledAt),
|
|
1968
|
+
...(wft.startedAt ? { startedTime: toTimestamp(wft.startedAt) } : {}), attempt: wft.attempt,
|
|
1969
|
+
},
|
|
1970
|
+
} : {}),
|
|
1971
|
+
};
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
private visibility(namespace: string, query: string): Run[] {
|
|
1975
|
+
let parsed;
|
|
1976
|
+
try {
|
|
1977
|
+
parsed = parseVisibilityQuery(query);
|
|
1978
|
+
} catch (error) {
|
|
1979
|
+
if (error instanceof VisibilityQueryError) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `invalid query: ${error.message}`);
|
|
1980
|
+
throw error;
|
|
1981
|
+
}
|
|
1982
|
+
const custom = this.customSearchAttributes(namespace);
|
|
1983
|
+
const fieldOf = (name: string): string => {
|
|
1984
|
+
if (SYSTEM_SEARCH_ATTRIBUTES[name] || custom[name] !== undefined) return name;
|
|
1985
|
+
throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `invalid query: unable to convert filter expression: unable to convert left side of "${name}": column name '${name}' is not a valid search attribute`);
|
|
1986
|
+
};
|
|
1987
|
+
for (const field of visibilityFields(parsed)) fieldOf(field);
|
|
1988
|
+
for (const order of parsed.orderBy) {
|
|
1989
|
+
if (!['StartTime', 'CloseTime', 'ExecutionTime'].includes(order.field)) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, `invalid query: ORDER BY ${order.field} is not supported by this server`);
|
|
1990
|
+
}
|
|
1991
|
+
const runs = this.runs(namespace).filter((run) => matchesVisibilityQuery(parsed, visibilityRow(run), fieldOf));
|
|
1992
|
+
const orders = parsed.orderBy.length ? parsed.orderBy : [{ field: 'StartTime', desc: true }];
|
|
1993
|
+
return runs.sort((a, b) => {
|
|
1994
|
+
for (const order of orders) {
|
|
1995
|
+
const x = Number(visibilityRow(a)[order.field] ?? 0);
|
|
1996
|
+
const y = Number(visibilityRow(b)[order.field] ?? 0);
|
|
1997
|
+
if (x !== y) return order.desc ? y - x : x - y;
|
|
1998
|
+
}
|
|
1999
|
+
return a.id < b.id ? 1 : -1;
|
|
2000
|
+
});
|
|
2001
|
+
}
|
|
2002
|
+
|
|
2003
|
+
listWorkflowExecutions(request: Obj): Obj {
|
|
2004
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
2005
|
+
const matching = this.visibility(namespace, String(request.query ?? ''));
|
|
2006
|
+
const offset = request.nextPageToken ? Number(decodeToken(request.nextPageToken, 'NextPageToken').offset ?? 0) : 0;
|
|
2007
|
+
const pageSize = Number(request.pageSize) > 0 ? Number(request.pageSize) : DEFAULT_PAGE_SIZE;
|
|
2008
|
+
const page = matching.slice(offset, offset + pageSize);
|
|
2009
|
+
return {
|
|
2010
|
+
executions: page.map(executionInfo),
|
|
2011
|
+
...(offset + pageSize < matching.length ? { nextPageToken: encodeToken({ offset: offset + pageSize }) } : {}),
|
|
2012
|
+
};
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
countWorkflowExecutions(request: Obj): Obj {
|
|
2016
|
+
const namespace = this.requireNamespace(request.namespace);
|
|
2017
|
+
const query = String(request.query ?? '');
|
|
2018
|
+
if (/\bGROUP\s+BY\b/i.test(query)) throw new GrpcError(GrpcStatus.INVALID_ARGUMENT, 'invalid query: GROUP BY is not supported by this server');
|
|
2019
|
+
return { count: this.visibility(namespace, query).length };
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
async shutdownWorker(request: Obj): Promise<Obj> {
|
|
2023
|
+
this.requireNamespace(request.namespace);
|
|
2024
|
+
const sticky = String(request.stickyTaskQueue ?? '');
|
|
2025
|
+
if (sticky) this.closedSticky.add(sticky);
|
|
2026
|
+
this.notify();
|
|
2027
|
+
return {};
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
|
|
2031
|
+
// cache: one engine per World root in a process — its lock is what serializes a root's transitions,
|
|
2032
|
+
// so the gRPC wire, the HTTP fetch and anything else here mounted on one root share it
|
|
2033
|
+
const engines = new Map<string, TemporalEngine>(); // cache: an engine holds no truth — the tree does
|
|
2034
|
+
|
|
2035
|
+
/** The process's engine for a root (created on first use). Two engines on one root would each
|
|
2036
|
+
* serialize only their own transitions and could lose one another's updates. */
|
|
2037
|
+
export function engineForRoot(options: EngineOptions = {}): TemporalEngine {
|
|
2038
|
+
// no root: a caller of its own (a test, a scratch fetch), never pooled with another rootless one
|
|
2039
|
+
if (options.root === undefined) return new TemporalEngine(options);
|
|
2040
|
+
const key = `${options.root}\0${options.readOnly ? 'ro' : 'rw'}`;
|
|
2041
|
+
let engine = engines.get(key);
|
|
2042
|
+
if (!engine) {
|
|
2043
|
+
engine = new TemporalEngine(options);
|
|
2044
|
+
engines.set(key, engine);
|
|
2045
|
+
}
|
|
2046
|
+
engine.users += 1;
|
|
2047
|
+
return engine;
|
|
2048
|
+
}
|
|
2049
|
+
|
|
2050
|
+
/** A user of a root's engine is done with it; the last one stops it and drops it from the registry. */
|
|
2051
|
+
export function releaseEngine(engine: TemporalEngine): void {
|
|
2052
|
+
engine.users -= 1;
|
|
2053
|
+
if (engine.users > 0) return;
|
|
2054
|
+
engine.stop();
|
|
2055
|
+
for (const [key, held] of engines) if (held === engine) engines.delete(key);
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
const CLOSING_COMMANDS = new Set([
|
|
2059
|
+
'COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION',
|
|
2060
|
+
'COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION',
|
|
2061
|
+
'COMMAND_TYPE_CANCEL_WORKFLOW_EXECUTION',
|
|
2062
|
+
'COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION',
|
|
2063
|
+
]);
|
|
2064
|
+
|
|
2065
|
+
function latest(runs: Run[]): Run | undefined {
|
|
2066
|
+
// the current run of a workflow id is the one created last (a chain's newest link); creation order,
|
|
2067
|
+
// not start time, since a pinned World clock gives every run the same start time
|
|
2068
|
+
return [...runs].sort((a, b) => b._state.seq - a._state.seq)[0];
|
|
2069
|
+
}
|
|
2070
|
+
|
|
2071
|
+
function mergeFields(base: Record<string, Obj> | undefined, updates: Record<string, Obj> | undefined): Record<string, Obj> {
|
|
2072
|
+
const out: Record<string, Obj> = { ...(base ?? {}) };
|
|
2073
|
+
for (const [key, payload] of Object.entries(updates ?? {})) {
|
|
2074
|
+
const value = payloadValue(payload);
|
|
2075
|
+
const empty = value === null || (Array.isArray(value) && value.length === 0) || (payload as { data?: string }).data === undefined;
|
|
2076
|
+
if (empty) delete out[key];
|
|
2077
|
+
else out[key] = payload;
|
|
2078
|
+
}
|
|
2079
|
+
return out;
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
function activityTimeout(activity: ActivityState, now: number): string | undefined {
|
|
2083
|
+
if (activity.scheduleToCloseMs && now >= activity.scheduledAt + activity.scheduleToCloseMs) return 'TIMEOUT_TYPE_SCHEDULE_TO_CLOSE';
|
|
2084
|
+
if (activity.startedAt === undefined) {
|
|
2085
|
+
if (activity.availableAt > now) return undefined;
|
|
2086
|
+
if (activity.scheduleToStartMs && now >= activity.attemptScheduledAt + activity.scheduleToStartMs) return 'TIMEOUT_TYPE_SCHEDULE_TO_START';
|
|
2087
|
+
return undefined;
|
|
2088
|
+
}
|
|
2089
|
+
if (activity.startToCloseMs && now >= activity.startedAt + activity.startToCloseMs) return 'TIMEOUT_TYPE_START_TO_CLOSE';
|
|
2090
|
+
if (activity.heartbeatMs && now >= (activity.lastHeartbeatAt ?? activity.startedAt) + activity.heartbeatMs) return 'TIMEOUT_TYPE_HEARTBEAT';
|
|
2091
|
+
return undefined;
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
/** The earliest moment still ahead at which the World clock has something to do for this run. A task
|
|
2095
|
+
* whose backoff has passed is the pollers' to take, not the clock's, so it is not counted once due:
|
|
2096
|
+
* counting it would re-enter the run on every tick (and write it) until a worker came. */
|
|
2097
|
+
function nextDue(run: Run, now: number): number | undefined {
|
|
2098
|
+
if (run.status !== RUNNING) return undefined;
|
|
2099
|
+
const state = run._state;
|
|
2100
|
+
const times: number[] = [];
|
|
2101
|
+
if (state.runExpiresAt !== undefined) times.push(state.runExpiresAt);
|
|
2102
|
+
if (state.executionExpiresAt !== undefined) times.push(state.executionExpiresAt);
|
|
2103
|
+
if (state.firstWftAt !== undefined) times.push(state.firstWftAt);
|
|
2104
|
+
if (state.wft?.timeoutAt !== undefined) times.push(state.wft.timeoutAt);
|
|
2105
|
+
if (state.wft && state.wft.startedEventId === undefined && state.wft.availableAt > now) times.push(state.wft.availableAt);
|
|
2106
|
+
for (const timer of Object.values(state.timers)) times.push(timer.fireAt);
|
|
2107
|
+
for (const activity of Object.values(state.activities)) {
|
|
2108
|
+
if (activity.scheduleToCloseMs) times.push(activity.scheduledAt + activity.scheduleToCloseMs);
|
|
2109
|
+
if (activity.startedAt === undefined) {
|
|
2110
|
+
if (activity.availableAt > now) times.push(activity.availableAt);
|
|
2111
|
+
if (activity.scheduleToStartMs) times.push(activity.attemptScheduledAt + activity.scheduleToStartMs);
|
|
2112
|
+
} else {
|
|
2113
|
+
if (activity.startToCloseMs) times.push(activity.startedAt + activity.startToCloseMs);
|
|
2114
|
+
if (activity.heartbeatMs) times.push((activity.lastHeartbeatAt ?? activity.startedAt) + activity.heartbeatMs);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
return times.length ? Math.min(...times) : undefined;
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
/** A run's WorkflowExecutionInfo: its stored fields without the server's bookkeeping. */
|
|
2121
|
+
export function executionInfo(run: Run): Obj {
|
|
2122
|
+
const { id: _id, namespace: _namespace, _state, ...info } = run;
|
|
2123
|
+
return info;
|
|
2124
|
+
}
|
|
2125
|
+
|
|
2126
|
+
function visibilityRow(run: Run): VisibilityRow {
|
|
2127
|
+
const row: VisibilityRow = {
|
|
2128
|
+
WorkflowId: run.execution.workflowId,
|
|
2129
|
+
RunId: run.execution.runId,
|
|
2130
|
+
WorkflowType: run.type.name,
|
|
2131
|
+
ExecutionStatus: STATUS_NAME[run.status] ?? run.status,
|
|
2132
|
+
TaskQueue: run.taskQueue,
|
|
2133
|
+
StartTime: timestampMs(run.startTime) ?? null,
|
|
2134
|
+
ExecutionTime: timestampMs(run.executionTime) ?? null,
|
|
2135
|
+
CloseTime: timestampMs(run.closeTime) ?? null,
|
|
2136
|
+
HistoryLength: run.historyLength,
|
|
2137
|
+
HistorySizeBytes: Number(run.historySizeBytes ?? 0),
|
|
2138
|
+
StateTransitionCount: Number(run.stateTransitionCount ?? 0),
|
|
2139
|
+
ExecutionDuration: run.executionDuration ? durationMs(run.executionDuration) * 1e6 : null,
|
|
2140
|
+
ParentWorkflowId: run.parentExecution?.workflowId ?? null,
|
|
2141
|
+
ParentRunId: run.parentExecution?.runId ?? null,
|
|
2142
|
+
RootWorkflowId: run.rootExecution?.workflowId ?? null,
|
|
2143
|
+
RootRunId: run.rootExecution?.runId ?? null,
|
|
2144
|
+
};
|
|
2145
|
+
for (const [key, payload] of Object.entries(run.searchAttributes?.indexedFields ?? {})) {
|
|
2146
|
+
const value = payloadValue(payload);
|
|
2147
|
+
if (value === undefined) continue;
|
|
2148
|
+
row[key] = Array.isArray(value) ? value.map(String) : typeof value === 'object' && value !== null ? JSON.stringify(value) : (value as string | number | boolean | null);
|
|
2149
|
+
}
|
|
2150
|
+
return row;
|
|
2151
|
+
}
|
|
2152
|
+
|
|
2153
|
+
/** A stored subject as the wire answers it: `_`-prefixed bookkeeping never leaves the twin. */
|
|
2154
|
+
function publicView(record: Obj): Obj {
|
|
2155
|
+
return Object.fromEntries(Object.entries(record).filter(([key]) => !key.startsWith('_')));
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
function namespaceCapabilities(): Obj {
|
|
2159
|
+
return { eagerWorkflowStart: false, syncUpdate: false, asyncUpdate: false };
|
|
2160
|
+
}
|
|
2161
|
+
|
|
2162
|
+
function defaultNamespace(): Obj {
|
|
2163
|
+
return {
|
|
2164
|
+
namespaceInfo: {
|
|
2165
|
+
name: DEFAULT_NAMESPACE, state: 'NAMESPACE_STATE_REGISTERED', id: uuidFrom(`namespace\0${DEFAULT_NAMESPACE}`),
|
|
2166
|
+
description: 'Default namespace for Temporal Server.', capabilities: namespaceCapabilities(), supportsSchedules: false,
|
|
2167
|
+
},
|
|
2168
|
+
config: {
|
|
2169
|
+
workflowExecutionRetentionTtl: { seconds: 86_400 },
|
|
2170
|
+
historyArchivalState: 'ARCHIVAL_STATE_DISABLED',
|
|
2171
|
+
visibilityArchivalState: 'ARCHIVAL_STATE_DISABLED',
|
|
2172
|
+
},
|
|
2173
|
+
replicationConfig: { activeClusterName: 'active', clusters: [{ clusterName: 'active' }], state: 'REPLICATION_STATE_NORMAL' },
|
|
2174
|
+
isGlobalNamespace: false,
|
|
2175
|
+
};
|
|
2176
|
+
}
|