deepline 0.1.177 → 0.1.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +1 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +36 -140
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/coordinator-progress.ts +289 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +10 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +74 -30
- package/dist/cli/index.js +4 -3
- package/dist/cli/index.mjs +4 -3
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -2281,6 +2281,7 @@ function runRequestFromPlayWorkflowParams(
|
|
|
2281
2281
|
inlineChildRunRegistered:
|
|
2282
2282
|
params.runtimeBackend === 'cf_workflows_dynamic_worker_inline_child',
|
|
2283
2283
|
coordinatorUrl: params.coordinatorUrl ?? null,
|
|
2284
|
+
coordinatorInternalToken: params.coordinatorInternalToken ?? null,
|
|
2284
2285
|
totalRows: params.totalRows,
|
|
2285
2286
|
};
|
|
2286
2287
|
}
|
|
@@ -181,6 +181,10 @@ import {
|
|
|
181
181
|
setHarnessBinding,
|
|
182
182
|
} from '../../../sdk/src/plays/harness-stub';
|
|
183
183
|
import { createHarnessWorkerReceiptStore } from './runtime/harness-receipt-store';
|
|
184
|
+
import {
|
|
185
|
+
createCoordinatorProgressPublisher,
|
|
186
|
+
isCoordinatorProgressAuthorizationError,
|
|
187
|
+
} from './runtime/coordinator-progress';
|
|
184
188
|
import {
|
|
185
189
|
hydrateSerializedResultDatasets,
|
|
186
190
|
projectTerminalResultDatasets,
|
|
@@ -1735,7 +1739,9 @@ function isRuntimeReceiptPersistenceError(error: unknown): boolean {
|
|
|
1735
1739
|
|
|
1736
1740
|
function isRunFatalWorkerRowError(error: unknown): boolean {
|
|
1737
1741
|
return (
|
|
1738
|
-
|
|
1742
|
+
isCoordinatorProgressAuthorizationError(error) ||
|
|
1743
|
+
isRowIsolationExemptError(error) ||
|
|
1744
|
+
isRuntimeReceiptPersistenceError(error)
|
|
1739
1745
|
);
|
|
1740
1746
|
}
|
|
1741
1747
|
|
|
@@ -1858,9 +1864,7 @@ class WorkerToolBatchScheduler {
|
|
|
1858
1864
|
id,
|
|
1859
1865
|
cacheKey: receiptKey,
|
|
1860
1866
|
receiptKey,
|
|
1861
|
-
force:
|
|
1862
|
-
options?.force === true ||
|
|
1863
|
-
!!this.req.force,
|
|
1867
|
+
force: options?.force === true || !!this.req.force,
|
|
1864
1868
|
receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
|
|
1865
1869
|
typeof runtimeOptions?.receiptWaitMs === 'number'
|
|
1866
1870
|
? { max_wait_ms: runtimeOptions.receiptWaitMs }
|
|
@@ -4147,10 +4151,15 @@ async function prepareMapRows(input: {
|
|
|
4147
4151
|
|
|
4148
4152
|
function assertNotAborted(signal: AbortSignal | undefined): void {
|
|
4149
4153
|
if (signal?.aborted) {
|
|
4154
|
+
if (isCoordinatorProgressAuthorizationError(signal.reason)) {
|
|
4155
|
+
throw signal.reason;
|
|
4156
|
+
}
|
|
4150
4157
|
throw new WorkflowAbortError(
|
|
4151
|
-
|
|
4152
|
-
? signal.reason
|
|
4153
|
-
: '
|
|
4158
|
+
signal.reason instanceof Error && signal.reason.message
|
|
4159
|
+
? signal.reason.message
|
|
4160
|
+
: typeof signal.reason === 'string' && signal.reason
|
|
4161
|
+
? signal.reason
|
|
4162
|
+
: 'Play run cancelled.',
|
|
4154
4163
|
);
|
|
4155
4164
|
}
|
|
4156
4165
|
}
|
|
@@ -5370,9 +5379,7 @@ function createMinimalWorkerCtx(
|
|
|
5370
5379
|
request.input,
|
|
5371
5380
|
workflowStep,
|
|
5372
5381
|
{
|
|
5373
|
-
force:
|
|
5374
|
-
request.force === true ||
|
|
5375
|
-
!!req.force,
|
|
5382
|
+
force: request.force === true || !!req.force,
|
|
5376
5383
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
5377
5384
|
},
|
|
5378
5385
|
);
|
|
@@ -5872,7 +5879,7 @@ function createMinimalWorkerCtx(
|
|
|
5872
5879
|
? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
|
|
5873
5880
|
: '';
|
|
5874
5881
|
callbacks?.onMapCompleted?.(mapNodeId, completedAt);
|
|
5875
|
-
|
|
5882
|
+
await updateMapProgress({
|
|
5876
5883
|
completed: totalRowsWritten,
|
|
5877
5884
|
total: totalRowsWritten + totalRowsFailed,
|
|
5878
5885
|
failed: totalRowsFailed,
|
|
@@ -6360,9 +6367,7 @@ function createMinimalWorkerCtx(
|
|
|
6360
6367
|
request.input,
|
|
6361
6368
|
workflowStep,
|
|
6362
6369
|
{
|
|
6363
|
-
force:
|
|
6364
|
-
request.force === true ||
|
|
6365
|
-
!!req.force,
|
|
6370
|
+
force: request.force === true || !!req.force,
|
|
6366
6371
|
staleAfterSeconds: request.staleAfterSeconds,
|
|
6367
6372
|
},
|
|
6368
6373
|
{
|
|
@@ -7196,9 +7201,6 @@ async function executeRunRequest(
|
|
|
7196
7201
|
let lastLedgerFlushAt = startedAt;
|
|
7197
7202
|
let ledgerFlushInFlight: Promise<void> = Promise.resolve();
|
|
7198
7203
|
let ledgerFlushQueueDepth = 0;
|
|
7199
|
-
let lastCoordinatorProgressPublishAt = 0;
|
|
7200
|
-
let coordinatorProgressPublishInFlight: Promise<void> = Promise.resolve();
|
|
7201
|
-
let coordinatorProgressPublishQueueDepth = 0;
|
|
7202
7204
|
|
|
7203
7205
|
const appendRunLogLine = (line: string) => {
|
|
7204
7206
|
const trimmed = redactSecretsFromLogString(line.trim());
|
|
@@ -7225,130 +7227,24 @@ async function executeRunRequest(
|
|
|
7225
7227
|
};
|
|
7226
7228
|
|
|
7227
7229
|
const stepProgressSnapshot = () => ({ ...stepProgressByNodeId });
|
|
7228
|
-
|
|
7229
|
-
|
|
7230
|
-
|
|
7231
|
-
|
|
7232
|
-
|
|
7233
|
-
|
|
7234
|
-
|
|
7235
|
-
|
|
7236
|
-
|
|
7237
|
-
|
|
7238
|
-
|
|
7239
|
-
});
|
|
7240
|
-
return;
|
|
7241
|
-
}
|
|
7242
|
-
const publishStartedAt = nowMs();
|
|
7243
|
-
const liveNodeProgress = stepProgressSnapshot();
|
|
7244
|
-
const activeEntry =
|
|
7245
|
-
Object.entries(liveNodeProgress).find(
|
|
7246
|
-
([, progress]) => typeof progress.completedAt !== 'number',
|
|
7247
|
-
) ?? Object.entries(liveNodeProgress).at(-1);
|
|
7248
|
-
const activeNodeId = activeEntry?.[0] ?? null;
|
|
7249
|
-
const activeProgress = activeEntry?.[1] ?? null;
|
|
7250
|
-
const activeArtifactTableNamespace =
|
|
7251
|
-
typeof activeProgress?.artifactTableNamespace === 'string'
|
|
7252
|
-
? activeProgress.artifactTableNamespace
|
|
7253
|
-
: null;
|
|
7254
|
-
const activeCompleted =
|
|
7255
|
-
typeof activeProgress?.completed === 'number'
|
|
7256
|
-
? activeProgress.completed
|
|
7257
|
-
: null;
|
|
7258
|
-
const activeTotal =
|
|
7259
|
-
typeof activeProgress?.total === 'number' ? activeProgress.total : null;
|
|
7260
|
-
const activeMessage =
|
|
7261
|
-
typeof activeProgress?.message === 'string'
|
|
7262
|
-
? activeProgress.message
|
|
7263
|
-
: null;
|
|
7264
|
-
const response = await fetch(
|
|
7265
|
-
`${coordinatorUrl.replace(/\/$/, '')}/dedup/${encodeURIComponent(
|
|
7266
|
-
req.runId,
|
|
7267
|
-
)}/event-add`,
|
|
7268
|
-
{
|
|
7269
|
-
method: 'POST',
|
|
7270
|
-
headers: {
|
|
7271
|
-
'x-deepline-request-id': makeRequestId(),
|
|
7272
|
-
...coordinatorRequestHeaders({
|
|
7273
|
-
runId: req.runId,
|
|
7274
|
-
contentType: 'application/json',
|
|
7275
|
-
internalToken: req.coordinatorInternalToken,
|
|
7276
|
-
}),
|
|
7277
|
-
},
|
|
7278
|
-
body: JSON.stringify({
|
|
7279
|
-
runId: req.runId,
|
|
7280
|
-
type: 'progress',
|
|
7281
|
-
status: 'running',
|
|
7282
|
-
ts: occurredAt,
|
|
7283
|
-
logs: runLogBuffer,
|
|
7284
|
-
activeNodeId,
|
|
7285
|
-
activeArtifactTableNamespace,
|
|
7286
|
-
updatedAt: occurredAt,
|
|
7287
|
-
liveNodeProgress,
|
|
7288
|
-
}),
|
|
7289
|
-
},
|
|
7290
|
-
);
|
|
7291
|
-
if (!response.ok) {
|
|
7230
|
+
const coordinatorProgressPublisher = createCoordinatorProgressPublisher({
|
|
7231
|
+
runId: req.runId,
|
|
7232
|
+
coordinatorUrl: req.coordinatorUrl,
|
|
7233
|
+
coordinatorInternalToken: req.coordinatorInternalToken,
|
|
7234
|
+
abortController,
|
|
7235
|
+
getRunLogBuffer: () => runLogBuffer,
|
|
7236
|
+
getLiveNodeProgress: stepProgressSnapshot,
|
|
7237
|
+
makeRequestId,
|
|
7238
|
+
nowMs,
|
|
7239
|
+
heartbeatIntervalMs: MAP_EXECUTION_HEARTBEAT_INTERVAL_MS,
|
|
7240
|
+
recordTrace: (trace) =>
|
|
7292
7241
|
recordRunnerPerfTrace({
|
|
7293
7242
|
req,
|
|
7294
|
-
phase:
|
|
7295
|
-
ms:
|
|
7296
|
-
extra:
|
|
7297
|
-
|
|
7298
|
-
|
|
7299
|
-
activeNodeId,
|
|
7300
|
-
activeArtifactTableNamespace,
|
|
7301
|
-
activeCompleted,
|
|
7302
|
-
activeTotal,
|
|
7303
|
-
activeMessage,
|
|
7304
|
-
},
|
|
7305
|
-
});
|
|
7306
|
-
throw new Error(`coordinator progress event failed ${response.status}`);
|
|
7307
|
-
}
|
|
7308
|
-
recordRunnerPerfTrace({
|
|
7309
|
-
req,
|
|
7310
|
-
phase: 'runner.coordinator_progress_publish',
|
|
7311
|
-
ms: nowMs() - publishStartedAt,
|
|
7312
|
-
extra: {
|
|
7313
|
-
status: 'ok',
|
|
7314
|
-
activeNodeId,
|
|
7315
|
-
activeArtifactTableNamespace,
|
|
7316
|
-
activeCompleted,
|
|
7317
|
-
activeTotal,
|
|
7318
|
-
activeMessage,
|
|
7319
|
-
},
|
|
7320
|
-
});
|
|
7321
|
-
};
|
|
7322
|
-
|
|
7323
|
-
const flushCoordinatorProgressEvent = (force: boolean): Promise<void> => {
|
|
7324
|
-
const now = nowMs();
|
|
7325
|
-
if (
|
|
7326
|
-
!force &&
|
|
7327
|
-
now - lastCoordinatorProgressPublishAt <
|
|
7328
|
-
MAP_EXECUTION_HEARTBEAT_INTERVAL_MS
|
|
7329
|
-
) {
|
|
7330
|
-
return Promise.resolve();
|
|
7331
|
-
}
|
|
7332
|
-
if (!force && coordinatorProgressPublishQueueDepth > 0) {
|
|
7333
|
-
return Promise.resolve();
|
|
7334
|
-
}
|
|
7335
|
-
lastCoordinatorProgressPublishAt = now;
|
|
7336
|
-
coordinatorProgressPublishQueueDepth += 1;
|
|
7337
|
-
coordinatorProgressPublishInFlight = coordinatorProgressPublishInFlight
|
|
7338
|
-
.catch(() => undefined)
|
|
7339
|
-
.then(async () => {
|
|
7340
|
-
try {
|
|
7341
|
-
await publishCoordinatorProgressEvent(now);
|
|
7342
|
-
} finally {
|
|
7343
|
-
coordinatorProgressPublishQueueDepth = Math.max(
|
|
7344
|
-
0,
|
|
7345
|
-
coordinatorProgressPublishQueueDepth - 1,
|
|
7346
|
-
);
|
|
7347
|
-
}
|
|
7348
|
-
})
|
|
7349
|
-
.catch(() => undefined);
|
|
7350
|
-
return force ? coordinatorProgressPublishInFlight : Promise.resolve();
|
|
7351
|
-
};
|
|
7243
|
+
phase: trace.phase,
|
|
7244
|
+
ms: trace.ms,
|
|
7245
|
+
extra: trace.extra,
|
|
7246
|
+
}),
|
|
7247
|
+
});
|
|
7352
7248
|
|
|
7353
7249
|
const appendStepLifecycleEvent = (event: PlayStepLifecycleEvent) => {
|
|
7354
7250
|
updateStepProgress({
|
|
@@ -7537,7 +7433,7 @@ async function executeRunRequest(
|
|
|
7537
7433
|
updateStepProgress(input);
|
|
7538
7434
|
const force = Boolean(input.forceFlush);
|
|
7539
7435
|
const ledgerFlush = flushLedgerEvents(force);
|
|
7540
|
-
const progressFlush =
|
|
7436
|
+
const progressFlush = coordinatorProgressPublisher.flush(force);
|
|
7541
7437
|
return force
|
|
7542
7438
|
? Promise.all([ledgerFlush, progressFlush]).then(() => undefined)
|
|
7543
7439
|
: Promise.resolve();
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import { coordinatorRequestHeaders } from '../../../../shared_libs/play-runtime/coordinator-headers';
|
|
2
|
+
import type {
|
|
3
|
+
LiveNodeProgressMap,
|
|
4
|
+
LiveNodeProgressSnapshot,
|
|
5
|
+
} from './live-progress';
|
|
6
|
+
|
|
7
|
+
type CoordinatorProgressTraceExtra = {
|
|
8
|
+
status: 'ok' | 'failed' | 'skipped_no_url';
|
|
9
|
+
httpStatus?: number;
|
|
10
|
+
activeNodeId?: string | null;
|
|
11
|
+
activeArtifactTableNamespace?: string | null;
|
|
12
|
+
activeCompleted?: number | null;
|
|
13
|
+
activeTotal?: number | null;
|
|
14
|
+
activeMessage?: string | null;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
type CoordinatorProgressTrace = {
|
|
18
|
+
phase: 'runner.coordinator_progress_publish';
|
|
19
|
+
ms: number;
|
|
20
|
+
extra: CoordinatorProgressTraceExtra;
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
export class CoordinatorProgressAuthorizationError extends Error {
|
|
24
|
+
constructor(
|
|
25
|
+
message: string,
|
|
26
|
+
readonly status: number,
|
|
27
|
+
) {
|
|
28
|
+
super(message);
|
|
29
|
+
this.name = 'CoordinatorProgressAuthorizationError';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function isCoordinatorProgressAuthorizationError(
|
|
34
|
+
error: unknown,
|
|
35
|
+
): error is CoordinatorProgressAuthorizationError {
|
|
36
|
+
return (
|
|
37
|
+
error instanceof CoordinatorProgressAuthorizationError ||
|
|
38
|
+
(typeof error === 'object' &&
|
|
39
|
+
error !== null &&
|
|
40
|
+
'name' in error &&
|
|
41
|
+
'status' in error &&
|
|
42
|
+
(error as { name?: unknown }).name ===
|
|
43
|
+
'CoordinatorProgressAuthorizationError' &&
|
|
44
|
+
typeof (error as { status?: unknown }).status === 'number')
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function activeProgressFields(liveNodeProgress: LiveNodeProgressMap): {
|
|
49
|
+
activeNodeId: string | null;
|
|
50
|
+
activeProgress: LiveNodeProgressSnapshot | null;
|
|
51
|
+
activeArtifactTableNamespace: string | null;
|
|
52
|
+
activeCompleted: number | null;
|
|
53
|
+
activeTotal: number | null;
|
|
54
|
+
activeMessage: string | null;
|
|
55
|
+
} {
|
|
56
|
+
const activeEntry =
|
|
57
|
+
Object.entries(liveNodeProgress).find(
|
|
58
|
+
([, progress]) => typeof progress.completedAt !== 'number',
|
|
59
|
+
) ?? Object.entries(liveNodeProgress).at(-1);
|
|
60
|
+
const activeNodeId = activeEntry?.[0] ?? null;
|
|
61
|
+
const activeProgress = activeEntry?.[1] ?? null;
|
|
62
|
+
const activeArtifactTableNamespace =
|
|
63
|
+
typeof activeProgress?.artifactTableNamespace === 'string'
|
|
64
|
+
? activeProgress.artifactTableNamespace
|
|
65
|
+
: null;
|
|
66
|
+
const activeCompleted =
|
|
67
|
+
typeof activeProgress?.completed === 'number'
|
|
68
|
+
? activeProgress.completed
|
|
69
|
+
: null;
|
|
70
|
+
const activeTotal =
|
|
71
|
+
typeof activeProgress?.total === 'number' ? activeProgress.total : null;
|
|
72
|
+
const activeMessage =
|
|
73
|
+
typeof activeProgress?.message === 'string' ? activeProgress.message : null;
|
|
74
|
+
return {
|
|
75
|
+
activeNodeId,
|
|
76
|
+
activeProgress,
|
|
77
|
+
activeArtifactTableNamespace,
|
|
78
|
+
activeCompleted,
|
|
79
|
+
activeTotal,
|
|
80
|
+
activeMessage,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function publishCoordinatorProgressEvent(input: {
|
|
85
|
+
runId: string;
|
|
86
|
+
coordinatorUrl?: string | null;
|
|
87
|
+
coordinatorInternalToken?: string | null;
|
|
88
|
+
occurredAt: number;
|
|
89
|
+
runLogBuffer: string[];
|
|
90
|
+
liveNodeProgress: LiveNodeProgressMap;
|
|
91
|
+
fetchFn: typeof fetch;
|
|
92
|
+
makeRequestId: () => string;
|
|
93
|
+
nowMs: () => number;
|
|
94
|
+
recordTrace: (trace: CoordinatorProgressTrace) => void;
|
|
95
|
+
}): Promise<void> {
|
|
96
|
+
const coordinatorUrl = input.coordinatorUrl?.trim();
|
|
97
|
+
if (!coordinatorUrl) {
|
|
98
|
+
input.recordTrace({
|
|
99
|
+
phase: 'runner.coordinator_progress_publish',
|
|
100
|
+
ms: 0,
|
|
101
|
+
extra: { status: 'skipped_no_url' },
|
|
102
|
+
});
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const publishStartedAt = input.nowMs();
|
|
107
|
+
const {
|
|
108
|
+
activeNodeId,
|
|
109
|
+
activeArtifactTableNamespace,
|
|
110
|
+
activeCompleted,
|
|
111
|
+
activeTotal,
|
|
112
|
+
activeMessage,
|
|
113
|
+
} = activeProgressFields(input.liveNodeProgress);
|
|
114
|
+
|
|
115
|
+
const response = await input.fetchFn(
|
|
116
|
+
`${coordinatorUrl.replace(/\/$/, '')}/dedup/${encodeURIComponent(
|
|
117
|
+
input.runId,
|
|
118
|
+
)}/event-add`,
|
|
119
|
+
{
|
|
120
|
+
method: 'POST',
|
|
121
|
+
headers: {
|
|
122
|
+
'x-deepline-request-id': input.makeRequestId(),
|
|
123
|
+
...coordinatorRequestHeaders({
|
|
124
|
+
runId: input.runId,
|
|
125
|
+
contentType: 'application/json',
|
|
126
|
+
internalToken: input.coordinatorInternalToken,
|
|
127
|
+
}),
|
|
128
|
+
},
|
|
129
|
+
body: JSON.stringify({
|
|
130
|
+
runId: input.runId,
|
|
131
|
+
type: 'progress',
|
|
132
|
+
status: 'running',
|
|
133
|
+
ts: input.occurredAt,
|
|
134
|
+
logs: input.runLogBuffer,
|
|
135
|
+
activeNodeId,
|
|
136
|
+
activeArtifactTableNamespace,
|
|
137
|
+
updatedAt: input.occurredAt,
|
|
138
|
+
liveNodeProgress: input.liveNodeProgress,
|
|
139
|
+
}),
|
|
140
|
+
},
|
|
141
|
+
);
|
|
142
|
+
|
|
143
|
+
if (response.status === 401 || response.status === 403) {
|
|
144
|
+
input.recordTrace({
|
|
145
|
+
phase: 'runner.coordinator_progress_publish',
|
|
146
|
+
ms: input.nowMs() - publishStartedAt,
|
|
147
|
+
extra: {
|
|
148
|
+
status: 'failed',
|
|
149
|
+
httpStatus: response.status,
|
|
150
|
+
activeNodeId,
|
|
151
|
+
activeArtifactTableNamespace,
|
|
152
|
+
activeCompleted,
|
|
153
|
+
activeTotal,
|
|
154
|
+
activeMessage,
|
|
155
|
+
},
|
|
156
|
+
});
|
|
157
|
+
throw new CoordinatorProgressAuthorizationError(
|
|
158
|
+
`Coordinator progress publish was rejected with ${response.status}; ` +
|
|
159
|
+
'the play runner no longer has authority to update its coordinator state. ' +
|
|
160
|
+
'Stopping before more provider work can execute with stale coordinator authority.',
|
|
161
|
+
response.status,
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (!response.ok) {
|
|
166
|
+
input.recordTrace({
|
|
167
|
+
phase: 'runner.coordinator_progress_publish',
|
|
168
|
+
ms: input.nowMs() - publishStartedAt,
|
|
169
|
+
extra: {
|
|
170
|
+
status: 'failed',
|
|
171
|
+
httpStatus: response.status,
|
|
172
|
+
activeNodeId,
|
|
173
|
+
activeArtifactTableNamespace,
|
|
174
|
+
activeCompleted,
|
|
175
|
+
activeTotal,
|
|
176
|
+
activeMessage,
|
|
177
|
+
},
|
|
178
|
+
});
|
|
179
|
+
throw new Error(`coordinator progress event failed ${response.status}`);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
input.recordTrace({
|
|
183
|
+
phase: 'runner.coordinator_progress_publish',
|
|
184
|
+
ms: input.nowMs() - publishStartedAt,
|
|
185
|
+
extra: {
|
|
186
|
+
status: 'ok',
|
|
187
|
+
activeNodeId,
|
|
188
|
+
activeArtifactTableNamespace,
|
|
189
|
+
activeCompleted,
|
|
190
|
+
activeTotal,
|
|
191
|
+
activeMessage,
|
|
192
|
+
},
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export function createCoordinatorProgressPublisher(input: {
|
|
197
|
+
runId: string;
|
|
198
|
+
coordinatorUrl?: string | null;
|
|
199
|
+
coordinatorInternalToken?: string | null;
|
|
200
|
+
abortController: AbortController;
|
|
201
|
+
getRunLogBuffer: () => string[];
|
|
202
|
+
getLiveNodeProgress: () => LiveNodeProgressMap;
|
|
203
|
+
recordTrace: (trace: CoordinatorProgressTrace) => void;
|
|
204
|
+
fetchFn?: typeof fetch;
|
|
205
|
+
makeRequestId: () => string;
|
|
206
|
+
nowMs: () => number;
|
|
207
|
+
heartbeatIntervalMs: number;
|
|
208
|
+
}): {
|
|
209
|
+
assertHealthy: () => void;
|
|
210
|
+
flush: (force: boolean) => Promise<void>;
|
|
211
|
+
fatalError: () => Error | null;
|
|
212
|
+
} {
|
|
213
|
+
let lastPublishAt = 0;
|
|
214
|
+
let publishInFlight: Promise<void> = Promise.resolve();
|
|
215
|
+
let publishQueueDepth = 0;
|
|
216
|
+
let fatalError: Error | null = null;
|
|
217
|
+
const abortSignal = input.abortController.signal;
|
|
218
|
+
const fetchFn = input.fetchFn ?? fetch;
|
|
219
|
+
|
|
220
|
+
const markFatal = (error: Error): void => {
|
|
221
|
+
fatalError ??= error;
|
|
222
|
+
if (!abortSignal.aborted) {
|
|
223
|
+
input.abortController.abort(error);
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
const assertHealthy = (): void => {
|
|
228
|
+
if (fatalError) {
|
|
229
|
+
throw fatalError;
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
const publish = async (occurredAt: number): Promise<void> => {
|
|
234
|
+
assertHealthy();
|
|
235
|
+
await publishCoordinatorProgressEvent({
|
|
236
|
+
runId: input.runId,
|
|
237
|
+
coordinatorUrl: input.coordinatorUrl,
|
|
238
|
+
coordinatorInternalToken: input.coordinatorInternalToken,
|
|
239
|
+
occurredAt,
|
|
240
|
+
runLogBuffer: input.getRunLogBuffer(),
|
|
241
|
+
liveNodeProgress: input.getLiveNodeProgress(),
|
|
242
|
+
fetchFn,
|
|
243
|
+
makeRequestId: input.makeRequestId,
|
|
244
|
+
nowMs: input.nowMs,
|
|
245
|
+
recordTrace: input.recordTrace,
|
|
246
|
+
});
|
|
247
|
+
};
|
|
248
|
+
|
|
249
|
+
const flush = (force: boolean): Promise<void> => {
|
|
250
|
+
if (fatalError) {
|
|
251
|
+
return force ? Promise.reject(fatalError) : Promise.resolve();
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const now = input.nowMs();
|
|
255
|
+
if (!force && now - lastPublishAt < input.heartbeatIntervalMs) {
|
|
256
|
+
return Promise.resolve();
|
|
257
|
+
}
|
|
258
|
+
if (!force && publishQueueDepth > 0) {
|
|
259
|
+
return Promise.resolve();
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
lastPublishAt = now;
|
|
263
|
+
publishQueueDepth += 1;
|
|
264
|
+
const publishTask = publishInFlight
|
|
265
|
+
.catch(() => undefined)
|
|
266
|
+
.then(async () => {
|
|
267
|
+
try {
|
|
268
|
+
await publish(now);
|
|
269
|
+
} finally {
|
|
270
|
+
publishQueueDepth = Math.max(0, publishQueueDepth - 1);
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
const handledPublishTask = publishTask.catch((error) => {
|
|
275
|
+
if (isCoordinatorProgressAuthorizationError(error)) {
|
|
276
|
+
markFatal(error);
|
|
277
|
+
throw error;
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
publishInFlight = handledPublishTask.catch(() => undefined);
|
|
281
|
+
return force ? handledPublishTask : Promise.resolve();
|
|
282
|
+
};
|
|
283
|
+
|
|
284
|
+
return {
|
|
285
|
+
assertHealthy,
|
|
286
|
+
flush,
|
|
287
|
+
fatalError: () => fatalError,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
@@ -105,10 +105,10 @@ export const SDK_RELEASE = {
|
|
|
105
105
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
106
106
|
// fields shipped in 0.1.153.
|
|
107
107
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
108
|
-
version: '0.1.
|
|
108
|
+
version: '0.1.179',
|
|
109
109
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
110
110
|
supportPolicy: {
|
|
111
|
-
latest: '0.1.
|
|
111
|
+
latest: '0.1.179',
|
|
112
112
|
minimumSupported: '0.1.53',
|
|
113
113
|
deprecatedBelow: '0.1.53',
|
|
114
114
|
commandMinimumSupported: [
|
|
@@ -303,7 +303,7 @@ function isRetryableAppRuntimeResponse(input: {
|
|
|
303
303
|
}
|
|
304
304
|
|
|
305
305
|
function isTransientAppRuntimeFailureBody(body: string): boolean {
|
|
306
|
-
return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT/i.test(
|
|
306
|
+
return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated/i.test(
|
|
307
307
|
body,
|
|
308
308
|
);
|
|
309
309
|
}
|
|
@@ -329,7 +329,15 @@ function isRetryableAppRuntimeAction(
|
|
|
329
329
|
action === 'append_run_events' ||
|
|
330
330
|
action === 'create_db_session' ||
|
|
331
331
|
action === 'create_signed_artifact_url' ||
|
|
332
|
-
action === 'create_signed_staged_file_url'
|
|
332
|
+
action === 'create_signed_staged_file_url' ||
|
|
333
|
+
action === 'get_runtime_step_receipt' ||
|
|
334
|
+
action === 'get_runtime_step_receipts' ||
|
|
335
|
+
action === 'claim_runtime_step_receipt' ||
|
|
336
|
+
action === 'claim_runtime_step_receipts' ||
|
|
337
|
+
action === 'complete_runtime_step_receipt' ||
|
|
338
|
+
action === 'complete_runtime_step_receipts' ||
|
|
339
|
+
action === 'fail_runtime_step_receipt' ||
|
|
340
|
+
action === 'fail_runtime_step_receipts'
|
|
333
341
|
);
|
|
334
342
|
}
|
|
335
343
|
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
normalizePlayRunFailure,
|
|
3
|
+
PLATFORM_DEPLOY_INTERRUPTED_MESSAGE,
|
|
4
|
+
} from './run-failure';
|
|
2
5
|
|
|
3
6
|
export type PlayRunLedgerStatus =
|
|
4
7
|
| 'queued'
|
|
@@ -530,6 +533,25 @@ const TERMINAL_STATUS_BY_EVENT_TYPE = {
|
|
|
530
533
|
'run.cancelled': 'cancelled',
|
|
531
534
|
} as const satisfies Partial<Record<string, PlayRunLedgerStatus>>;
|
|
532
535
|
|
|
536
|
+
function isRawPlatformDeployResetText(
|
|
537
|
+
error: string | null | undefined,
|
|
538
|
+
): boolean {
|
|
539
|
+
return Boolean(
|
|
540
|
+
error &&
|
|
541
|
+
normalizePlayRunFailure(error).code === 'PLATFORM_DEPLOY_INTERRUPTED',
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function isPlatformDeployFailureText(
|
|
546
|
+
error: string | null | undefined,
|
|
547
|
+
): boolean {
|
|
548
|
+
return Boolean(
|
|
549
|
+
error &&
|
|
550
|
+
(isRawPlatformDeployResetText(error) ||
|
|
551
|
+
error.includes(PLATFORM_DEPLOY_INTERRUPTED_MESSAGE)),
|
|
552
|
+
);
|
|
553
|
+
}
|
|
554
|
+
|
|
533
555
|
/**
|
|
534
556
|
* Append already-deduplicated log lines to the snapshot tail.
|
|
535
557
|
*
|
|
@@ -566,31 +588,29 @@ function appendLogLines(
|
|
|
566
588
|
}
|
|
567
589
|
|
|
568
590
|
/**
|
|
569
|
-
* Terminal-status precedence.
|
|
570
|
-
*
|
|
571
|
-
*
|
|
572
|
-
*
|
|
573
|
-
*
|
|
574
|
-
*
|
|
575
|
-
*
|
|
591
|
+
* Terminal-status precedence. Conflicting terminal events use
|
|
592
|
+
* newest-terminal-wins by event time: older events are ignored and logged,
|
|
593
|
+
* newer events reconcile the snapshot. Same-status terminal events continue to
|
|
594
|
+
* the reducer so recovery/transport paths can fill in canonical payloads after
|
|
595
|
+
* an earlier partial terminal snapshot. Retryable platform-deploy failures are
|
|
596
|
+
* the narrow exception: they are never allowed to replace a real terminal
|
|
597
|
+
* result from the runtime.
|
|
576
598
|
*/
|
|
577
599
|
function conflictingTerminalSnapshot(
|
|
578
600
|
base: PlayRunLedgerSnapshot,
|
|
579
601
|
eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
|
|
580
602
|
occurredAt: number,
|
|
581
603
|
eventError?: string | null,
|
|
604
|
+
options?: { allowStaleSameStatusPayloadMerge?: boolean },
|
|
582
605
|
): PlayRunLedgerSnapshot | null {
|
|
583
606
|
if (!isTerminalPlayRunLedgerStatus(base.status)) {
|
|
584
607
|
return null;
|
|
585
608
|
}
|
|
586
|
-
|
|
587
|
-
return null;
|
|
588
|
-
}
|
|
609
|
+
const terminalAt = base.finishedAt ?? base.updatedAt ?? 0;
|
|
589
610
|
if (
|
|
590
611
|
base.status === 'completed' &&
|
|
591
612
|
eventType === 'run.failed' &&
|
|
592
|
-
eventError
|
|
593
|
-
normalizePlayRunFailure(eventError).code === 'PLATFORM_DEPLOY_INTERRUPTED'
|
|
613
|
+
isPlatformDeployFailureText(eventError)
|
|
594
614
|
) {
|
|
595
615
|
return withTiming(
|
|
596
616
|
appendLogLines(base, [
|
|
@@ -598,7 +618,22 @@ function conflictingTerminalSnapshot(
|
|
|
598
618
|
]),
|
|
599
619
|
);
|
|
600
620
|
}
|
|
601
|
-
|
|
621
|
+
if (
|
|
622
|
+
base.status === 'failed' &&
|
|
623
|
+
eventType === 'run.failed' &&
|
|
624
|
+
isPlatformDeployFailureText(eventError)
|
|
625
|
+
) {
|
|
626
|
+
return withTiming(base);
|
|
627
|
+
}
|
|
628
|
+
if (TERMINAL_STATUS_BY_EVENT_TYPE[eventType] === base.status) {
|
|
629
|
+
if (
|
|
630
|
+
options?.allowStaleSameStatusPayloadMerge !== true ||
|
|
631
|
+
occurredAt < terminalAt
|
|
632
|
+
) {
|
|
633
|
+
return withTiming(base);
|
|
634
|
+
}
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
602
637
|
if (occurredAt > terminalAt) {
|
|
603
638
|
// Newer terminal evidence reconciles the run. This covers replay/receipt
|
|
604
639
|
// races where an earlier attempt failed but a later attempt recovered and
|
|
@@ -619,7 +654,7 @@ function retryablePlatformDeployFailureSnapshot(
|
|
|
619
654
|
if (
|
|
620
655
|
isTerminalPlayRunLedgerStatus(base.status) ||
|
|
621
656
|
!eventError ||
|
|
622
|
-
|
|
657
|
+
!isRawPlatformDeployResetText(eventError)
|
|
623
658
|
) {
|
|
624
659
|
return null;
|
|
625
660
|
}
|
|
@@ -717,7 +752,12 @@ export function reducePlayRunLedgerEvent(
|
|
|
717
752
|
});
|
|
718
753
|
case 'run.completed':
|
|
719
754
|
return (
|
|
720
|
-
conflictingTerminalSnapshot(base, event.type, occurredAt
|
|
755
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt, null, {
|
|
756
|
+
allowStaleSameStatusPayloadMerge:
|
|
757
|
+
(base.result === undefined && event.result !== undefined) ||
|
|
758
|
+
(base.resultSummary === undefined &&
|
|
759
|
+
event.resultSummary !== undefined),
|
|
760
|
+
}) ??
|
|
721
761
|
withTiming({
|
|
722
762
|
...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
|
|
723
763
|
status: 'completed',
|
|
@@ -733,16 +773,18 @@ export function reducePlayRunLedgerEvent(
|
|
|
733
773
|
case 'run.failed':
|
|
734
774
|
return (
|
|
735
775
|
retryablePlatformDeployFailureSnapshot(base, event.error) ??
|
|
736
|
-
conflictingTerminalSnapshot(
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
) ??
|
|
776
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
|
|
777
|
+
allowStaleSameStatusPayloadMerge:
|
|
778
|
+
(base.error == null && event.error != null) ||
|
|
779
|
+
(base.result === undefined && event.result !== undefined),
|
|
780
|
+
}) ??
|
|
742
781
|
withTiming({
|
|
743
782
|
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
744
783
|
status: 'failed',
|
|
745
|
-
error:
|
|
784
|
+
error:
|
|
785
|
+
base.status === 'failed'
|
|
786
|
+
? (base.error ?? event.error ?? null)
|
|
787
|
+
: (event.error ?? base.error ?? null),
|
|
746
788
|
startedAt: base.startedAt ?? occurredAt,
|
|
747
789
|
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
748
790
|
activeStepId: null,
|
|
@@ -752,16 +794,18 @@ export function reducePlayRunLedgerEvent(
|
|
|
752
794
|
);
|
|
753
795
|
case 'run.cancelled':
|
|
754
796
|
return (
|
|
755
|
-
conflictingTerminalSnapshot(
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
) ??
|
|
797
|
+
conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
|
|
798
|
+
allowStaleSameStatusPayloadMerge:
|
|
799
|
+
(base.error == null && event.error != null) ||
|
|
800
|
+
(base.result === undefined && event.result !== undefined),
|
|
801
|
+
}) ??
|
|
761
802
|
withTiming({
|
|
762
803
|
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
763
804
|
status: 'cancelled',
|
|
764
|
-
error:
|
|
805
|
+
error:
|
|
806
|
+
base.status === 'cancelled'
|
|
807
|
+
? (base.error ?? event.error ?? null)
|
|
808
|
+
: (event.error ?? base.error ?? null),
|
|
765
809
|
startedAt: base.startedAt ?? occurredAt,
|
|
766
810
|
finishedAt: terminalFinishedAt(base, event.type, occurredAt),
|
|
767
811
|
activeStepId: null,
|
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.179",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.179",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
|
@@ -16929,7 +16929,8 @@ var HEAVY_ENRICH_AUTO_BATCH_PLAYS = /* @__PURE__ */ new Set([
|
|
|
16929
16929
|
var HEAVY_ENRICH_AUTO_BATCH_TOOLS = /* @__PURE__ */ new Set([
|
|
16930
16930
|
"ai_inference",
|
|
16931
16931
|
"aiinference",
|
|
16932
|
-
"deeplineagent"
|
|
16932
|
+
"deeplineagent",
|
|
16933
|
+
"exa_answer"
|
|
16933
16934
|
]);
|
|
16934
16935
|
var ENRICH_TEST_AI_INFERENCE_PAYLOAD_MARKER = "__deeplineTestAiInference";
|
|
16935
16936
|
var PLAN_SHAPING_OPTION_NAMES = [
|
package/dist/cli/index.mjs
CHANGED
|
@@ -608,10 +608,10 @@ var SDK_RELEASE = {
|
|
|
608
608
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
609
609
|
// fields shipped in 0.1.153.
|
|
610
610
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
611
|
-
version: "0.1.
|
|
611
|
+
version: "0.1.179",
|
|
612
612
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
613
613
|
supportPolicy: {
|
|
614
|
-
latest: "0.1.
|
|
614
|
+
latest: "0.1.179",
|
|
615
615
|
minimumSupported: "0.1.53",
|
|
616
616
|
deprecatedBelow: "0.1.53",
|
|
617
617
|
commandMinimumSupported: [
|
|
@@ -16956,7 +16956,8 @@ var HEAVY_ENRICH_AUTO_BATCH_PLAYS = /* @__PURE__ */ new Set([
|
|
|
16956
16956
|
var HEAVY_ENRICH_AUTO_BATCH_TOOLS = /* @__PURE__ */ new Set([
|
|
16957
16957
|
"ai_inference",
|
|
16958
16958
|
"aiinference",
|
|
16959
|
-
"deeplineagent"
|
|
16959
|
+
"deeplineagent",
|
|
16960
|
+
"exa_answer"
|
|
16960
16961
|
]);
|
|
16961
16962
|
var ENRICH_TEST_AI_INFERENCE_PAYLOAD_MARKER = "__deeplineTestAiInference";
|
|
16962
16963
|
var PLAN_SHAPING_OPTION_NAMES = [
|
package/dist/index.js
CHANGED
|
@@ -422,10 +422,10 @@ var SDK_RELEASE = {
|
|
|
422
422
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
423
423
|
// fields shipped in 0.1.153.
|
|
424
424
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
425
|
-
version: "0.1.
|
|
425
|
+
version: "0.1.179",
|
|
426
426
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
427
427
|
supportPolicy: {
|
|
428
|
-
latest: "0.1.
|
|
428
|
+
latest: "0.1.179",
|
|
429
429
|
minimumSupported: "0.1.53",
|
|
430
430
|
deprecatedBelow: "0.1.53",
|
|
431
431
|
commandMinimumSupported: [
|
package/dist/index.mjs
CHANGED
|
@@ -352,10 +352,10 @@ var SDK_RELEASE = {
|
|
|
352
352
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
353
353
|
// fields shipped in 0.1.153.
|
|
354
354
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
355
|
-
version: "0.1.
|
|
355
|
+
version: "0.1.179",
|
|
356
356
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
357
357
|
supportPolicy: {
|
|
358
|
-
latest: "0.1.
|
|
358
|
+
latest: "0.1.179",
|
|
359
359
|
minimumSupported: "0.1.53",
|
|
360
360
|
deprecatedBelow: "0.1.53",
|
|
361
361
|
commandMinimumSupported: [
|