deepline 0.1.90 → 0.1.93
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +1356 -225
- package/dist/cli/index.mjs +1356 -225
- package/dist/index.d.mts +74 -5
- package/dist/index.d.ts +74 -5
- package/dist/index.js +1018 -62
- package/dist/index.mjs +1007 -62
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +87 -20
- package/dist/repo/apps/play-runner-workers/src/entry.ts +52 -14
- package/dist/repo/sdk/src/client.ts +289 -40
- package/dist/repo/sdk/src/index.ts +1 -0
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/sdk/src/runs/observe-transport.ts +481 -0
- package/dist/repo/sdk/src/stream-reconnect.ts +44 -0
- package/dist/repo/sdk/src/types.ts +10 -3
- package/dist/repo/shared_libs/play-runtime/live-events.ts +217 -0
- package/dist/repo/shared_libs/play-runtime/run-ledger.ts +1074 -0
- package/dist/repo/shared_libs/play-runtime/run-snapshot-stream.ts +581 -0
- package/package.json +5 -2
|
@@ -0,0 +1,1074 @@
|
|
|
1
|
+
export type PlayRunLedgerStatus =
|
|
2
|
+
| 'queued'
|
|
3
|
+
| 'running'
|
|
4
|
+
| 'waiting'
|
|
5
|
+
| 'completed'
|
|
6
|
+
| 'failed'
|
|
7
|
+
| 'cancelled'
|
|
8
|
+
| 'terminated'
|
|
9
|
+
| 'timed_out'
|
|
10
|
+
| 'unknown';
|
|
11
|
+
|
|
12
|
+
export type PlayRunLedgerStepStatus =
|
|
13
|
+
| 'running'
|
|
14
|
+
| 'completed'
|
|
15
|
+
| 'failed'
|
|
16
|
+
| 'skipped';
|
|
17
|
+
|
|
18
|
+
export type PlayRunLedgerEventSource =
|
|
19
|
+
| 'worker'
|
|
20
|
+
| 'temporal'
|
|
21
|
+
| 'convex'
|
|
22
|
+
| 'coordinator'
|
|
23
|
+
| 'system';
|
|
24
|
+
|
|
25
|
+
export type PlayRunLedgerStepProgress = {
|
|
26
|
+
completed?: number;
|
|
27
|
+
total?: number;
|
|
28
|
+
failed?: number;
|
|
29
|
+
message?: string;
|
|
30
|
+
artifactTableNamespace?: string | null;
|
|
31
|
+
startedAt?: number | null;
|
|
32
|
+
completedAt?: number | null;
|
|
33
|
+
updatedAt?: number | null;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export type PlayRunLedgerStepSnapshot = {
|
|
37
|
+
stepId: string;
|
|
38
|
+
label?: string;
|
|
39
|
+
kind?: string;
|
|
40
|
+
status: PlayRunLedgerStepStatus;
|
|
41
|
+
artifactTableNamespace?: string | null;
|
|
42
|
+
startedAt?: number | null;
|
|
43
|
+
completedAt?: number | null;
|
|
44
|
+
updatedAt?: number | null;
|
|
45
|
+
progress?: PlayRunLedgerStepProgress | null;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export type PlayRunLedgerSnapshot = {
|
|
49
|
+
runId: string;
|
|
50
|
+
playName?: string | null;
|
|
51
|
+
status: PlayRunLedgerStatus;
|
|
52
|
+
error?: string | null;
|
|
53
|
+
createdAt?: number | null;
|
|
54
|
+
startedAt?: number | null;
|
|
55
|
+
updatedAt?: number | null;
|
|
56
|
+
finishedAt?: number | null;
|
|
57
|
+
durationMs?: number | null;
|
|
58
|
+
orderedStepIds: string[];
|
|
59
|
+
stepsById: Record<string, PlayRunLedgerStepSnapshot>;
|
|
60
|
+
/**
|
|
61
|
+
* Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
|
|
62
|
+
* Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
|
|
63
|
+
* the snapshot only carries enough tail for live previews.
|
|
64
|
+
* `totalLogCount` is the monotonic count of every line ever ingested
|
|
65
|
+
* (post channel-dedupe), so readers can derive the absolute sequence
|
|
66
|
+
* number of the first retained tail line:
|
|
67
|
+
* `totalLogCount - logTail.length + 1`.
|
|
68
|
+
*/
|
|
69
|
+
logTail: string[];
|
|
70
|
+
totalLogCount: number;
|
|
71
|
+
/**
|
|
72
|
+
* True once the Run Log Stream stopped storing line bodies because the run
|
|
73
|
+
* crossed the retention cap (25k lines / 5MB). `totalLogCount` keeps
|
|
74
|
+
* counting past the cap; a loud truncation marker is the last stored line.
|
|
75
|
+
*/
|
|
76
|
+
logsTruncated: boolean;
|
|
77
|
+
activeStepId?: string | null;
|
|
78
|
+
activeArtifactTableNamespace?: string | null;
|
|
79
|
+
resultTableNamespace?: string | null;
|
|
80
|
+
resultSummary?: unknown;
|
|
81
|
+
result?: unknown;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
type PlayRunLedgerBaseEvent = {
|
|
85
|
+
runId: string;
|
|
86
|
+
seq?: number;
|
|
87
|
+
occurredAt: number;
|
|
88
|
+
source: PlayRunLedgerEventSource;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
export type PlayRunLedgerEvent =
|
|
92
|
+
| (PlayRunLedgerBaseEvent & {
|
|
93
|
+
type: 'run.created';
|
|
94
|
+
playName?: string | null;
|
|
95
|
+
status?: PlayRunLedgerStatus;
|
|
96
|
+
runtimeBackend?: string | null;
|
|
97
|
+
})
|
|
98
|
+
| (PlayRunLedgerBaseEvent & {
|
|
99
|
+
type: 'run.started';
|
|
100
|
+
playName?: string | null;
|
|
101
|
+
runtimeBackend?: string | null;
|
|
102
|
+
})
|
|
103
|
+
| (PlayRunLedgerBaseEvent & {
|
|
104
|
+
type: 'run.completed';
|
|
105
|
+
result?: unknown;
|
|
106
|
+
resultSummary?: unknown;
|
|
107
|
+
})
|
|
108
|
+
| (PlayRunLedgerBaseEvent & {
|
|
109
|
+
type: 'run.failed';
|
|
110
|
+
error?: string | null;
|
|
111
|
+
result?: unknown;
|
|
112
|
+
})
|
|
113
|
+
| (PlayRunLedgerBaseEvent & {
|
|
114
|
+
type: 'run.cancelled';
|
|
115
|
+
error?: string | null;
|
|
116
|
+
result?: unknown;
|
|
117
|
+
})
|
|
118
|
+
| (PlayRunLedgerBaseEvent & {
|
|
119
|
+
type: 'step.started';
|
|
120
|
+
stepId: string;
|
|
121
|
+
label?: string;
|
|
122
|
+
kind?: string;
|
|
123
|
+
artifactTableNamespace?: string | null;
|
|
124
|
+
})
|
|
125
|
+
| (PlayRunLedgerBaseEvent & {
|
|
126
|
+
type: 'step.progress';
|
|
127
|
+
stepId: string;
|
|
128
|
+
label?: string;
|
|
129
|
+
kind?: string;
|
|
130
|
+
status?: PlayRunLedgerStepStatus;
|
|
131
|
+
progress: PlayRunLedgerStepProgress;
|
|
132
|
+
})
|
|
133
|
+
| (PlayRunLedgerBaseEvent & {
|
|
134
|
+
type: 'step.completed';
|
|
135
|
+
stepId: string;
|
|
136
|
+
label?: string;
|
|
137
|
+
kind?: string;
|
|
138
|
+
artifactTableNamespace?: string | null;
|
|
139
|
+
})
|
|
140
|
+
| (PlayRunLedgerBaseEvent & {
|
|
141
|
+
type: 'step.failed';
|
|
142
|
+
stepId: string;
|
|
143
|
+
label?: string;
|
|
144
|
+
kind?: string;
|
|
145
|
+
error?: string | null;
|
|
146
|
+
artifactTableNamespace?: string | null;
|
|
147
|
+
})
|
|
148
|
+
| (PlayRunLedgerBaseEvent & {
|
|
149
|
+
type: 'step.skipped';
|
|
150
|
+
stepId: string;
|
|
151
|
+
label?: string;
|
|
152
|
+
kind?: string;
|
|
153
|
+
artifactTableNamespace?: string | null;
|
|
154
|
+
})
|
|
155
|
+
| (PlayRunLedgerBaseEvent & {
|
|
156
|
+
type: 'log.appended';
|
|
157
|
+
lines: string[];
|
|
158
|
+
/**
|
|
159
|
+
* Absolute per-run sequence (1-based) of `lines[0]`. Assigned by Run
|
|
160
|
+
* Log Stream ingestion in Convex only; producers never set it.
|
|
161
|
+
*/
|
|
162
|
+
firstSeq?: number;
|
|
163
|
+
/**
|
|
164
|
+
* Positional cursor for exactly-once delivery on the worker channel:
|
|
165
|
+
* the count of lines this producer emitted before `lines[0]`. Ingestion
|
|
166
|
+
* skips the already-ingested prefix positionally, which preserves
|
|
167
|
+
* repeated identical lines while absorbing redundant re-sends.
|
|
168
|
+
*/
|
|
169
|
+
channelOffset?: number;
|
|
170
|
+
/** Set by ingestion when this append crossed the retention cap. */
|
|
171
|
+
logsTruncated?: boolean;
|
|
172
|
+
})
|
|
173
|
+
| (PlayRunLedgerBaseEvent & {
|
|
174
|
+
type: 'sheet.summary.updated';
|
|
175
|
+
tableNamespace: string;
|
|
176
|
+
deltaCursor?: number;
|
|
177
|
+
summary?: unknown;
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
export type PlayRunLedgerStatusPatch = {
|
|
181
|
+
runId: string;
|
|
182
|
+
playName?: string | null;
|
|
183
|
+
status: string;
|
|
184
|
+
error?: string | null;
|
|
185
|
+
runtimeBackend?: string | null;
|
|
186
|
+
lastCheckpointAt?: number | null;
|
|
187
|
+
liveLogs?: readonly string[] | null;
|
|
188
|
+
/**
|
|
189
|
+
* Positional cursor for `liveLogs[0]` on this producer's log channel (the
|
|
190
|
+
* count of lines emitted before it). Computed by the producer-side cursor
|
|
191
|
+
* (see `slicePositionalLogLines`), forwarded onto the `log.appended` event.
|
|
192
|
+
*/
|
|
193
|
+
liveLogsChannelOffset?: number | null;
|
|
194
|
+
liveNodeProgress?: unknown;
|
|
195
|
+
result?: unknown;
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
export const LOG_TAIL_LIMIT = 100;
|
|
199
|
+
const TERMINAL_STATUSES = new Set<PlayRunLedgerStatus>([
|
|
200
|
+
'completed',
|
|
201
|
+
'failed',
|
|
202
|
+
'cancelled',
|
|
203
|
+
'terminated',
|
|
204
|
+
'timed_out',
|
|
205
|
+
]);
|
|
206
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
207
|
+
return Boolean(value && typeof value === 'object' && !Array.isArray(value));
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function finiteNumber(value: unknown): number | null {
|
|
211
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function optionalFiniteNumber(value: unknown): number | undefined {
|
|
215
|
+
const normalized = finiteNumber(value);
|
|
216
|
+
return normalized === null ? undefined : normalized;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function optionalString(value: unknown): string | undefined {
|
|
220
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function optionalNullableString(value: unknown): string | null | undefined {
|
|
224
|
+
if (value === null) return null;
|
|
225
|
+
return optionalString(value);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export function normalizePlayRunLedgerStatus(
|
|
229
|
+
value: unknown,
|
|
230
|
+
): PlayRunLedgerStatus {
|
|
231
|
+
const normalized = String(value ?? '')
|
|
232
|
+
.trim()
|
|
233
|
+
.toLowerCase();
|
|
234
|
+
switch (normalized) {
|
|
235
|
+
case 'queued':
|
|
236
|
+
case 'pending':
|
|
237
|
+
return 'queued';
|
|
238
|
+
case 'running':
|
|
239
|
+
case 'started':
|
|
240
|
+
return 'running';
|
|
241
|
+
case 'waiting':
|
|
242
|
+
return 'waiting';
|
|
243
|
+
case 'completed':
|
|
244
|
+
case 'complete':
|
|
245
|
+
case 'succeeded':
|
|
246
|
+
return 'completed';
|
|
247
|
+
case 'failed':
|
|
248
|
+
case 'error':
|
|
249
|
+
return 'failed';
|
|
250
|
+
case 'cancelled':
|
|
251
|
+
case 'canceled':
|
|
252
|
+
return 'cancelled';
|
|
253
|
+
case 'terminated':
|
|
254
|
+
return 'terminated';
|
|
255
|
+
case 'timed_out':
|
|
256
|
+
case 'timeout':
|
|
257
|
+
return 'timed_out';
|
|
258
|
+
default:
|
|
259
|
+
return 'unknown';
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export function isTerminalPlayRunLedgerStatus(
|
|
264
|
+
status: PlayRunLedgerStatus,
|
|
265
|
+
): boolean {
|
|
266
|
+
return TERMINAL_STATUSES.has(status);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export function createEmptyPlayRunLedgerSnapshot(input: {
|
|
270
|
+
runId: string;
|
|
271
|
+
playName?: string | null;
|
|
272
|
+
status?: unknown;
|
|
273
|
+
error?: string | null;
|
|
274
|
+
createdAt?: number | null;
|
|
275
|
+
startedAt?: number | null;
|
|
276
|
+
updatedAt?: number | null;
|
|
277
|
+
finishedAt?: number | null;
|
|
278
|
+
}): PlayRunLedgerSnapshot {
|
|
279
|
+
const status = normalizePlayRunLedgerStatus(input.status ?? 'unknown');
|
|
280
|
+
const startedAt = finiteNumber(input.startedAt) ?? null;
|
|
281
|
+
const finishedAt = finiteNumber(input.finishedAt) ?? null;
|
|
282
|
+
return {
|
|
283
|
+
runId: input.runId,
|
|
284
|
+
playName: input.playName ?? null,
|
|
285
|
+
status,
|
|
286
|
+
error: optionalNullableString(input.error) ?? null,
|
|
287
|
+
createdAt: finiteNumber(input.createdAt) ?? null,
|
|
288
|
+
startedAt,
|
|
289
|
+
updatedAt:
|
|
290
|
+
finiteNumber(input.updatedAt) ??
|
|
291
|
+
finishedAt ??
|
|
292
|
+
startedAt ??
|
|
293
|
+
finiteNumber(input.createdAt) ??
|
|
294
|
+
null,
|
|
295
|
+
finishedAt,
|
|
296
|
+
durationMs:
|
|
297
|
+
startedAt !== null && finishedAt !== null
|
|
298
|
+
? Math.max(0, finishedAt - startedAt)
|
|
299
|
+
: null,
|
|
300
|
+
orderedStepIds: [],
|
|
301
|
+
stepsById: {},
|
|
302
|
+
logTail: [],
|
|
303
|
+
totalLogCount: 0,
|
|
304
|
+
logsTruncated: false,
|
|
305
|
+
activeStepId: null,
|
|
306
|
+
activeArtifactTableNamespace: null,
|
|
307
|
+
resultTableNamespace: null,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export function normalizePlayRunLedgerSnapshot(
|
|
312
|
+
value: unknown,
|
|
313
|
+
fallback: Parameters<typeof createEmptyPlayRunLedgerSnapshot>[0],
|
|
314
|
+
): PlayRunLedgerSnapshot {
|
|
315
|
+
if (!isRecord(value)) {
|
|
316
|
+
return createEmptyPlayRunLedgerSnapshot(fallback);
|
|
317
|
+
}
|
|
318
|
+
const orderedStepIds = Array.isArray(value.orderedStepIds)
|
|
319
|
+
? value.orderedStepIds.filter(
|
|
320
|
+
(entry): entry is string =>
|
|
321
|
+
typeof entry === 'string' && Boolean(entry.trim()),
|
|
322
|
+
)
|
|
323
|
+
: [];
|
|
324
|
+
const rawSteps = isRecord(value.stepsById) ? value.stepsById : {};
|
|
325
|
+
const stepsById: Record<string, PlayRunLedgerStepSnapshot> = {};
|
|
326
|
+
for (const [stepId, rawStep] of Object.entries(rawSteps)) {
|
|
327
|
+
if (!stepId.trim() || !isRecord(rawStep)) continue;
|
|
328
|
+
const rawStatus = normalizeStepStatus(rawStep.status);
|
|
329
|
+
if (!rawStatus) continue;
|
|
330
|
+
const completedAt = finiteNumber(rawStep.completedAt);
|
|
331
|
+
const status =
|
|
332
|
+
rawStatus === 'running' && completedAt !== null ? 'completed' : rawStatus;
|
|
333
|
+
const rawProgress = isRecord(rawStep.progress) ? rawStep.progress : null;
|
|
334
|
+
stepsById[stepId] = {
|
|
335
|
+
stepId,
|
|
336
|
+
label: optionalString(rawStep.label),
|
|
337
|
+
kind: optionalString(rawStep.kind),
|
|
338
|
+
status,
|
|
339
|
+
artifactTableNamespace: optionalNullableString(
|
|
340
|
+
rawStep.artifactTableNamespace,
|
|
341
|
+
),
|
|
342
|
+
startedAt: finiteNumber(rawStep.startedAt),
|
|
343
|
+
completedAt,
|
|
344
|
+
updatedAt: finiteNumber(rawStep.updatedAt),
|
|
345
|
+
progress: rawProgress ? normalizeStepProgress(rawProgress) : null,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
const createdAt = finiteNumber(value.createdAt) ?? fallback.createdAt ?? null;
|
|
349
|
+
const startedAt = finiteNumber(value.startedAt) ?? fallback.startedAt ?? null;
|
|
350
|
+
const finishedAt =
|
|
351
|
+
finiteNumber(value.finishedAt) ?? fallback.finishedAt ?? null;
|
|
352
|
+
const updatedAt =
|
|
353
|
+
finiteNumber(value.updatedAt) ??
|
|
354
|
+
fallback.updatedAt ??
|
|
355
|
+
finishedAt ??
|
|
356
|
+
startedAt ??
|
|
357
|
+
createdAt ??
|
|
358
|
+
null;
|
|
359
|
+
const error = Object.prototype.hasOwnProperty.call(value, 'error')
|
|
360
|
+
? (optionalNullableString(value.error) ?? null)
|
|
361
|
+
: (fallback.error ?? null);
|
|
362
|
+
// `logTail` is canonical; `logs` is the pre-Run-Log-Stream persisted name.
|
|
363
|
+
// Reading both is the deploy-window seam that lets the first post-cutover
|
|
364
|
+
// append backfill an old snapshot's retained tail into chunk storage.
|
|
365
|
+
const rawTail = Array.isArray(value.logTail) ? value.logTail : value.logs;
|
|
366
|
+
const logTail = Array.isArray(rawTail)
|
|
367
|
+
? rawTail.filter((line): line is string => typeof line === 'string')
|
|
368
|
+
: [];
|
|
369
|
+
return {
|
|
370
|
+
runId: optionalString(value.runId) ?? fallback.runId,
|
|
371
|
+
playName:
|
|
372
|
+
optionalNullableString(value.playName) ?? fallback.playName ?? null,
|
|
373
|
+
status: normalizePlayRunLedgerStatus(value.status ?? fallback.status),
|
|
374
|
+
error,
|
|
375
|
+
createdAt,
|
|
376
|
+
startedAt,
|
|
377
|
+
updatedAt,
|
|
378
|
+
finishedAt,
|
|
379
|
+
durationMs:
|
|
380
|
+
startedAt !== null && finishedAt !== null
|
|
381
|
+
? Math.max(0, finishedAt - startedAt)
|
|
382
|
+
: finiteNumber(value.durationMs),
|
|
383
|
+
orderedStepIds: orderedStepIds.filter((stepId) => stepsById[stepId]),
|
|
384
|
+
stepsById,
|
|
385
|
+
logTail: logTail.slice(-LOG_TAIL_LIMIT),
|
|
386
|
+
// Snapshots persisted before totalLogCount existed only know the retained
|
|
387
|
+
// tail, so the best lower bound for the cumulative count is the tail size.
|
|
388
|
+
totalLogCount: Math.max(
|
|
389
|
+
finiteNumber(value.totalLogCount) ?? logTail.length,
|
|
390
|
+
logTail.length,
|
|
391
|
+
),
|
|
392
|
+
logsTruncated: value.logsTruncated === true,
|
|
393
|
+
activeStepId: optionalNullableString(value.activeStepId),
|
|
394
|
+
activeArtifactTableNamespace: optionalNullableString(
|
|
395
|
+
value.activeArtifactTableNamespace,
|
|
396
|
+
),
|
|
397
|
+
resultTableNamespace: optionalNullableString(value.resultTableNamespace),
|
|
398
|
+
resultSummary: value.resultSummary,
|
|
399
|
+
result: value.result,
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
function normalizeStepStatus(value: unknown): PlayRunLedgerStepStatus | null {
|
|
404
|
+
const normalized = String(value ?? '')
|
|
405
|
+
.trim()
|
|
406
|
+
.toLowerCase();
|
|
407
|
+
if (
|
|
408
|
+
normalized === 'running' ||
|
|
409
|
+
normalized === 'completed' ||
|
|
410
|
+
normalized === 'failed' ||
|
|
411
|
+
normalized === 'skipped'
|
|
412
|
+
) {
|
|
413
|
+
return normalized;
|
|
414
|
+
}
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function normalizeStepProgress(
|
|
419
|
+
value: Record<string, unknown>,
|
|
420
|
+
): PlayRunLedgerStepProgress {
|
|
421
|
+
return {
|
|
422
|
+
...(optionalFiniteNumber(value.completed) !== undefined
|
|
423
|
+
? { completed: optionalFiniteNumber(value.completed) }
|
|
424
|
+
: {}),
|
|
425
|
+
...(optionalFiniteNumber(value.total) !== undefined
|
|
426
|
+
? { total: optionalFiniteNumber(value.total) }
|
|
427
|
+
: {}),
|
|
428
|
+
...(optionalFiniteNumber(value.failed) !== undefined
|
|
429
|
+
? { failed: optionalFiniteNumber(value.failed) }
|
|
430
|
+
: {}),
|
|
431
|
+
...(optionalString(value.message)
|
|
432
|
+
? { message: optionalString(value.message) }
|
|
433
|
+
: {}),
|
|
434
|
+
...(optionalNullableString(value.artifactTableNamespace) !== undefined
|
|
435
|
+
? {
|
|
436
|
+
artifactTableNamespace: optionalNullableString(
|
|
437
|
+
value.artifactTableNamespace,
|
|
438
|
+
),
|
|
439
|
+
}
|
|
440
|
+
: {}),
|
|
441
|
+
...(finiteNumber(value.updatedAt) !== null
|
|
442
|
+
? { updatedAt: finiteNumber(value.updatedAt) }
|
|
443
|
+
: {}),
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function normalizeLiveProgressMap(
|
|
448
|
+
value: unknown,
|
|
449
|
+
): Record<string, PlayRunLedgerStepProgress> {
|
|
450
|
+
if (!isRecord(value)) {
|
|
451
|
+
return {};
|
|
452
|
+
}
|
|
453
|
+
const normalized: Record<string, PlayRunLedgerStepProgress> = {};
|
|
454
|
+
for (const [stepId, rawProgress] of Object.entries(value)) {
|
|
455
|
+
if (!stepId.trim() || !isRecord(rawProgress)) continue;
|
|
456
|
+
normalized[stepId] = {
|
|
457
|
+
...normalizeStepProgress(rawProgress),
|
|
458
|
+
...(optionalFiniteNumber(rawProgress.startedAt) !== undefined
|
|
459
|
+
? { startedAt: optionalFiniteNumber(rawProgress.startedAt) }
|
|
460
|
+
: {}),
|
|
461
|
+
...(optionalFiniteNumber(rawProgress.completedAt) !== undefined
|
|
462
|
+
? { completedAt: optionalFiniteNumber(rawProgress.completedAt) }
|
|
463
|
+
: {}),
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
return normalized;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function appendOrderedStepId(
|
|
470
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
471
|
+
stepId: string,
|
|
472
|
+
): string[] {
|
|
473
|
+
return snapshot.orderedStepIds.includes(stepId)
|
|
474
|
+
? snapshot.orderedStepIds
|
|
475
|
+
: [...snapshot.orderedStepIds, stepId];
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function resolveResultTableNamespace(
|
|
479
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
480
|
+
): string | null {
|
|
481
|
+
const terminalStepNamespace = [...snapshot.orderedStepIds]
|
|
482
|
+
.reverse()
|
|
483
|
+
.map((stepId) => snapshot.stepsById[stepId]?.artifactTableNamespace)
|
|
484
|
+
.find((namespace): namespace is string => Boolean(namespace?.trim()));
|
|
485
|
+
return snapshot.resultTableNamespace ?? terminalStepNamespace ?? null;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
function withTiming(snapshot: PlayRunLedgerSnapshot): PlayRunLedgerSnapshot {
|
|
489
|
+
const startedAt = snapshot.startedAt ?? null;
|
|
490
|
+
const finishedAt = snapshot.finishedAt ?? null;
|
|
491
|
+
return {
|
|
492
|
+
...snapshot,
|
|
493
|
+
durationMs:
|
|
494
|
+
startedAt !== null && finishedAt !== null
|
|
495
|
+
? Math.max(0, finishedAt - startedAt)
|
|
496
|
+
: null,
|
|
497
|
+
resultTableNamespace: isTerminalPlayRunLedgerStatus(snapshot.status)
|
|
498
|
+
? resolveResultTableNamespace(snapshot)
|
|
499
|
+
: (snapshot.resultTableNamespace ?? null),
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const TERMINAL_STATUS_BY_EVENT_TYPE = {
|
|
504
|
+
'run.completed': 'completed',
|
|
505
|
+
'run.failed': 'failed',
|
|
506
|
+
'run.cancelled': 'cancelled',
|
|
507
|
+
} as const satisfies Partial<Record<string, PlayRunLedgerStatus>>;
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Append already-deduplicated log lines to the snapshot tail.
|
|
511
|
+
*
|
|
512
|
+
* Cross-channel reconciliation (positional worker cursors, bounded
|
|
513
|
+
* text-window dedupe for recovery channels) happens at Run Log Stream
|
|
514
|
+
* ingestion in Convex, BEFORE events reach this reducer. The reducer trusts
|
|
515
|
+
* the event: every line counts, repeated identical lines included.
|
|
516
|
+
*
|
|
517
|
+
* Events stamped with `firstSeq` (assigned by ingestion) pin the cumulative
|
|
518
|
+
* count exactly; unstamped events (reducer-internal anomaly markers,
|
|
519
|
+
* producer-side snapshot caches) advance it by line count.
|
|
520
|
+
*/
|
|
521
|
+
function appendLogLines(
|
|
522
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
523
|
+
lines: readonly string[],
|
|
524
|
+
options?: { firstSeq?: number; logsTruncated?: boolean },
|
|
525
|
+
): PlayRunLedgerSnapshot {
|
|
526
|
+
const nextLines = lines
|
|
527
|
+
.map((line) => line.trim())
|
|
528
|
+
.filter((line) => line.length > 0);
|
|
529
|
+
const totalLogCount =
|
|
530
|
+
typeof options?.firstSeq === 'number' && options.firstSeq > 0
|
|
531
|
+
? Math.max(
|
|
532
|
+
snapshot.totalLogCount,
|
|
533
|
+
options.firstSeq + nextLines.length - 1,
|
|
534
|
+
)
|
|
535
|
+
: snapshot.totalLogCount + nextLines.length;
|
|
536
|
+
return {
|
|
537
|
+
...snapshot,
|
|
538
|
+
logTail: [...snapshot.logTail, ...nextLines].slice(-LOG_TAIL_LIMIT),
|
|
539
|
+
totalLogCount,
|
|
540
|
+
logsTruncated: snapshot.logsTruncated || options?.logsTruncated === true,
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/**
|
|
545
|
+
* Terminal-status precedence. Re-delivery of the same terminal status is a
|
|
546
|
+
* benign no-op handled by the regular reduction. A terminal event that
|
|
547
|
+
* disagrees with an already-terminal snapshot is ignored (keeping e.g. an
|
|
548
|
+
* explicit user cancellation from being flipped to completed by a late
|
|
549
|
+
* worker terminal), and the conflict is recorded as one anomaly log line —
|
|
550
|
+
* with ONE exception: `run.failed` DEMOTES a `completed` snapshot. The
|
|
551
|
+
* worker appends run.completed BEFORE awaiting post-completion accounting
|
|
552
|
+
* (compute billing finalize), so a billing business denial there — e.g. the
|
|
553
|
+
* per-run credit cap (maxCreditsPerRun) — arrives as a later run.failed
|
|
554
|
+
* that MUST fail the run. Blanket first-terminal-wins silently completed
|
|
555
|
+
* capped runs (regression pinned by
|
|
556
|
+
* tests/v2-plays/plays/44-compute-billing-cap.play.ts). A cancelled run
|
|
557
|
+
* stays cancelled, and a failed run can never be flipped to completed.
|
|
558
|
+
*/
|
|
559
|
+
function conflictingTerminalSnapshot(
|
|
560
|
+
base: PlayRunLedgerSnapshot,
|
|
561
|
+
eventType: keyof typeof TERMINAL_STATUS_BY_EVENT_TYPE,
|
|
562
|
+
): PlayRunLedgerSnapshot | null {
|
|
563
|
+
if (!isTerminalPlayRunLedgerStatus(base.status)) {
|
|
564
|
+
return null;
|
|
565
|
+
}
|
|
566
|
+
if (TERMINAL_STATUS_BY_EVENT_TYPE[eventType] === base.status) {
|
|
567
|
+
return null;
|
|
568
|
+
}
|
|
569
|
+
if (base.status === 'completed' && eventType === 'run.failed') {
|
|
570
|
+
// Post-completion accounting demotion (e.g. per-run billing cap denial):
|
|
571
|
+
// let the regular run.failed reduction flip the run to failed.
|
|
572
|
+
return null;
|
|
573
|
+
}
|
|
574
|
+
return withTiming(
|
|
575
|
+
appendLogLines(base, [
|
|
576
|
+
`[ledger] conflicting terminal event ${eventType} ignored; status already ${base.status}`,
|
|
577
|
+
]),
|
|
578
|
+
);
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
export function reducePlayRunLedgerEvent(
|
|
582
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
583
|
+
event: PlayRunLedgerEvent,
|
|
584
|
+
): PlayRunLedgerSnapshot {
|
|
585
|
+
if (event.runId !== snapshot.runId) {
|
|
586
|
+
return snapshot;
|
|
587
|
+
}
|
|
588
|
+
const occurredAt = Math.max(0, event.occurredAt);
|
|
589
|
+
const base: PlayRunLedgerSnapshot = {
|
|
590
|
+
...snapshot,
|
|
591
|
+
updatedAt: Math.max(snapshot.updatedAt ?? 0, occurredAt),
|
|
592
|
+
};
|
|
593
|
+
|
|
594
|
+
switch (event.type) {
|
|
595
|
+
case 'run.created':
|
|
596
|
+
return withTiming({
|
|
597
|
+
...base,
|
|
598
|
+
playName: event.playName ?? base.playName ?? null,
|
|
599
|
+
status: event.status ?? base.status,
|
|
600
|
+
createdAt: base.createdAt ?? occurredAt,
|
|
601
|
+
});
|
|
602
|
+
case 'run.started':
|
|
603
|
+
return withTiming({
|
|
604
|
+
...base,
|
|
605
|
+
playName: event.playName ?? base.playName ?? null,
|
|
606
|
+
status: isTerminalPlayRunLedgerStatus(base.status)
|
|
607
|
+
? base.status
|
|
608
|
+
: 'running',
|
|
609
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
610
|
+
});
|
|
611
|
+
case 'run.completed':
|
|
612
|
+
return (
|
|
613
|
+
conflictingTerminalSnapshot(base, event.type) ??
|
|
614
|
+
withTiming({
|
|
615
|
+
...base,
|
|
616
|
+
status: 'completed',
|
|
617
|
+
error: null,
|
|
618
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
619
|
+
finishedAt: base.finishedAt ?? occurredAt,
|
|
620
|
+
activeStepId: null,
|
|
621
|
+
activeArtifactTableNamespace: null,
|
|
622
|
+
result: event.result ?? base.result,
|
|
623
|
+
resultSummary: event.resultSummary ?? base.resultSummary,
|
|
624
|
+
})
|
|
625
|
+
);
|
|
626
|
+
case 'run.failed':
|
|
627
|
+
return (
|
|
628
|
+
conflictingTerminalSnapshot(base, event.type) ??
|
|
629
|
+
withTiming({
|
|
630
|
+
...base,
|
|
631
|
+
status: 'failed',
|
|
632
|
+
error: event.error ?? base.error ?? null,
|
|
633
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
634
|
+
finishedAt: base.finishedAt ?? occurredAt,
|
|
635
|
+
activeStepId: null,
|
|
636
|
+
activeArtifactTableNamespace: null,
|
|
637
|
+
result: event.result ?? base.result,
|
|
638
|
+
})
|
|
639
|
+
);
|
|
640
|
+
case 'run.cancelled':
|
|
641
|
+
return (
|
|
642
|
+
conflictingTerminalSnapshot(base, event.type) ??
|
|
643
|
+
withTiming({
|
|
644
|
+
...base,
|
|
645
|
+
status: 'cancelled',
|
|
646
|
+
error: event.error ?? base.error ?? null,
|
|
647
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
648
|
+
finishedAt: base.finishedAt ?? occurredAt,
|
|
649
|
+
activeStepId: null,
|
|
650
|
+
activeArtifactTableNamespace: null,
|
|
651
|
+
result: event.result ?? base.result,
|
|
652
|
+
})
|
|
653
|
+
);
|
|
654
|
+
case 'log.appended':
|
|
655
|
+
return withTiming(
|
|
656
|
+
appendLogLines(base, event.lines, {
|
|
657
|
+
...(typeof event.firstSeq === 'number'
|
|
658
|
+
? { firstSeq: event.firstSeq }
|
|
659
|
+
: {}),
|
|
660
|
+
...(event.logsTruncated === true ? { logsTruncated: true } : {}),
|
|
661
|
+
}),
|
|
662
|
+
);
|
|
663
|
+
case 'step.started': {
|
|
664
|
+
const current = base.stepsById[event.stepId];
|
|
665
|
+
const nextStep: PlayRunLedgerStepSnapshot = {
|
|
666
|
+
...(current ?? { stepId: event.stepId, status: 'running' as const }),
|
|
667
|
+
stepId: event.stepId,
|
|
668
|
+
label: event.label ?? current?.label,
|
|
669
|
+
kind: event.kind ?? current?.kind,
|
|
670
|
+
status:
|
|
671
|
+
current?.status === 'failed' || current?.status === 'completed'
|
|
672
|
+
? current.status
|
|
673
|
+
: 'running',
|
|
674
|
+
artifactTableNamespace:
|
|
675
|
+
event.artifactTableNamespace ??
|
|
676
|
+
current?.artifactTableNamespace ??
|
|
677
|
+
null,
|
|
678
|
+
startedAt: current?.startedAt ?? occurredAt,
|
|
679
|
+
updatedAt: occurredAt,
|
|
680
|
+
};
|
|
681
|
+
return withTiming({
|
|
682
|
+
...base,
|
|
683
|
+
status: isTerminalPlayRunLedgerStatus(base.status)
|
|
684
|
+
? base.status
|
|
685
|
+
: 'running',
|
|
686
|
+
startedAt: base.startedAt ?? occurredAt,
|
|
687
|
+
orderedStepIds: appendOrderedStepId(base, event.stepId),
|
|
688
|
+
stepsById: { ...base.stepsById, [event.stepId]: nextStep },
|
|
689
|
+
activeStepId:
|
|
690
|
+
nextStep.status === 'running' ? event.stepId : base.activeStepId,
|
|
691
|
+
activeArtifactTableNamespace:
|
|
692
|
+
nextStep.status === 'running'
|
|
693
|
+
? (nextStep.artifactTableNamespace ?? null)
|
|
694
|
+
: (base.activeArtifactTableNamespace ?? null),
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
case 'step.progress': {
|
|
698
|
+
const current = base.stepsById[event.stepId];
|
|
699
|
+
const progress = {
|
|
700
|
+
...(current?.progress ?? {}),
|
|
701
|
+
...event.progress,
|
|
702
|
+
updatedAt: event.progress.updatedAt ?? occurredAt,
|
|
703
|
+
};
|
|
704
|
+
const inferredStatus =
|
|
705
|
+
typeof progress.completedAt === 'number' ||
|
|
706
|
+
(typeof progress.total === 'number' &&
|
|
707
|
+
typeof progress.completed === 'number' &&
|
|
708
|
+
progress.total > 0 &&
|
|
709
|
+
progress.completed >= progress.total)
|
|
710
|
+
? 'completed'
|
|
711
|
+
: 'running';
|
|
712
|
+
const status =
|
|
713
|
+
current?.status === 'failed' ||
|
|
714
|
+
current?.status === 'skipped' ||
|
|
715
|
+
current?.status === 'completed'
|
|
716
|
+
? current.status
|
|
717
|
+
: event.status === 'running' && inferredStatus === 'completed'
|
|
718
|
+
? 'completed'
|
|
719
|
+
: (event.status ?? current?.status ?? inferredStatus);
|
|
720
|
+
const nextStep: PlayRunLedgerStepSnapshot = {
|
|
721
|
+
...(current ?? { stepId: event.stepId, status }),
|
|
722
|
+
stepId: event.stepId,
|
|
723
|
+
label: event.label ?? current?.label,
|
|
724
|
+
kind: event.kind ?? current?.kind,
|
|
725
|
+
status,
|
|
726
|
+
artifactTableNamespace:
|
|
727
|
+
progress.artifactTableNamespace ??
|
|
728
|
+
current?.artifactTableNamespace ??
|
|
729
|
+
null,
|
|
730
|
+
startedAt: current?.startedAt ?? event.progress.startedAt ?? null,
|
|
731
|
+
completedAt:
|
|
732
|
+
current?.completedAt ??
|
|
733
|
+
event.progress.completedAt ??
|
|
734
|
+
(status === 'completed'
|
|
735
|
+
? (event.progress.updatedAt ?? occurredAt)
|
|
736
|
+
: null),
|
|
737
|
+
updatedAt: occurredAt,
|
|
738
|
+
progress,
|
|
739
|
+
};
|
|
740
|
+
return withTiming({
|
|
741
|
+
...base,
|
|
742
|
+
orderedStepIds: appendOrderedStepId(base, event.stepId),
|
|
743
|
+
stepsById: { ...base.stepsById, [event.stepId]: nextStep },
|
|
744
|
+
activeStepId:
|
|
745
|
+
status === 'running'
|
|
746
|
+
? event.stepId
|
|
747
|
+
: base.activeStepId === event.stepId
|
|
748
|
+
? null
|
|
749
|
+
: base.activeStepId,
|
|
750
|
+
activeArtifactTableNamespace:
|
|
751
|
+
status === 'running'
|
|
752
|
+
? (nextStep.artifactTableNamespace ?? null)
|
|
753
|
+
: base.activeStepId === event.stepId
|
|
754
|
+
? null
|
|
755
|
+
: (base.activeArtifactTableNamespace ?? null),
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
case 'step.completed':
|
|
759
|
+
case 'step.failed':
|
|
760
|
+
case 'step.skipped': {
|
|
761
|
+
const current = base.stepsById[event.stepId];
|
|
762
|
+
const preserveFailedStatus =
|
|
763
|
+
current?.status === 'failed' && event.type !== 'step.failed';
|
|
764
|
+
const status = preserveFailedStatus
|
|
765
|
+
? 'failed'
|
|
766
|
+
: event.type === 'step.completed'
|
|
767
|
+
? 'completed'
|
|
768
|
+
: event.type === 'step.failed'
|
|
769
|
+
? 'failed'
|
|
770
|
+
: 'skipped';
|
|
771
|
+
const nextStep: PlayRunLedgerStepSnapshot = {
|
|
772
|
+
...(current ?? { stepId: event.stepId, status }),
|
|
773
|
+
stepId: event.stepId,
|
|
774
|
+
label: event.label ?? current?.label,
|
|
775
|
+
kind: event.kind ?? current?.kind,
|
|
776
|
+
status,
|
|
777
|
+
artifactTableNamespace:
|
|
778
|
+
event.artifactTableNamespace ??
|
|
779
|
+
current?.artifactTableNamespace ??
|
|
780
|
+
null,
|
|
781
|
+
startedAt: current?.startedAt ?? null,
|
|
782
|
+
completedAt:
|
|
783
|
+
status === 'skipped' || preserveFailedStatus
|
|
784
|
+
? (current?.completedAt ?? null)
|
|
785
|
+
: occurredAt,
|
|
786
|
+
updatedAt: occurredAt,
|
|
787
|
+
progress: current?.progress ?? null,
|
|
788
|
+
};
|
|
789
|
+
const runningStepIds = base.orderedStepIds.filter((stepId) => {
|
|
790
|
+
if (stepId === event.stepId) return false;
|
|
791
|
+
return base.stepsById[stepId]?.status === 'running';
|
|
792
|
+
});
|
|
793
|
+
const activeStepId = runningStepIds.at(-1) ?? null;
|
|
794
|
+
return withTiming({
|
|
795
|
+
...base,
|
|
796
|
+
orderedStepIds: appendOrderedStepId(base, event.stepId),
|
|
797
|
+
stepsById: { ...base.stepsById, [event.stepId]: nextStep },
|
|
798
|
+
activeStepId,
|
|
799
|
+
activeArtifactTableNamespace: activeStepId
|
|
800
|
+
? (base.stepsById[activeStepId]?.artifactTableNamespace ?? null)
|
|
801
|
+
: null,
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
case 'sheet.summary.updated':
|
|
805
|
+
return withTiming({
|
|
806
|
+
...base,
|
|
807
|
+
resultTableNamespace:
|
|
808
|
+
base.resultTableNamespace ?? event.tableNamespace ?? null,
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
export function reducePlayRunLedgerEvents(
|
|
814
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
815
|
+
events: readonly PlayRunLedgerEvent[],
|
|
816
|
+
): PlayRunLedgerSnapshot {
|
|
817
|
+
return events.reduce(reducePlayRunLedgerEvent, snapshot);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function progressSignature(
|
|
821
|
+
progress: PlayRunLedgerStepProgress | null | undefined,
|
|
822
|
+
): string {
|
|
823
|
+
return JSON.stringify({
|
|
824
|
+
completed: progress?.completed ?? null,
|
|
825
|
+
total: progress?.total ?? null,
|
|
826
|
+
failed: progress?.failed ?? null,
|
|
827
|
+
message: progress?.message ?? null,
|
|
828
|
+
artifactTableNamespace: progress?.artifactTableNamespace ?? null,
|
|
829
|
+
startedAt: progress?.startedAt ?? null,
|
|
830
|
+
completedAt: progress?.completedAt ?? null,
|
|
831
|
+
});
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
function terminalEventTypeForStatus(
|
|
835
|
+
status: PlayRunLedgerStatus,
|
|
836
|
+
): 'run.completed' | 'run.failed' | 'run.cancelled' | null {
|
|
837
|
+
if (status === 'completed') return 'run.completed';
|
|
838
|
+
if (
|
|
839
|
+
status === 'failed' ||
|
|
840
|
+
status === 'terminated' ||
|
|
841
|
+
status === 'timed_out'
|
|
842
|
+
) {
|
|
843
|
+
return 'run.failed';
|
|
844
|
+
}
|
|
845
|
+
if (status === 'cancelled') return 'run.cancelled';
|
|
846
|
+
return null;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
/**
|
|
850
|
+
* Slice the not-yet-sent suffix out of a producer's rotating log buffer,
|
|
851
|
+
* using positional counts instead of text comparison.
|
|
852
|
+
*
|
|
853
|
+
* `bufferTotalCount` is the count of lines the producer ever emitted on this
|
|
854
|
+
* channel (monotonic); `bufferLines` is its retained tail of those lines.
|
|
855
|
+
* `sentCount` is the producer-side cursor: lines already handed to the
|
|
856
|
+
* ledger. Returns the new lines plus the `channelOffset` of the first one,
|
|
857
|
+
* or null when there is nothing new. Lines that rotated out of the buffer
|
|
858
|
+
* before they were ever sent surface as a positive gap at ingestion (which
|
|
859
|
+
* records a loud gap marker) instead of being silently re-numbered.
|
|
860
|
+
*/
|
|
861
|
+
export function slicePositionalLogLines(input: {
|
|
862
|
+
bufferLines: readonly string[];
|
|
863
|
+
bufferTotalCount: number;
|
|
864
|
+
sentCount: number;
|
|
865
|
+
}): { lines: string[]; channelOffset: number } | null {
|
|
866
|
+
const total = Math.max(input.bufferTotalCount, input.bufferLines.length);
|
|
867
|
+
const sent = Math.max(0, input.sentCount);
|
|
868
|
+
if (total <= sent) {
|
|
869
|
+
return null;
|
|
870
|
+
}
|
|
871
|
+
const bufferFirstOffset = total - input.bufferLines.length;
|
|
872
|
+
const sendFromOffset = Math.max(sent, bufferFirstOffset);
|
|
873
|
+
const lines = input.bufferLines.slice(sendFromOffset - bufferFirstOffset);
|
|
874
|
+
if (lines.length === 0) {
|
|
875
|
+
return null;
|
|
876
|
+
}
|
|
877
|
+
return { lines: [...lines], channelOffset: sendFromOffset };
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
/**
|
|
881
|
+
* Forward producer log lines as one `log.appended` event.
|
|
882
|
+
*
|
|
883
|
+
* No text dedupe here: redundant-delivery reconciliation moved to Run Log
|
|
884
|
+
* Stream ingestion (positional `channelOffset` cursors for the worker
|
|
885
|
+
* channel, bounded tail-window text dedupe for recovery channels).
|
|
886
|
+
*/
|
|
887
|
+
export function buildPlayRunLedgerEventsFromLogLines(input: {
|
|
888
|
+
runId: string;
|
|
889
|
+
lines: readonly string[];
|
|
890
|
+
source?: PlayRunLedgerEventSource;
|
|
891
|
+
occurredAt?: number;
|
|
892
|
+
channelOffset?: number | null;
|
|
893
|
+
includeLogAppend?: boolean;
|
|
894
|
+
}): PlayRunLedgerEvent[] {
|
|
895
|
+
const source = input.source ?? 'worker';
|
|
896
|
+
const occurredAt = input.occurredAt ?? Date.now();
|
|
897
|
+
const positional =
|
|
898
|
+
typeof input.channelOffset === 'number' &&
|
|
899
|
+
input.channelOffset >= 0 &&
|
|
900
|
+
Number.isFinite(input.channelOffset);
|
|
901
|
+
// Positional batches must keep one entry per emitted line — dropping a
|
|
902
|
+
// blank would shift every later line off its channel position.
|
|
903
|
+
const newLines = positional
|
|
904
|
+
? input.lines.map((line) => line.trim() || '(blank log line)')
|
|
905
|
+
: input.lines.map((line) => line.trim()).filter((line) => line.length > 0);
|
|
906
|
+
if (newLines.length === 0 || input.includeLogAppend === false) {
|
|
907
|
+
return [];
|
|
908
|
+
}
|
|
909
|
+
return [
|
|
910
|
+
{
|
|
911
|
+
type: 'log.appended',
|
|
912
|
+
runId: input.runId,
|
|
913
|
+
source,
|
|
914
|
+
occurredAt,
|
|
915
|
+
lines: newLines,
|
|
916
|
+
...(positional ? { channelOffset: input.channelOffset! } : {}),
|
|
917
|
+
},
|
|
918
|
+
];
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
export function buildPlayRunLedgerEventsFromStatusPatch(input: {
|
|
922
|
+
patch: PlayRunLedgerStatusPatch;
|
|
923
|
+
previousSnapshot: PlayRunLedgerSnapshot;
|
|
924
|
+
now?: number;
|
|
925
|
+
source?: PlayRunLedgerEventSource;
|
|
926
|
+
}): PlayRunLedgerEvent[] {
|
|
927
|
+
const now = input.now ?? Date.now();
|
|
928
|
+
const source = input.source ?? 'worker';
|
|
929
|
+
const patch = input.patch;
|
|
930
|
+
const status = normalizePlayRunLedgerStatus(patch.status);
|
|
931
|
+
const liveProgress = normalizeLiveProgressMap(patch.liveNodeProgress);
|
|
932
|
+
const checkpointAt = finiteNumber(patch.lastCheckpointAt) ?? now;
|
|
933
|
+
const progressStartedAts = Object.values(liveProgress)
|
|
934
|
+
.map((progress) => progress.startedAt)
|
|
935
|
+
.filter((value): value is number => typeof value === 'number');
|
|
936
|
+
const progressCompletedAts = Object.values(liveProgress)
|
|
937
|
+
.map((progress) => progress.completedAt)
|
|
938
|
+
.filter((value): value is number => typeof value === 'number');
|
|
939
|
+
const existingStepStartedAts = Object.values(input.previousSnapshot.stepsById)
|
|
940
|
+
.map((step) => step.startedAt)
|
|
941
|
+
.filter((value): value is number => typeof value === 'number');
|
|
942
|
+
const existingStepCompletedAts = Object.values(
|
|
943
|
+
input.previousSnapshot.stepsById,
|
|
944
|
+
)
|
|
945
|
+
.map((step) => step.completedAt)
|
|
946
|
+
.filter((value): value is number => typeof value === 'number');
|
|
947
|
+
const logEvents = Array.isArray(patch.liveLogs)
|
|
948
|
+
? buildPlayRunLedgerEventsFromLogLines({
|
|
949
|
+
runId: patch.runId,
|
|
950
|
+
lines: patch.liveLogs,
|
|
951
|
+
source,
|
|
952
|
+
occurredAt: checkpointAt,
|
|
953
|
+
channelOffset: patch.liveLogsChannelOffset ?? null,
|
|
954
|
+
})
|
|
955
|
+
: [];
|
|
956
|
+
const earliestStartedAt =
|
|
957
|
+
progressStartedAts.length > 0 || existingStepStartedAts.length > 0
|
|
958
|
+
? Math.min(...progressStartedAts, ...existingStepStartedAts)
|
|
959
|
+
: null;
|
|
960
|
+
const latestCompletedAt =
|
|
961
|
+
progressCompletedAts.length > 0 || existingStepCompletedAts.length > 0
|
|
962
|
+
? Math.max(...progressCompletedAts, ...existingStepCompletedAts)
|
|
963
|
+
: null;
|
|
964
|
+
const events: PlayRunLedgerEvent[] = [];
|
|
965
|
+
|
|
966
|
+
if (
|
|
967
|
+
!input.previousSnapshot.startedAt &&
|
|
968
|
+
(Object.keys(liveProgress).length > 0 ||
|
|
969
|
+
status === 'running' ||
|
|
970
|
+
isTerminalPlayRunLedgerStatus(status))
|
|
971
|
+
) {
|
|
972
|
+
events.push({
|
|
973
|
+
type: 'run.started',
|
|
974
|
+
runId: patch.runId,
|
|
975
|
+
playName: patch.playName ?? input.previousSnapshot.playName ?? null,
|
|
976
|
+
source,
|
|
977
|
+
occurredAt: earliestStartedAt ?? checkpointAt,
|
|
978
|
+
runtimeBackend: patch.runtimeBackend ?? null,
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
events.push(...logEvents);
|
|
983
|
+
|
|
984
|
+
for (const [stepId, progress] of Object.entries(liveProgress)) {
|
|
985
|
+
const previousStep = input.previousSnapshot.stepsById[stepId];
|
|
986
|
+
if (progress.startedAt && !previousStep?.startedAt) {
|
|
987
|
+
events.push({
|
|
988
|
+
type: 'step.started',
|
|
989
|
+
runId: patch.runId,
|
|
990
|
+
source,
|
|
991
|
+
occurredAt: progress.startedAt,
|
|
992
|
+
stepId,
|
|
993
|
+
artifactTableNamespace: progress.artifactTableNamespace ?? null,
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
const normalizedProgress: PlayRunLedgerStepProgress = {
|
|
998
|
+
...(progress.completed !== undefined
|
|
999
|
+
? { completed: progress.completed }
|
|
1000
|
+
: {}),
|
|
1001
|
+
...(progress.total !== undefined ? { total: progress.total } : {}),
|
|
1002
|
+
...(progress.failed !== undefined ? { failed: progress.failed } : {}),
|
|
1003
|
+
...(progress.message !== undefined ? { message: progress.message } : {}),
|
|
1004
|
+
...(progress.artifactTableNamespace !== undefined
|
|
1005
|
+
? { artifactTableNamespace: progress.artifactTableNamespace }
|
|
1006
|
+
: {}),
|
|
1007
|
+
...(progress.startedAt !== undefined
|
|
1008
|
+
? { startedAt: progress.startedAt }
|
|
1009
|
+
: {}),
|
|
1010
|
+
...(progress.completedAt !== undefined
|
|
1011
|
+
? { completedAt: progress.completedAt }
|
|
1012
|
+
: {}),
|
|
1013
|
+
updatedAt: progress.updatedAt ?? checkpointAt,
|
|
1014
|
+
};
|
|
1015
|
+
if (
|
|
1016
|
+
progressSignature(normalizedProgress) !==
|
|
1017
|
+
progressSignature(previousStep?.progress)
|
|
1018
|
+
) {
|
|
1019
|
+
events.push({
|
|
1020
|
+
type: 'step.progress',
|
|
1021
|
+
runId: patch.runId,
|
|
1022
|
+
source,
|
|
1023
|
+
occurredAt: progress.updatedAt ?? checkpointAt,
|
|
1024
|
+
stepId,
|
|
1025
|
+
status:
|
|
1026
|
+
typeof progress.completedAt === 'number'
|
|
1027
|
+
? 'completed'
|
|
1028
|
+
: previousStep?.status === 'failed'
|
|
1029
|
+
? 'failed'
|
|
1030
|
+
: 'running',
|
|
1031
|
+
progress: normalizedProgress,
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
if (progress.completedAt && !previousStep?.completedAt) {
|
|
1036
|
+
events.push({
|
|
1037
|
+
type: 'step.completed',
|
|
1038
|
+
runId: patch.runId,
|
|
1039
|
+
source,
|
|
1040
|
+
occurredAt: progress.completedAt,
|
|
1041
|
+
stepId,
|
|
1042
|
+
artifactTableNamespace: progress.artifactTableNamespace ?? null,
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
const terminalType = terminalEventTypeForStatus(status);
|
|
1048
|
+
if (
|
|
1049
|
+
terminalType &&
|
|
1050
|
+
!isTerminalPlayRunLedgerStatus(input.previousSnapshot.status)
|
|
1051
|
+
) {
|
|
1052
|
+
const occurredAt = latestCompletedAt ?? checkpointAt;
|
|
1053
|
+
if (terminalType === 'run.completed') {
|
|
1054
|
+
events.push({
|
|
1055
|
+
type: terminalType,
|
|
1056
|
+
runId: patch.runId,
|
|
1057
|
+
source,
|
|
1058
|
+
occurredAt,
|
|
1059
|
+
result: patch.result,
|
|
1060
|
+
});
|
|
1061
|
+
} else {
|
|
1062
|
+
events.push({
|
|
1063
|
+
type: terminalType,
|
|
1064
|
+
runId: patch.runId,
|
|
1065
|
+
source,
|
|
1066
|
+
occurredAt,
|
|
1067
|
+
error: patch.error ?? null,
|
|
1068
|
+
result: patch.result,
|
|
1069
|
+
});
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
return events;
|
|
1074
|
+
}
|