deepline 0.3.229 → 0.3.231
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +47 -32
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +38 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +8 -3
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +39 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +293 -4
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +29 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +50 -10
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/progress-sidecar.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +187 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-work-counters.ts +52 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +30 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +118 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +135 -21
- package/dist/cli/index.js +109 -7
- package/dist/cli/index.mjs +109 -7
- package/dist/index.js +93 -2
- package/dist/index.mjs +93 -2
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -122,6 +122,28 @@ export type PlayRunLedgerDatasetSnapshot = {
|
|
|
122
122
|
updatedAt: number;
|
|
123
123
|
};
|
|
124
124
|
|
|
125
|
+
/** Fixed-size, attempt-fenced call evidence. Counts are per scheduler attempt. */
|
|
126
|
+
export type PlayRunLedgerWorkCounters = {
|
|
127
|
+
attempt: number;
|
|
128
|
+
invocationSeq: number;
|
|
129
|
+
segmentId: string;
|
|
130
|
+
logicalCalls: number;
|
|
131
|
+
providerRequests: number;
|
|
132
|
+
provider429s: number;
|
|
133
|
+
final: boolean;
|
|
134
|
+
completedLogicalCalls: number;
|
|
135
|
+
completedProviderRequests: number;
|
|
136
|
+
completedProvider429s: number;
|
|
137
|
+
completeThroughAttempt: boolean;
|
|
138
|
+
consistent: boolean;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
/** Runner invocation which actually produced the accepted terminal result. */
|
|
142
|
+
export type PlayRunLedgerTerminalWorkCounterSegment = Pick<
|
|
143
|
+
PlayRunLedgerWorkCounters,
|
|
144
|
+
'attempt' | 'invocationSeq' | 'segmentId'
|
|
145
|
+
>;
|
|
146
|
+
|
|
125
147
|
export type PlayRunLedgerSnapshot = {
|
|
126
148
|
runId: string;
|
|
127
149
|
playName?: string | null;
|
|
@@ -141,6 +163,10 @@ export type PlayRunLedgerSnapshot = {
|
|
|
141
163
|
datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
|
|
142
164
|
orderedActivityIds: string[];
|
|
143
165
|
activitiesById: Record<string, PlayActivityObservation>;
|
|
166
|
+
/** Absent for runs started before call accounting was enabled. */
|
|
167
|
+
workCounters?: PlayRunLedgerWorkCounters;
|
|
168
|
+
/** Absent on terminals produced before runner invocation evidence existed. */
|
|
169
|
+
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
144
170
|
/**
|
|
145
171
|
* Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
|
|
146
172
|
* Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
|
|
@@ -214,12 +240,14 @@ export type PlayRunLedgerEvent =
|
|
|
214
240
|
type: 'run.completed';
|
|
215
241
|
result?: unknown;
|
|
216
242
|
resultSummary?: unknown;
|
|
243
|
+
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
217
244
|
})
|
|
218
245
|
| (PlayRunLedgerBaseEvent & {
|
|
219
246
|
type: 'run.failed';
|
|
220
247
|
error?: string | null;
|
|
221
248
|
result?: unknown;
|
|
222
249
|
resultSummary?: unknown;
|
|
250
|
+
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
223
251
|
})
|
|
224
252
|
| (PlayRunLedgerBaseEvent & {
|
|
225
253
|
type: 'run.cancelled';
|
|
@@ -305,6 +333,16 @@ export type PlayRunLedgerEvent =
|
|
|
305
333
|
| (PlayRunLedgerBaseEvent & {
|
|
306
334
|
type: 'activity.observed';
|
|
307
335
|
observation: PlayActivityObservation;
|
|
336
|
+
})
|
|
337
|
+
| (PlayRunLedgerBaseEvent & {
|
|
338
|
+
type: 'work.counters';
|
|
339
|
+
attempt: number;
|
|
340
|
+
invocationSeq: number;
|
|
341
|
+
segmentId: string;
|
|
342
|
+
logicalCalls: number;
|
|
343
|
+
providerRequests: number;
|
|
344
|
+
provider429s: number;
|
|
345
|
+
final: boolean;
|
|
308
346
|
});
|
|
309
347
|
|
|
310
348
|
/**
|
|
@@ -391,6 +429,219 @@ function finiteNumber(value: unknown): number | null {
|
|
|
391
429
|
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
392
430
|
}
|
|
393
431
|
|
|
432
|
+
function nonnegativeSafeInteger(value: unknown): value is number {
|
|
433
|
+
return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function isWorkCounterSegmentId(value: unknown): value is string {
|
|
437
|
+
return (
|
|
438
|
+
typeof value === 'string' &&
|
|
439
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
|
440
|
+
value,
|
|
441
|
+
)
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
export function normalizePlayRunLedgerTerminalWorkCounterSegment(
|
|
446
|
+
value: unknown,
|
|
447
|
+
): PlayRunLedgerTerminalWorkCounterSegment | undefined {
|
|
448
|
+
if (!isRecord(value)) return undefined;
|
|
449
|
+
if (
|
|
450
|
+
!nonnegativeSafeInteger(value.attempt) ||
|
|
451
|
+
!nonnegativeSafeInteger(value.invocationSeq) ||
|
|
452
|
+
value.invocationSeq === 0 ||
|
|
453
|
+
!isWorkCounterSegmentId(value.segmentId)
|
|
454
|
+
) {
|
|
455
|
+
return undefined;
|
|
456
|
+
}
|
|
457
|
+
return {
|
|
458
|
+
attempt: value.attempt,
|
|
459
|
+
invocationSeq: value.invocationSeq,
|
|
460
|
+
segmentId: value.segmentId,
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
/** Final counts are exact only if the accepted terminal names the final report. */
|
|
465
|
+
export function hasConfirmedPlayRunLedgerWorkCounters(
|
|
466
|
+
snapshot: PlayRunLedgerSnapshot,
|
|
467
|
+
): boolean {
|
|
468
|
+
const counters = snapshot.workCounters;
|
|
469
|
+
const terminal = snapshot.terminalWorkCounterSegment;
|
|
470
|
+
return Boolean(
|
|
471
|
+
isTerminalPlayRunLedgerStatus(snapshot.status) &&
|
|
472
|
+
(snapshot.terminalSource === 'worker' ||
|
|
473
|
+
snapshot.terminalSource === 'system') &&
|
|
474
|
+
counters?.final &&
|
|
475
|
+
counters.completeThroughAttempt &&
|
|
476
|
+
counters.consistent &&
|
|
477
|
+
terminal &&
|
|
478
|
+
counters.attempt === terminal.attempt &&
|
|
479
|
+
counters.invocationSeq === terminal.invocationSeq &&
|
|
480
|
+
counters.segmentId === terminal.segmentId &&
|
|
481
|
+
Number.isSafeInteger(
|
|
482
|
+
counters.completedLogicalCalls + counters.logicalCalls,
|
|
483
|
+
) &&
|
|
484
|
+
Number.isSafeInteger(
|
|
485
|
+
counters.completedProviderRequests + counters.providerRequests,
|
|
486
|
+
) &&
|
|
487
|
+
Number.isSafeInteger(
|
|
488
|
+
counters.completedProvider429s + counters.provider429s,
|
|
489
|
+
),
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function normalizeWorkCounters(
|
|
494
|
+
value: unknown,
|
|
495
|
+
): PlayRunLedgerWorkCounters | undefined {
|
|
496
|
+
if (!isRecord(value)) return undefined;
|
|
497
|
+
for (const field of [
|
|
498
|
+
'attempt',
|
|
499
|
+
'invocationSeq',
|
|
500
|
+
'logicalCalls',
|
|
501
|
+
'providerRequests',
|
|
502
|
+
'provider429s',
|
|
503
|
+
'completedLogicalCalls',
|
|
504
|
+
'completedProviderRequests',
|
|
505
|
+
'completedProvider429s',
|
|
506
|
+
] as const) {
|
|
507
|
+
if (!nonnegativeSafeInteger(value[field])) return undefined;
|
|
508
|
+
}
|
|
509
|
+
if (
|
|
510
|
+
value.invocationSeq === 0 ||
|
|
511
|
+
!isWorkCounterSegmentId(value.segmentId) ||
|
|
512
|
+
typeof value.final !== 'boolean' ||
|
|
513
|
+
typeof value.completeThroughAttempt !== 'boolean' ||
|
|
514
|
+
typeof value.consistent !== 'boolean'
|
|
515
|
+
) {
|
|
516
|
+
return undefined;
|
|
517
|
+
}
|
|
518
|
+
return {
|
|
519
|
+
...value,
|
|
520
|
+
attempt: value.attempt as number,
|
|
521
|
+
invocationSeq: value.invocationSeq as number,
|
|
522
|
+
segmentId: value.segmentId,
|
|
523
|
+
logicalCalls: value.logicalCalls as number,
|
|
524
|
+
providerRequests: value.providerRequests as number,
|
|
525
|
+
provider429s: value.provider429s as number,
|
|
526
|
+
final: value.final,
|
|
527
|
+
completedLogicalCalls: value.completedLogicalCalls as number,
|
|
528
|
+
completedProviderRequests: value.completedProviderRequests as number,
|
|
529
|
+
completedProvider429s: value.completedProvider429s as number,
|
|
530
|
+
completeThroughAttempt: value.completeThroughAttempt,
|
|
531
|
+
consistent: value.consistent,
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function advanceNextWorkCounterSegment(
|
|
536
|
+
current: PlayRunLedgerWorkCounters,
|
|
537
|
+
event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
|
|
538
|
+
): PlayRunLedgerWorkCounters {
|
|
539
|
+
const completedLogicalCalls =
|
|
540
|
+
current.completedLogicalCalls + current.logicalCalls;
|
|
541
|
+
const completedProviderRequests =
|
|
542
|
+
current.completedProviderRequests + current.providerRequests;
|
|
543
|
+
const completedProvider429s =
|
|
544
|
+
current.completedProvider429s + current.provider429s;
|
|
545
|
+
const safe = [
|
|
546
|
+
completedLogicalCalls,
|
|
547
|
+
completedProviderRequests,
|
|
548
|
+
completedProvider429s,
|
|
549
|
+
].every(Number.isSafeInteger);
|
|
550
|
+
const contiguous =
|
|
551
|
+
event.attempt === current.attempt
|
|
552
|
+
? event.invocationSeq === current.invocationSeq + 1
|
|
553
|
+
: event.attempt === current.attempt + 1 && event.invocationSeq === 1;
|
|
554
|
+
return {
|
|
555
|
+
attempt: event.attempt,
|
|
556
|
+
invocationSeq: event.invocationSeq,
|
|
557
|
+
segmentId: event.segmentId,
|
|
558
|
+
logicalCalls: event.logicalCalls,
|
|
559
|
+
providerRequests: event.providerRequests,
|
|
560
|
+
provider429s: event.provider429s,
|
|
561
|
+
final: event.final === true,
|
|
562
|
+
completedLogicalCalls: Math.min(
|
|
563
|
+
completedLogicalCalls,
|
|
564
|
+
Number.MAX_SAFE_INTEGER,
|
|
565
|
+
),
|
|
566
|
+
completedProviderRequests: Math.min(
|
|
567
|
+
completedProviderRequests,
|
|
568
|
+
Number.MAX_SAFE_INTEGER,
|
|
569
|
+
),
|
|
570
|
+
completedProvider429s: Math.min(
|
|
571
|
+
completedProvider429s,
|
|
572
|
+
Number.MAX_SAFE_INTEGER,
|
|
573
|
+
),
|
|
574
|
+
completeThroughAttempt:
|
|
575
|
+
current.completeThroughAttempt && current.final && contiguous,
|
|
576
|
+
consistent:
|
|
577
|
+
current.consistent && safe && event.segmentId !== current.segmentId,
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function advanceWorkCounters(
|
|
582
|
+
current: PlayRunLedgerWorkCounters | undefined,
|
|
583
|
+
event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
|
|
584
|
+
): PlayRunLedgerWorkCounters | undefined {
|
|
585
|
+
if (
|
|
586
|
+
!nonnegativeSafeInteger(event.attempt) ||
|
|
587
|
+
!nonnegativeSafeInteger(event.invocationSeq) ||
|
|
588
|
+
event.invocationSeq === 0 ||
|
|
589
|
+
!isWorkCounterSegmentId(event.segmentId) ||
|
|
590
|
+
!nonnegativeSafeInteger(event.logicalCalls) ||
|
|
591
|
+
!nonnegativeSafeInteger(event.providerRequests) ||
|
|
592
|
+
!nonnegativeSafeInteger(event.provider429s)
|
|
593
|
+
) {
|
|
594
|
+
return current;
|
|
595
|
+
}
|
|
596
|
+
if (!current) {
|
|
597
|
+
return {
|
|
598
|
+
attempt: event.attempt,
|
|
599
|
+
invocationSeq: event.invocationSeq,
|
|
600
|
+
segmentId: event.segmentId,
|
|
601
|
+
logicalCalls: event.logicalCalls,
|
|
602
|
+
providerRequests: event.providerRequests,
|
|
603
|
+
provider429s: event.provider429s,
|
|
604
|
+
final: event.final === true,
|
|
605
|
+
completedLogicalCalls: 0,
|
|
606
|
+
completedProviderRequests: 0,
|
|
607
|
+
completedProvider429s: 0,
|
|
608
|
+
completeThroughAttempt: event.attempt <= 1 && event.invocationSeq === 1,
|
|
609
|
+
consistent: true,
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
if (event.attempt < current.attempt) return current;
|
|
613
|
+
if (event.attempt === current.attempt) {
|
|
614
|
+
if (event.invocationSeq < current.invocationSeq) return current;
|
|
615
|
+
if (event.invocationSeq > current.invocationSeq)
|
|
616
|
+
return advanceNextWorkCounterSegment(current, event);
|
|
617
|
+
const counters = [
|
|
618
|
+
['logicalCalls', event.logicalCalls],
|
|
619
|
+
['providerRequests', event.providerRequests],
|
|
620
|
+
['provider429s', event.provider429s],
|
|
621
|
+
] as const;
|
|
622
|
+
const finalDisagrees = counters.some(
|
|
623
|
+
([field, value]) =>
|
|
624
|
+
(event.final && value < current[field]) ||
|
|
625
|
+
(current.final && value > current[field]),
|
|
626
|
+
);
|
|
627
|
+
return {
|
|
628
|
+
...current,
|
|
629
|
+
logicalCalls: Math.max(current.logicalCalls, event.logicalCalls),
|
|
630
|
+
providerRequests: Math.max(
|
|
631
|
+
current.providerRequests,
|
|
632
|
+
event.providerRequests,
|
|
633
|
+
),
|
|
634
|
+
provider429s: Math.max(current.provider429s, event.provider429s),
|
|
635
|
+
final: current.final || event.final === true,
|
|
636
|
+
consistent:
|
|
637
|
+
current.consistent &&
|
|
638
|
+
current.segmentId === event.segmentId &&
|
|
639
|
+
!finalDisagrees,
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
return advanceNextWorkCounterSegment(current, event);
|
|
643
|
+
}
|
|
644
|
+
|
|
394
645
|
function optionalFiniteNumber(value: unknown): number | undefined {
|
|
395
646
|
const normalized = finiteNumber(value);
|
|
396
647
|
return normalized === null ? undefined : normalized;
|
|
@@ -679,6 +930,11 @@ export function normalizePlayRunLedgerSnapshot(
|
|
|
679
930
|
const logTail = Array.isArray(rawTail)
|
|
680
931
|
? rawTail.filter((line): line is string => typeof line === 'string')
|
|
681
932
|
: [];
|
|
933
|
+
const workCounters = normalizeWorkCounters(value.workCounters);
|
|
934
|
+
const terminalWorkCounterSegment =
|
|
935
|
+
normalizePlayRunLedgerTerminalWorkCounterSegment(
|
|
936
|
+
value.terminalWorkCounterSegment,
|
|
937
|
+
);
|
|
682
938
|
return {
|
|
683
939
|
runId: optionalString(value.runId) ?? fallback.runId,
|
|
684
940
|
playName:
|
|
@@ -713,6 +969,8 @@ export function normalizePlayRunLedgerSnapshot(
|
|
|
713
969
|
typeof activityId === 'string' && Boolean(activitiesById[activityId]),
|
|
714
970
|
),
|
|
715
971
|
activitiesById,
|
|
972
|
+
...(workCounters ? { workCounters } : {}),
|
|
973
|
+
...(terminalWorkCounterSegment ? { terminalWorkCounterSegment } : {}),
|
|
716
974
|
logTail: logTail.slice(-LOG_TAIL_LIMIT),
|
|
717
975
|
// Snapshots persisted before totalLogCount existed only know the retained
|
|
718
976
|
// tail, so the best lower bound for the cumulative count is the tail size.
|
|
@@ -1283,6 +1541,11 @@ export function reducePlayRunLedgerEvent(
|
|
|
1283
1541
|
shouldIgnoreNonTerminalAfterStickyTerminal(base);
|
|
1284
1542
|
|
|
1285
1543
|
switch (event.type) {
|
|
1544
|
+
case 'work.counters': {
|
|
1545
|
+
const workCounters = advanceWorkCounters(base.workCounters, event);
|
|
1546
|
+
if (workCounters === base.workCounters) return snapshot;
|
|
1547
|
+
return withTiming({ ...base, workCounters });
|
|
1548
|
+
}
|
|
1286
1549
|
case 'run.created': {
|
|
1287
1550
|
if (ignoreNonTerminalAfterStickyTerminal) {
|
|
1288
1551
|
return withTiming(base);
|
|
@@ -1333,14 +1596,24 @@ export function reducePlayRunLedgerEvent(
|
|
|
1333
1596
|
waitUntil: null,
|
|
1334
1597
|
activeActivityId: null,
|
|
1335
1598
|
});
|
|
1336
|
-
case 'run.completed':
|
|
1599
|
+
case 'run.completed': {
|
|
1600
|
+
const terminalWorkCounterSegment =
|
|
1601
|
+
(event.source === 'worker' || event.source === 'system') &&
|
|
1602
|
+
(!isTerminalPlayRunLedgerStatus(base.status) ||
|
|
1603
|
+
base.terminalSource === 'coordinator')
|
|
1604
|
+
? normalizePlayRunLedgerTerminalWorkCounterSegment(
|
|
1605
|
+
event.terminalWorkCounterSegment,
|
|
1606
|
+
)
|
|
1607
|
+
: undefined;
|
|
1337
1608
|
return (
|
|
1338
1609
|
conflictingTerminalSnapshot(base, event.type, occurredAt, null, {
|
|
1339
1610
|
allowStaleSameStatusPayloadMerge:
|
|
1340
1611
|
(base.result === undefined && event.result !== undefined) ||
|
|
1341
1612
|
canPromoteTerminalResultReference(base.result, event.result) ||
|
|
1342
1613
|
(base.resultSummary === undefined &&
|
|
1343
|
-
event.resultSummary !== undefined)
|
|
1614
|
+
event.resultSummary !== undefined) ||
|
|
1615
|
+
(base.terminalWorkCounterSegment === undefined &&
|
|
1616
|
+
terminalWorkCounterSegment !== undefined),
|
|
1344
1617
|
}) ??
|
|
1345
1618
|
withTiming({
|
|
1346
1619
|
...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
|
|
@@ -1358,10 +1631,21 @@ export function reducePlayRunLedgerEvent(
|
|
|
1358
1631
|
next: event.result,
|
|
1359
1632
|
}),
|
|
1360
1633
|
resultSummary: event.resultSummary ?? base.resultSummary,
|
|
1634
|
+
terminalWorkCounterSegment:
|
|
1635
|
+
base.terminalWorkCounterSegment ?? terminalWorkCounterSegment,
|
|
1361
1636
|
terminalSource: nextTerminalSource(base, event.source),
|
|
1362
1637
|
})
|
|
1363
1638
|
);
|
|
1364
|
-
|
|
1639
|
+
}
|
|
1640
|
+
case 'run.failed': {
|
|
1641
|
+
const terminalWorkCounterSegment =
|
|
1642
|
+
(event.source === 'worker' || event.source === 'system') &&
|
|
1643
|
+
(!isTerminalPlayRunLedgerStatus(base.status) ||
|
|
1644
|
+
base.terminalSource === 'coordinator')
|
|
1645
|
+
? normalizePlayRunLedgerTerminalWorkCounterSegment(
|
|
1646
|
+
event.terminalWorkCounterSegment,
|
|
1647
|
+
)
|
|
1648
|
+
: undefined;
|
|
1365
1649
|
return (
|
|
1366
1650
|
retryablePlatformDeployFailureSnapshot(base, event.error) ??
|
|
1367
1651
|
conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
|
|
@@ -1370,7 +1654,9 @@ export function reducePlayRunLedgerEvent(
|
|
|
1370
1654
|
(base.result === undefined && event.result !== undefined) ||
|
|
1371
1655
|
(base.resultSummary === undefined &&
|
|
1372
1656
|
event.resultSummary !== undefined) ||
|
|
1373
|
-
canPromoteTerminalResultReference(base.result, event.result)
|
|
1657
|
+
canPromoteTerminalResultReference(base.result, event.result) ||
|
|
1658
|
+
(base.terminalWorkCounterSegment === undefined &&
|
|
1659
|
+
terminalWorkCounterSegment !== undefined),
|
|
1374
1660
|
}) ??
|
|
1375
1661
|
withTiming({
|
|
1376
1662
|
...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
|
|
@@ -1391,9 +1677,12 @@ export function reducePlayRunLedgerEvent(
|
|
|
1391
1677
|
next: event.result,
|
|
1392
1678
|
}),
|
|
1393
1679
|
resultSummary: event.resultSummary ?? base.resultSummary,
|
|
1680
|
+
terminalWorkCounterSegment:
|
|
1681
|
+
base.terminalWorkCounterSegment ?? terminalWorkCounterSegment,
|
|
1394
1682
|
terminalSource: nextTerminalSource(base, event.source),
|
|
1395
1683
|
})
|
|
1396
1684
|
);
|
|
1685
|
+
}
|
|
1397
1686
|
case 'run.cancelled':
|
|
1398
1687
|
return (
|
|
1399
1688
|
conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { resolveTimingWindow } from './live-events';
|
|
2
2
|
import {
|
|
3
3
|
LOG_TAIL_LIMIT,
|
|
4
|
+
hasConfirmedPlayRunLedgerWorkCounters,
|
|
4
5
|
normalizePlayRunLedgerSnapshot,
|
|
5
6
|
summarizeRunRowOutcomes,
|
|
6
7
|
type PlayRunLedgerSnapshot,
|
|
@@ -110,6 +111,13 @@ export type PlayRunLiveSnapshot = {
|
|
|
110
111
|
* without row outcomes.
|
|
111
112
|
*/
|
|
112
113
|
rowOutcomes?: PlayRunRowOutcomeSummary;
|
|
114
|
+
/** Observed run-wide call attempts. Absent for runs without counter evidence. */
|
|
115
|
+
calls?: {
|
|
116
|
+
logicalCalls: number;
|
|
117
|
+
providerRequests: number;
|
|
118
|
+
provider429s: number;
|
|
119
|
+
confirmed: boolean;
|
|
120
|
+
};
|
|
113
121
|
};
|
|
114
122
|
|
|
115
123
|
export function normalizePlayRunLiveStatus(value: unknown): PlayRunLiveStatus {
|
|
@@ -236,6 +244,21 @@ function buildSnapshotFromLedger(
|
|
|
236
244
|
(terminalRowOutcomes.supersededRows ?? 0) > 0)
|
|
237
245
|
? terminalRowOutcomes
|
|
238
246
|
: null;
|
|
247
|
+
const workCounters = snapshot.workCounters;
|
|
248
|
+
const calls = workCounters
|
|
249
|
+
? {
|
|
250
|
+
logicalCalls:
|
|
251
|
+
workCounters.completedLogicalCalls + workCounters.logicalCalls,
|
|
252
|
+
providerRequests:
|
|
253
|
+
workCounters.completedProviderRequests +
|
|
254
|
+
workCounters.providerRequests,
|
|
255
|
+
provider429s:
|
|
256
|
+
workCounters.completedProvider429s + workCounters.provider429s,
|
|
257
|
+
confirmed:
|
|
258
|
+
isTerminalPlayRunLiveStatus(liveStatus) &&
|
|
259
|
+
hasConfirmedPlayRunLedgerWorkCounters(snapshot),
|
|
260
|
+
}
|
|
261
|
+
: null;
|
|
239
262
|
return {
|
|
240
263
|
runId: snapshot.runId,
|
|
241
264
|
status: liveStatus,
|
|
@@ -268,6 +291,12 @@ function buildSnapshotFromLedger(
|
|
|
268
291
|
nodeStates,
|
|
269
292
|
activeNodeId: snapshot.activeStepId ?? null,
|
|
270
293
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
294
|
+
...(calls &&
|
|
295
|
+
Object.values(calls).every(
|
|
296
|
+
(value) => typeof value === 'boolean' || Number.isSafeInteger(value),
|
|
297
|
+
)
|
|
298
|
+
? { calls }
|
|
299
|
+
: {}),
|
|
271
300
|
};
|
|
272
301
|
}
|
|
273
302
|
|
|
@@ -2,7 +2,7 @@ import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
|
|
|
2
2
|
import { readFile } from 'node:fs/promises';
|
|
3
3
|
import { dirname, isAbsolute, resolve } from 'node:path';
|
|
4
4
|
import { createRequire } from 'node:module';
|
|
5
|
-
import { createHash } from 'node:crypto';
|
|
5
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
6
6
|
import { formatWithOptions } from 'node:util';
|
|
7
7
|
import { compileFunction } from 'node:vm';
|
|
8
8
|
import { gunzipSync } from 'node:zlib';
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
} from '../run-failure';
|
|
43
43
|
import { createPlayContext } from '../context';
|
|
44
44
|
import { augmentSheetContractWithDatasetFields } from '../../play-data-plane/sheet-contract';
|
|
45
|
+
import { nextWorkCounterInvocationSeq } from '../runtime-work-counters';
|
|
45
46
|
import type { RuntimeApiResponseMeta } from '../app-runtime-api';
|
|
46
47
|
import { onMemoryPressure } from '../memory-pressure-signal';
|
|
47
48
|
import {
|
|
@@ -1894,6 +1895,16 @@ async function run(
|
|
|
1894
1895
|
options?: ExecutePlayRunnerOptions,
|
|
1895
1896
|
): Promise<PlayRunnerResult> {
|
|
1896
1897
|
const runStartedAt = Date.now();
|
|
1898
|
+
const runnerRunAttempt =
|
|
1899
|
+
typeof config.context.runAttempt === 'number' &&
|
|
1900
|
+
Number.isFinite(config.context.runAttempt)
|
|
1901
|
+
? Math.max(0, Math.floor(config.context.runAttempt))
|
|
1902
|
+
: 0;
|
|
1903
|
+
const workCounterInvocationSeq = nextWorkCounterInvocationSeq(
|
|
1904
|
+
config.checkpoint,
|
|
1905
|
+
runnerRunAttempt,
|
|
1906
|
+
);
|
|
1907
|
+
const workCounterSegmentId = randomUUID();
|
|
1897
1908
|
const persistenceSignal =
|
|
1898
1909
|
options?.signal ??
|
|
1899
1910
|
AbortSignal.timeout(
|
|
@@ -2010,11 +2021,6 @@ async function run(
|
|
|
2010
2021
|
runId: string;
|
|
2011
2022
|
tableNamespace: string;
|
|
2012
2023
|
}) => `${input.playName}\u0000${input.runId}\u0000${input.tableNamespace}`;
|
|
2013
|
-
const runnerRunAttempt =
|
|
2014
|
-
typeof config.context.runAttempt === 'number' &&
|
|
2015
|
-
Number.isFinite(config.context.runAttempt)
|
|
2016
|
-
? Math.max(0, Math.floor(config.context.runAttempt))
|
|
2017
|
-
: 0;
|
|
2018
2024
|
const cachePolicy = playRunnerContextCachePolicy(config.context);
|
|
2019
2025
|
const rateStateBackend = createRateStateBackendForConfig(config);
|
|
2020
2026
|
const runtimeInput = config.input ?? {};
|
|
@@ -2980,6 +2986,13 @@ async function run(
|
|
|
2980
2986
|
);
|
|
2981
2987
|
}
|
|
2982
2988
|
};
|
|
2989
|
+
const runtimeStats = () => ({
|
|
2990
|
+
...ctx.getStats(),
|
|
2991
|
+
runtimeWorkCounterSegment: {
|
|
2992
|
+
invocationSeq: workCounterInvocationSeq,
|
|
2993
|
+
segmentId: workCounterSegmentId,
|
|
2994
|
+
},
|
|
2995
|
+
});
|
|
2983
2996
|
options?.onRuntimeReady?.({
|
|
2984
2997
|
log: (line) => ctx.logRunnerDiagnostic(line),
|
|
2985
2998
|
timeoutResult: (runtimeLimitSeconds) => {
|
|
@@ -2989,7 +3002,7 @@ async function run(
|
|
|
2989
3002
|
error: runtimeLimitExceededCause(runtimeLimitSeconds),
|
|
2990
3003
|
errors: [runtimeLimitExceededFailure(runtimeLimitSeconds)],
|
|
2991
3004
|
logs: terminalLogsWithPhaseSummary(ctx),
|
|
2992
|
-
stats:
|
|
3005
|
+
stats: runtimeStats(),
|
|
2993
3006
|
steps: ctx.getSteps(),
|
|
2994
3007
|
tableNamespace: null,
|
|
2995
3008
|
totalRows,
|
|
@@ -3007,6 +3020,30 @@ async function run(
|
|
|
3007
3020
|
}
|
|
3008
3021
|
: runtimeInput;
|
|
3009
3022
|
|
|
3023
|
+
// Sample scalar counters, not the potentially large resource snapshot. The
|
|
3024
|
+
// reporter coalesces transport updates; this timer emits no event if no
|
|
3025
|
+
// counter changed and cannot hold the runner process open.
|
|
3026
|
+
let lastWorkCounterKey: string | null = null;
|
|
3027
|
+
const emitWorkCounters = () => {
|
|
3028
|
+
const counters = ctx.getRuntimeWorkCounters();
|
|
3029
|
+
const key = `${counters.logicalCalls}:${counters.providerRequests}:${counters.provider429s}`;
|
|
3030
|
+
if (key === lastWorkCounterKey) return;
|
|
3031
|
+
lastWorkCounterKey = key;
|
|
3032
|
+
emitRunnerEvent(
|
|
3033
|
+
{
|
|
3034
|
+
type: 'work_counters',
|
|
3035
|
+
attempt: runnerRunAttempt,
|
|
3036
|
+
invocationSeq: workCounterInvocationSeq,
|
|
3037
|
+
segmentId: workCounterSegmentId,
|
|
3038
|
+
...counters,
|
|
3039
|
+
},
|
|
3040
|
+
options,
|
|
3041
|
+
);
|
|
3042
|
+
};
|
|
3043
|
+
emitWorkCounters();
|
|
3044
|
+
const workCounterTimer = setInterval(emitWorkCounters, 1_500);
|
|
3045
|
+
workCounterTimer.unref();
|
|
3046
|
+
|
|
3010
3047
|
try {
|
|
3011
3048
|
emitWorkerLog('Play started', options);
|
|
3012
3049
|
const rawResult = await executeRuntimePlayCode({
|
|
@@ -3210,7 +3247,7 @@ async function run(
|
|
|
3210
3247
|
logs,
|
|
3211
3248
|
...(runtimeDiagnostics.length > 0 ? { runtimeDiagnostics } : {}),
|
|
3212
3249
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
3213
|
-
stats:
|
|
3250
|
+
stats: runtimeStats(),
|
|
3214
3251
|
steps: ctx.getSteps(),
|
|
3215
3252
|
checkpoint,
|
|
3216
3253
|
tableNamespace: resultTableNamespace,
|
|
@@ -3262,7 +3299,7 @@ async function run(
|
|
|
3262
3299
|
status: 'suspended',
|
|
3263
3300
|
suspension: error.suspension,
|
|
3264
3301
|
logs: terminalLogsWithPhaseSummary(ctx),
|
|
3265
|
-
stats:
|
|
3302
|
+
stats: runtimeStats(),
|
|
3266
3303
|
steps: ctx.getSteps(),
|
|
3267
3304
|
checkpoint: ctx.getCheckpoint(),
|
|
3268
3305
|
tableNamespace: null,
|
|
@@ -3298,7 +3335,7 @@ async function run(
|
|
|
3298
3335
|
errors: [failure],
|
|
3299
3336
|
logs: terminalLogsWithPhaseSummary(ctx),
|
|
3300
3337
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
3301
|
-
stats:
|
|
3338
|
+
stats: runtimeStats(),
|
|
3302
3339
|
steps: ctx.getSteps(),
|
|
3303
3340
|
checkpoint,
|
|
3304
3341
|
tableNamespace: null,
|
|
@@ -3306,6 +3343,9 @@ async function run(
|
|
|
3306
3343
|
inserted,
|
|
3307
3344
|
skipped,
|
|
3308
3345
|
};
|
|
3346
|
+
} finally {
|
|
3347
|
+
clearInterval(workCounterTimer);
|
|
3348
|
+
emitWorkCounters();
|
|
3309
3349
|
}
|
|
3310
3350
|
}
|
|
3311
3351
|
|
|
@@ -24,6 +24,7 @@ function shouldEmitProgressSidecarLog(line: string): boolean {
|
|
|
24
24
|
export function shouldEmitProgressSidecarEvent(
|
|
25
25
|
event: PlayRunnerEvent,
|
|
26
26
|
): boolean {
|
|
27
|
+
if (event.type === 'work_counters') return true;
|
|
27
28
|
if (event.type === 'execution_event') {
|
|
28
29
|
return event.event.type !== 'map.row.updated';
|
|
29
30
|
}
|
package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts
CHANGED
|
@@ -509,6 +509,10 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
509
509
|
}
|
|
510
510
|
if (event.type === 'execution_event') {
|
|
511
511
|
callbacks?.onExecutionEvent?.(event.event);
|
|
512
|
+
return;
|
|
513
|
+
}
|
|
514
|
+
if (event.type === 'work_counters') {
|
|
515
|
+
callbacks?.onWorkCounters?.(event);
|
|
512
516
|
}
|
|
513
517
|
},
|
|
514
518
|
},
|
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
} from '../ctx-types';
|
|
6
6
|
import type {
|
|
7
7
|
PlayRunnerExecutionConfig,
|
|
8
|
+
PlayRunnerEvent,
|
|
8
9
|
PlayRunnerLogEvent,
|
|
9
10
|
PlayRunnerResult,
|
|
10
11
|
} from '../protocol';
|
|
@@ -155,6 +156,9 @@ export interface PlayRunnerCallbacks {
|
|
|
155
156
|
onCheckpoint?: (checkpoint: PlayCheckpoint) => void;
|
|
156
157
|
onRowUpdate?: (update: PlayRowUpdate) => void;
|
|
157
158
|
onExecutionEvent?: (event: PlayExecutionEvent) => void;
|
|
159
|
+
onWorkCounters?: (
|
|
160
|
+
event: Extract<PlayRunnerEvent, { type: 'work_counters' }>,
|
|
161
|
+
) => void;
|
|
158
162
|
onRuntimeResourceAcquired?: (
|
|
159
163
|
resource: PlayRunnerRuntimeResource,
|
|
160
164
|
) => void | Promise<void>;
|