deepline 0.3.229 → 0.3.231

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
  3. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +19 -0
  4. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +47 -32
  5. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +3 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +38 -2
  7. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts +8 -3
  8. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +9 -0
  9. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +39 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +293 -4
  11. package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +29 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +50 -10
  13. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/progress-sidecar.ts +1 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +4 -0
  15. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +4 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +187 -14
  17. package/dist/bundling-sources/shared_libs/play-runtime/runtime-work-counters.ts +52 -2
  18. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +5 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +30 -2
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +118 -6
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +135 -21
  22. package/dist/cli/index.js +109 -7
  23. package/dist/cli/index.mjs +109 -7
  24. package/dist/index.js +93 -2
  25. package/dist/index.mjs +93 -2
  26. package/dist/release.d.mts +1 -1
  27. package/dist/release.d.ts +1 -1
  28. package/dist/release.js +1 -1
  29. package/dist/release.mjs +1 -1
  30. package/package.json +1 -1
@@ -122,6 +122,28 @@ export type PlayRunLedgerDatasetSnapshot = {
122
122
  updatedAt: number;
123
123
  };
124
124
 
125
+ /** Fixed-size, attempt-fenced call evidence. Counts are per scheduler attempt. */
126
+ export type PlayRunLedgerWorkCounters = {
127
+ attempt: number;
128
+ invocationSeq: number;
129
+ segmentId: string;
130
+ logicalCalls: number;
131
+ providerRequests: number;
132
+ provider429s: number;
133
+ final: boolean;
134
+ completedLogicalCalls: number;
135
+ completedProviderRequests: number;
136
+ completedProvider429s: number;
137
+ completeThroughAttempt: boolean;
138
+ consistent: boolean;
139
+ };
140
+
141
+ /** Runner invocation which actually produced the accepted terminal result. */
142
+ export type PlayRunLedgerTerminalWorkCounterSegment = Pick<
143
+ PlayRunLedgerWorkCounters,
144
+ 'attempt' | 'invocationSeq' | 'segmentId'
145
+ >;
146
+
125
147
  export type PlayRunLedgerSnapshot = {
126
148
  runId: string;
127
149
  playName?: string | null;
@@ -141,6 +163,10 @@ export type PlayRunLedgerSnapshot = {
141
163
  datasetsById: Record<string, PlayRunLedgerDatasetSnapshot>;
142
164
  orderedActivityIds: string[];
143
165
  activitiesById: Record<string, PlayActivityObservation>;
166
+ /** Absent for runs started before call accounting was enabled. */
167
+ workCounters?: PlayRunLedgerWorkCounters;
168
+ /** Absent on terminals produced before runner invocation evidence existed. */
169
+ terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
144
170
  /**
145
171
  * Bounded tail of the run's log lines (last {@link LOG_TAIL_LIMIT}).
146
172
  * Full retention lives in the Run Log Stream (Convex `playRunLogChunks`);
@@ -214,12 +240,14 @@ export type PlayRunLedgerEvent =
214
240
  type: 'run.completed';
215
241
  result?: unknown;
216
242
  resultSummary?: unknown;
243
+ terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
217
244
  })
218
245
  | (PlayRunLedgerBaseEvent & {
219
246
  type: 'run.failed';
220
247
  error?: string | null;
221
248
  result?: unknown;
222
249
  resultSummary?: unknown;
250
+ terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
223
251
  })
224
252
  | (PlayRunLedgerBaseEvent & {
225
253
  type: 'run.cancelled';
@@ -305,6 +333,16 @@ export type PlayRunLedgerEvent =
305
333
  | (PlayRunLedgerBaseEvent & {
306
334
  type: 'activity.observed';
307
335
  observation: PlayActivityObservation;
336
+ })
337
+ | (PlayRunLedgerBaseEvent & {
338
+ type: 'work.counters';
339
+ attempt: number;
340
+ invocationSeq: number;
341
+ segmentId: string;
342
+ logicalCalls: number;
343
+ providerRequests: number;
344
+ provider429s: number;
345
+ final: boolean;
308
346
  });
309
347
 
310
348
  /**
@@ -391,6 +429,219 @@ function finiteNumber(value: unknown): number | null {
391
429
  return typeof value === 'number' && Number.isFinite(value) ? value : null;
392
430
  }
393
431
 
432
+ function nonnegativeSafeInteger(value: unknown): value is number {
433
+ return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
434
+ }
435
+
436
+ function isWorkCounterSegmentId(value: unknown): value is string {
437
+ return (
438
+ typeof value === 'string' &&
439
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
440
+ value,
441
+ )
442
+ );
443
+ }
444
+
445
+ export function normalizePlayRunLedgerTerminalWorkCounterSegment(
446
+ value: unknown,
447
+ ): PlayRunLedgerTerminalWorkCounterSegment | undefined {
448
+ if (!isRecord(value)) return undefined;
449
+ if (
450
+ !nonnegativeSafeInteger(value.attempt) ||
451
+ !nonnegativeSafeInteger(value.invocationSeq) ||
452
+ value.invocationSeq === 0 ||
453
+ !isWorkCounterSegmentId(value.segmentId)
454
+ ) {
455
+ return undefined;
456
+ }
457
+ return {
458
+ attempt: value.attempt,
459
+ invocationSeq: value.invocationSeq,
460
+ segmentId: value.segmentId,
461
+ };
462
+ }
463
+
464
+ /** Final counts are exact only if the accepted terminal names the final report. */
465
+ export function hasConfirmedPlayRunLedgerWorkCounters(
466
+ snapshot: PlayRunLedgerSnapshot,
467
+ ): boolean {
468
+ const counters = snapshot.workCounters;
469
+ const terminal = snapshot.terminalWorkCounterSegment;
470
+ return Boolean(
471
+ isTerminalPlayRunLedgerStatus(snapshot.status) &&
472
+ (snapshot.terminalSource === 'worker' ||
473
+ snapshot.terminalSource === 'system') &&
474
+ counters?.final &&
475
+ counters.completeThroughAttempt &&
476
+ counters.consistent &&
477
+ terminal &&
478
+ counters.attempt === terminal.attempt &&
479
+ counters.invocationSeq === terminal.invocationSeq &&
480
+ counters.segmentId === terminal.segmentId &&
481
+ Number.isSafeInteger(
482
+ counters.completedLogicalCalls + counters.logicalCalls,
483
+ ) &&
484
+ Number.isSafeInteger(
485
+ counters.completedProviderRequests + counters.providerRequests,
486
+ ) &&
487
+ Number.isSafeInteger(
488
+ counters.completedProvider429s + counters.provider429s,
489
+ ),
490
+ );
491
+ }
492
+
493
+ function normalizeWorkCounters(
494
+ value: unknown,
495
+ ): PlayRunLedgerWorkCounters | undefined {
496
+ if (!isRecord(value)) return undefined;
497
+ for (const field of [
498
+ 'attempt',
499
+ 'invocationSeq',
500
+ 'logicalCalls',
501
+ 'providerRequests',
502
+ 'provider429s',
503
+ 'completedLogicalCalls',
504
+ 'completedProviderRequests',
505
+ 'completedProvider429s',
506
+ ] as const) {
507
+ if (!nonnegativeSafeInteger(value[field])) return undefined;
508
+ }
509
+ if (
510
+ value.invocationSeq === 0 ||
511
+ !isWorkCounterSegmentId(value.segmentId) ||
512
+ typeof value.final !== 'boolean' ||
513
+ typeof value.completeThroughAttempt !== 'boolean' ||
514
+ typeof value.consistent !== 'boolean'
515
+ ) {
516
+ return undefined;
517
+ }
518
+ return {
519
+ ...value,
520
+ attempt: value.attempt as number,
521
+ invocationSeq: value.invocationSeq as number,
522
+ segmentId: value.segmentId,
523
+ logicalCalls: value.logicalCalls as number,
524
+ providerRequests: value.providerRequests as number,
525
+ provider429s: value.provider429s as number,
526
+ final: value.final,
527
+ completedLogicalCalls: value.completedLogicalCalls as number,
528
+ completedProviderRequests: value.completedProviderRequests as number,
529
+ completedProvider429s: value.completedProvider429s as number,
530
+ completeThroughAttempt: value.completeThroughAttempt,
531
+ consistent: value.consistent,
532
+ };
533
+ }
534
+
535
+ function advanceNextWorkCounterSegment(
536
+ current: PlayRunLedgerWorkCounters,
537
+ event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
538
+ ): PlayRunLedgerWorkCounters {
539
+ const completedLogicalCalls =
540
+ current.completedLogicalCalls + current.logicalCalls;
541
+ const completedProviderRequests =
542
+ current.completedProviderRequests + current.providerRequests;
543
+ const completedProvider429s =
544
+ current.completedProvider429s + current.provider429s;
545
+ const safe = [
546
+ completedLogicalCalls,
547
+ completedProviderRequests,
548
+ completedProvider429s,
549
+ ].every(Number.isSafeInteger);
550
+ const contiguous =
551
+ event.attempt === current.attempt
552
+ ? event.invocationSeq === current.invocationSeq + 1
553
+ : event.attempt === current.attempt + 1 && event.invocationSeq === 1;
554
+ return {
555
+ attempt: event.attempt,
556
+ invocationSeq: event.invocationSeq,
557
+ segmentId: event.segmentId,
558
+ logicalCalls: event.logicalCalls,
559
+ providerRequests: event.providerRequests,
560
+ provider429s: event.provider429s,
561
+ final: event.final === true,
562
+ completedLogicalCalls: Math.min(
563
+ completedLogicalCalls,
564
+ Number.MAX_SAFE_INTEGER,
565
+ ),
566
+ completedProviderRequests: Math.min(
567
+ completedProviderRequests,
568
+ Number.MAX_SAFE_INTEGER,
569
+ ),
570
+ completedProvider429s: Math.min(
571
+ completedProvider429s,
572
+ Number.MAX_SAFE_INTEGER,
573
+ ),
574
+ completeThroughAttempt:
575
+ current.completeThroughAttempt && current.final && contiguous,
576
+ consistent:
577
+ current.consistent && safe && event.segmentId !== current.segmentId,
578
+ };
579
+ }
580
+
581
+ function advanceWorkCounters(
582
+ current: PlayRunLedgerWorkCounters | undefined,
583
+ event: Extract<PlayRunLedgerEvent, { type: 'work.counters' }>,
584
+ ): PlayRunLedgerWorkCounters | undefined {
585
+ if (
586
+ !nonnegativeSafeInteger(event.attempt) ||
587
+ !nonnegativeSafeInteger(event.invocationSeq) ||
588
+ event.invocationSeq === 0 ||
589
+ !isWorkCounterSegmentId(event.segmentId) ||
590
+ !nonnegativeSafeInteger(event.logicalCalls) ||
591
+ !nonnegativeSafeInteger(event.providerRequests) ||
592
+ !nonnegativeSafeInteger(event.provider429s)
593
+ ) {
594
+ return current;
595
+ }
596
+ if (!current) {
597
+ return {
598
+ attempt: event.attempt,
599
+ invocationSeq: event.invocationSeq,
600
+ segmentId: event.segmentId,
601
+ logicalCalls: event.logicalCalls,
602
+ providerRequests: event.providerRequests,
603
+ provider429s: event.provider429s,
604
+ final: event.final === true,
605
+ completedLogicalCalls: 0,
606
+ completedProviderRequests: 0,
607
+ completedProvider429s: 0,
608
+ completeThroughAttempt: event.attempt <= 1 && event.invocationSeq === 1,
609
+ consistent: true,
610
+ };
611
+ }
612
+ if (event.attempt < current.attempt) return current;
613
+ if (event.attempt === current.attempt) {
614
+ if (event.invocationSeq < current.invocationSeq) return current;
615
+ if (event.invocationSeq > current.invocationSeq)
616
+ return advanceNextWorkCounterSegment(current, event);
617
+ const counters = [
618
+ ['logicalCalls', event.logicalCalls],
619
+ ['providerRequests', event.providerRequests],
620
+ ['provider429s', event.provider429s],
621
+ ] as const;
622
+ const finalDisagrees = counters.some(
623
+ ([field, value]) =>
624
+ (event.final && value < current[field]) ||
625
+ (current.final && value > current[field]),
626
+ );
627
+ return {
628
+ ...current,
629
+ logicalCalls: Math.max(current.logicalCalls, event.logicalCalls),
630
+ providerRequests: Math.max(
631
+ current.providerRequests,
632
+ event.providerRequests,
633
+ ),
634
+ provider429s: Math.max(current.provider429s, event.provider429s),
635
+ final: current.final || event.final === true,
636
+ consistent:
637
+ current.consistent &&
638
+ current.segmentId === event.segmentId &&
639
+ !finalDisagrees,
640
+ };
641
+ }
642
+ return advanceNextWorkCounterSegment(current, event);
643
+ }
644
+
394
645
  function optionalFiniteNumber(value: unknown): number | undefined {
395
646
  const normalized = finiteNumber(value);
396
647
  return normalized === null ? undefined : normalized;
@@ -679,6 +930,11 @@ export function normalizePlayRunLedgerSnapshot(
679
930
  const logTail = Array.isArray(rawTail)
680
931
  ? rawTail.filter((line): line is string => typeof line === 'string')
681
932
  : [];
933
+ const workCounters = normalizeWorkCounters(value.workCounters);
934
+ const terminalWorkCounterSegment =
935
+ normalizePlayRunLedgerTerminalWorkCounterSegment(
936
+ value.terminalWorkCounterSegment,
937
+ );
682
938
  return {
683
939
  runId: optionalString(value.runId) ?? fallback.runId,
684
940
  playName:
@@ -713,6 +969,8 @@ export function normalizePlayRunLedgerSnapshot(
713
969
  typeof activityId === 'string' && Boolean(activitiesById[activityId]),
714
970
  ),
715
971
  activitiesById,
972
+ ...(workCounters ? { workCounters } : {}),
973
+ ...(terminalWorkCounterSegment ? { terminalWorkCounterSegment } : {}),
716
974
  logTail: logTail.slice(-LOG_TAIL_LIMIT),
717
975
  // Snapshots persisted before totalLogCount existed only know the retained
718
976
  // tail, so the best lower bound for the cumulative count is the tail size.
@@ -1283,6 +1541,11 @@ export function reducePlayRunLedgerEvent(
1283
1541
  shouldIgnoreNonTerminalAfterStickyTerminal(base);
1284
1542
 
1285
1543
  switch (event.type) {
1544
+ case 'work.counters': {
1545
+ const workCounters = advanceWorkCounters(base.workCounters, event);
1546
+ if (workCounters === base.workCounters) return snapshot;
1547
+ return withTiming({ ...base, workCounters });
1548
+ }
1286
1549
  case 'run.created': {
1287
1550
  if (ignoreNonTerminalAfterStickyTerminal) {
1288
1551
  return withTiming(base);
@@ -1333,14 +1596,24 @@ export function reducePlayRunLedgerEvent(
1333
1596
  waitUntil: null,
1334
1597
  activeActivityId: null,
1335
1598
  });
1336
- case 'run.completed':
1599
+ case 'run.completed': {
1600
+ const terminalWorkCounterSegment =
1601
+ (event.source === 'worker' || event.source === 'system') &&
1602
+ (!isTerminalPlayRunLedgerStatus(base.status) ||
1603
+ base.terminalSource === 'coordinator')
1604
+ ? normalizePlayRunLedgerTerminalWorkCounterSegment(
1605
+ event.terminalWorkCounterSegment,
1606
+ )
1607
+ : undefined;
1337
1608
  return (
1338
1609
  conflictingTerminalSnapshot(base, event.type, occurredAt, null, {
1339
1610
  allowStaleSameStatusPayloadMerge:
1340
1611
  (base.result === undefined && event.result !== undefined) ||
1341
1612
  canPromoteTerminalResultReference(base.result, event.result) ||
1342
1613
  (base.resultSummary === undefined &&
1343
- event.resultSummary !== undefined),
1614
+ event.resultSummary !== undefined) ||
1615
+ (base.terminalWorkCounterSegment === undefined &&
1616
+ terminalWorkCounterSegment !== undefined),
1344
1617
  }) ??
1345
1618
  withTiming({
1346
1619
  ...settleRunningStepsOnTerminal(base, 'completed', occurredAt),
@@ -1358,10 +1631,21 @@ export function reducePlayRunLedgerEvent(
1358
1631
  next: event.result,
1359
1632
  }),
1360
1633
  resultSummary: event.resultSummary ?? base.resultSummary,
1634
+ terminalWorkCounterSegment:
1635
+ base.terminalWorkCounterSegment ?? terminalWorkCounterSegment,
1361
1636
  terminalSource: nextTerminalSource(base, event.source),
1362
1637
  })
1363
1638
  );
1364
- case 'run.failed':
1639
+ }
1640
+ case 'run.failed': {
1641
+ const terminalWorkCounterSegment =
1642
+ (event.source === 'worker' || event.source === 'system') &&
1643
+ (!isTerminalPlayRunLedgerStatus(base.status) ||
1644
+ base.terminalSource === 'coordinator')
1645
+ ? normalizePlayRunLedgerTerminalWorkCounterSegment(
1646
+ event.terminalWorkCounterSegment,
1647
+ )
1648
+ : undefined;
1365
1649
  return (
1366
1650
  retryablePlatformDeployFailureSnapshot(base, event.error) ??
1367
1651
  conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
@@ -1370,7 +1654,9 @@ export function reducePlayRunLedgerEvent(
1370
1654
  (base.result === undefined && event.result !== undefined) ||
1371
1655
  (base.resultSummary === undefined &&
1372
1656
  event.resultSummary !== undefined) ||
1373
- canPromoteTerminalResultReference(base.result, event.result),
1657
+ canPromoteTerminalResultReference(base.result, event.result) ||
1658
+ (base.terminalWorkCounterSegment === undefined &&
1659
+ terminalWorkCounterSegment !== undefined),
1374
1660
  }) ??
1375
1661
  withTiming({
1376
1662
  ...settleRunningStepsOnTerminal(base, 'failed', occurredAt),
@@ -1391,9 +1677,12 @@ export function reducePlayRunLedgerEvent(
1391
1677
  next: event.result,
1392
1678
  }),
1393
1679
  resultSummary: event.resultSummary ?? base.resultSummary,
1680
+ terminalWorkCounterSegment:
1681
+ base.terminalWorkCounterSegment ?? terminalWorkCounterSegment,
1394
1682
  terminalSource: nextTerminalSource(base, event.source),
1395
1683
  })
1396
1684
  );
1685
+ }
1397
1686
  case 'run.cancelled':
1398
1687
  return (
1399
1688
  conflictingTerminalSnapshot(base, event.type, occurredAt, event.error, {
@@ -1,6 +1,7 @@
1
1
  import { resolveTimingWindow } from './live-events';
2
2
  import {
3
3
  LOG_TAIL_LIMIT,
4
+ hasConfirmedPlayRunLedgerWorkCounters,
4
5
  normalizePlayRunLedgerSnapshot,
5
6
  summarizeRunRowOutcomes,
6
7
  type PlayRunLedgerSnapshot,
@@ -110,6 +111,13 @@ export type PlayRunLiveSnapshot = {
110
111
  * without row outcomes.
111
112
  */
112
113
  rowOutcomes?: PlayRunRowOutcomeSummary;
114
+ /** Observed run-wide call attempts. Absent for runs without counter evidence. */
115
+ calls?: {
116
+ logicalCalls: number;
117
+ providerRequests: number;
118
+ provider429s: number;
119
+ confirmed: boolean;
120
+ };
113
121
  };
114
122
 
115
123
  export function normalizePlayRunLiveStatus(value: unknown): PlayRunLiveStatus {
@@ -236,6 +244,21 @@ function buildSnapshotFromLedger(
236
244
  (terminalRowOutcomes.supersededRows ?? 0) > 0)
237
245
  ? terminalRowOutcomes
238
246
  : null;
247
+ const workCounters = snapshot.workCounters;
248
+ const calls = workCounters
249
+ ? {
250
+ logicalCalls:
251
+ workCounters.completedLogicalCalls + workCounters.logicalCalls,
252
+ providerRequests:
253
+ workCounters.completedProviderRequests +
254
+ workCounters.providerRequests,
255
+ provider429s:
256
+ workCounters.completedProvider429s + workCounters.provider429s,
257
+ confirmed:
258
+ isTerminalPlayRunLiveStatus(liveStatus) &&
259
+ hasConfirmedPlayRunLedgerWorkCounters(snapshot),
260
+ }
261
+ : null;
239
262
  return {
240
263
  runId: snapshot.runId,
241
264
  status: liveStatus,
@@ -268,6 +291,12 @@ function buildSnapshotFromLedger(
268
291
  nodeStates,
269
292
  activeNodeId: snapshot.activeStepId ?? null,
270
293
  ...(rowOutcomes ? { rowOutcomes } : {}),
294
+ ...(calls &&
295
+ Object.values(calls).every(
296
+ (value) => typeof value === 'boolean' || Number.isSafeInteger(value),
297
+ )
298
+ ? { calls }
299
+ : {}),
271
300
  };
272
301
  }
273
302
 
@@ -2,7 +2,7 @@ import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
2
2
  import { readFile } from 'node:fs/promises';
3
3
  import { dirname, isAbsolute, resolve } from 'node:path';
4
4
  import { createRequire } from 'node:module';
5
- import { createHash } from 'node:crypto';
5
+ import { createHash, randomUUID } from 'node:crypto';
6
6
  import { formatWithOptions } from 'node:util';
7
7
  import { compileFunction } from 'node:vm';
8
8
  import { gunzipSync } from 'node:zlib';
@@ -42,6 +42,7 @@ import {
42
42
  } from '../run-failure';
43
43
  import { createPlayContext } from '../context';
44
44
  import { augmentSheetContractWithDatasetFields } from '../../play-data-plane/sheet-contract';
45
+ import { nextWorkCounterInvocationSeq } from '../runtime-work-counters';
45
46
  import type { RuntimeApiResponseMeta } from '../app-runtime-api';
46
47
  import { onMemoryPressure } from '../memory-pressure-signal';
47
48
  import {
@@ -1894,6 +1895,16 @@ async function run(
1894
1895
  options?: ExecutePlayRunnerOptions,
1895
1896
  ): Promise<PlayRunnerResult> {
1896
1897
  const runStartedAt = Date.now();
1898
+ const runnerRunAttempt =
1899
+ typeof config.context.runAttempt === 'number' &&
1900
+ Number.isFinite(config.context.runAttempt)
1901
+ ? Math.max(0, Math.floor(config.context.runAttempt))
1902
+ : 0;
1903
+ const workCounterInvocationSeq = nextWorkCounterInvocationSeq(
1904
+ config.checkpoint,
1905
+ runnerRunAttempt,
1906
+ );
1907
+ const workCounterSegmentId = randomUUID();
1897
1908
  const persistenceSignal =
1898
1909
  options?.signal ??
1899
1910
  AbortSignal.timeout(
@@ -2010,11 +2021,6 @@ async function run(
2010
2021
  runId: string;
2011
2022
  tableNamespace: string;
2012
2023
  }) => `${input.playName}\u0000${input.runId}\u0000${input.tableNamespace}`;
2013
- const runnerRunAttempt =
2014
- typeof config.context.runAttempt === 'number' &&
2015
- Number.isFinite(config.context.runAttempt)
2016
- ? Math.max(0, Math.floor(config.context.runAttempt))
2017
- : 0;
2018
2024
  const cachePolicy = playRunnerContextCachePolicy(config.context);
2019
2025
  const rateStateBackend = createRateStateBackendForConfig(config);
2020
2026
  const runtimeInput = config.input ?? {};
@@ -2980,6 +2986,13 @@ async function run(
2980
2986
  );
2981
2987
  }
2982
2988
  };
2989
+ const runtimeStats = () => ({
2990
+ ...ctx.getStats(),
2991
+ runtimeWorkCounterSegment: {
2992
+ invocationSeq: workCounterInvocationSeq,
2993
+ segmentId: workCounterSegmentId,
2994
+ },
2995
+ });
2983
2996
  options?.onRuntimeReady?.({
2984
2997
  log: (line) => ctx.logRunnerDiagnostic(line),
2985
2998
  timeoutResult: (runtimeLimitSeconds) => {
@@ -2989,7 +3002,7 @@ async function run(
2989
3002
  error: runtimeLimitExceededCause(runtimeLimitSeconds),
2990
3003
  errors: [runtimeLimitExceededFailure(runtimeLimitSeconds)],
2991
3004
  logs: terminalLogsWithPhaseSummary(ctx),
2992
- stats: ctx.getStats(),
3005
+ stats: runtimeStats(),
2993
3006
  steps: ctx.getSteps(),
2994
3007
  tableNamespace: null,
2995
3008
  totalRows,
@@ -3007,6 +3020,30 @@ async function run(
3007
3020
  }
3008
3021
  : runtimeInput;
3009
3022
 
3023
+ // Sample scalar counters, not the potentially large resource snapshot. The
3024
+ // reporter coalesces transport updates; this timer emits no event if no
3025
+ // counter changed and cannot hold the runner process open.
3026
+ let lastWorkCounterKey: string | null = null;
3027
+ const emitWorkCounters = () => {
3028
+ const counters = ctx.getRuntimeWorkCounters();
3029
+ const key = `${counters.logicalCalls}:${counters.providerRequests}:${counters.provider429s}`;
3030
+ if (key === lastWorkCounterKey) return;
3031
+ lastWorkCounterKey = key;
3032
+ emitRunnerEvent(
3033
+ {
3034
+ type: 'work_counters',
3035
+ attempt: runnerRunAttempt,
3036
+ invocationSeq: workCounterInvocationSeq,
3037
+ segmentId: workCounterSegmentId,
3038
+ ...counters,
3039
+ },
3040
+ options,
3041
+ );
3042
+ };
3043
+ emitWorkCounters();
3044
+ const workCounterTimer = setInterval(emitWorkCounters, 1_500);
3045
+ workCounterTimer.unref();
3046
+
3010
3047
  try {
3011
3048
  emitWorkerLog('Play started', options);
3012
3049
  const rawResult = await executeRuntimePlayCode({
@@ -3210,7 +3247,7 @@ async function run(
3210
3247
  logs,
3211
3248
  ...(runtimeDiagnostics.length > 0 ? { runtimeDiagnostics } : {}),
3212
3249
  ...(rowOutcomes ? { rowOutcomes } : {}),
3213
- stats: ctx.getStats(),
3250
+ stats: runtimeStats(),
3214
3251
  steps: ctx.getSteps(),
3215
3252
  checkpoint,
3216
3253
  tableNamespace: resultTableNamespace,
@@ -3262,7 +3299,7 @@ async function run(
3262
3299
  status: 'suspended',
3263
3300
  suspension: error.suspension,
3264
3301
  logs: terminalLogsWithPhaseSummary(ctx),
3265
- stats: ctx.getStats(),
3302
+ stats: runtimeStats(),
3266
3303
  steps: ctx.getSteps(),
3267
3304
  checkpoint: ctx.getCheckpoint(),
3268
3305
  tableNamespace: null,
@@ -3298,7 +3335,7 @@ async function run(
3298
3335
  errors: [failure],
3299
3336
  logs: terminalLogsWithPhaseSummary(ctx),
3300
3337
  ...(rowOutcomes ? { rowOutcomes } : {}),
3301
- stats: ctx.getStats(),
3338
+ stats: runtimeStats(),
3302
3339
  steps: ctx.getSteps(),
3303
3340
  checkpoint,
3304
3341
  tableNamespace: null,
@@ -3306,6 +3343,9 @@ async function run(
3306
3343
  inserted,
3307
3344
  skipped,
3308
3345
  };
3346
+ } finally {
3347
+ clearInterval(workCounterTimer);
3348
+ emitWorkCounters();
3309
3349
  }
3310
3350
  }
3311
3351
 
@@ -24,6 +24,7 @@ function shouldEmitProgressSidecarLog(line: string): boolean {
24
24
  export function shouldEmitProgressSidecarEvent(
25
25
  event: PlayRunnerEvent,
26
26
  ): boolean {
27
+ if (event.type === 'work_counters') return true;
27
28
  if (event.type === 'execution_event') {
28
29
  return event.event.type !== 'map.row.updated';
29
30
  }
@@ -509,6 +509,10 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
509
509
  }
510
510
  if (event.type === 'execution_event') {
511
511
  callbacks?.onExecutionEvent?.(event.event);
512
+ return;
513
+ }
514
+ if (event.type === 'work_counters') {
515
+ callbacks?.onWorkCounters?.(event);
512
516
  }
513
517
  },
514
518
  },
@@ -5,6 +5,7 @@ import type {
5
5
  } from '../ctx-types';
6
6
  import type {
7
7
  PlayRunnerExecutionConfig,
8
+ PlayRunnerEvent,
8
9
  PlayRunnerLogEvent,
9
10
  PlayRunnerResult,
10
11
  } from '../protocol';
@@ -155,6 +156,9 @@ export interface PlayRunnerCallbacks {
155
156
  onCheckpoint?: (checkpoint: PlayCheckpoint) => void;
156
157
  onRowUpdate?: (update: PlayRowUpdate) => void;
157
158
  onExecutionEvent?: (event: PlayExecutionEvent) => void;
159
+ onWorkCounters?: (
160
+ event: Extract<PlayRunnerEvent, { type: 'work_counters' }>,
161
+ ) => void;
158
162
  onRuntimeResourceAcquired?: (
159
163
  resource: PlayRunnerRuntimeResource,
160
164
  ) => void | Promise<void>;