deepline 0.3.288 → 0.3.290
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +30 -10
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +108 -12
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts +6 -5
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-batching-registry.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-event-catalog.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +31 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +38 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +22 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-pool-options.ts +145 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +1 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -139
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +19 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +37 -15
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +7 -5
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +961 -281
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +55 -25
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-dispatch-authority.ts +97 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +33 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +452 -43
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-admission-batching.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/zerobounce-batching.ts +75 -28
- package/dist/cli/index.js +24 -11
- package/dist/cli/index.mjs +24 -11
- package/dist/index.js +3 -1
- package/dist/index.mjs +3 -1
- package/dist/install-integrity.json +3 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.290',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -111,12 +111,16 @@ export type RuntimeStatusUpdate = {
|
|
|
111
111
|
liveLogs?: string[];
|
|
112
112
|
/** Search-only context aligned to `liveLogs`; stripped before Convex. */
|
|
113
113
|
liveLogContexts?: Array<PlayLogContext | null>;
|
|
114
|
+
liveLogIds?: Array<string | null>;
|
|
114
115
|
/**
|
|
115
116
|
* Monotonic count of log lines the producer ever emitted on this run's
|
|
116
117
|
* channel. `liveLogs` is a rotating tail of those lines; the total lets the
|
|
117
118
|
* positional cursor stay correct once the buffer has rotated.
|
|
118
119
|
*/
|
|
119
120
|
liveLogTotalCount?: number;
|
|
121
|
+
/** Absolute start of a stateless gateway batch. Its durable positions must
|
|
122
|
+
* reach ingestion even if another request already forwarded a newer batch. */
|
|
123
|
+
liveLogBatchOffset?: number;
|
|
120
124
|
/**
|
|
121
125
|
* Monotonic scheduler attempt that owns the positional log channel. A new
|
|
122
126
|
* attempt starts at offset zero without colliding with earlier attempts.
|
|
@@ -156,6 +160,7 @@ export type RuntimeStatusUpdate = {
|
|
|
156
160
|
terminalLogReplay?: {
|
|
157
161
|
lines: string[];
|
|
158
162
|
lineContexts?: Array<PlayLogContext | null>;
|
|
163
|
+
lineIds?: Array<string | null>;
|
|
159
164
|
totalCount?: number;
|
|
160
165
|
dedupeMode?: 'semantic' | 'exact';
|
|
161
166
|
producerAttempt?: number;
|
|
@@ -355,6 +360,8 @@ type RuntimeApiRequest =
|
|
|
355
360
|
export type WorkerRuntimeApiContext = {
|
|
356
361
|
baseUrl: string;
|
|
357
362
|
executorToken: string;
|
|
363
|
+
/** Exact ingress-prefix verifier advertised by the immutable launch receiver. */
|
|
364
|
+
runLedgerIngressPrefixBatching?: true;
|
|
358
365
|
boundary?: 'app_runtime' | 'receipt_gateway';
|
|
359
366
|
integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
|
|
360
367
|
vercelProtectionBypassToken?: string | null;
|
|
@@ -2197,18 +2204,22 @@ async function updateRunStatusViaAppRuntimeUnlocked(
|
|
|
2197
2204
|
runId: update.playId,
|
|
2198
2205
|
status: 'queued',
|
|
2199
2206
|
});
|
|
2200
|
-
//
|
|
2201
|
-
//
|
|
2202
|
-
//
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2207
|
+
// Stateful producers forward their unsent suffix. A stateless gateway batch
|
|
2208
|
+
// owns an absolute start: a newer request's process-local cursor is no proof
|
|
2209
|
+
// that this batch reached ingestion. Durable positional identities reconcile
|
|
2210
|
+
// overlap and terminal recovery.
|
|
2211
|
+
const sentCount =
|
|
2212
|
+
update.liveLogBatchOffset ??
|
|
2213
|
+
(isTerminalRuntimeStatus(update.status)
|
|
2214
|
+
? 0
|
|
2215
|
+
: (runLogChannelSentCounts.get(
|
|
2216
|
+
runLogChannelKey(update.playId, update.liveLogProducerAttempt),
|
|
2217
|
+
) ?? 0));
|
|
2208
2218
|
const logSlice = Array.isArray(update.liveLogs)
|
|
2209
2219
|
? slicePositionalLogLines({
|
|
2210
2220
|
bufferLines: update.liveLogs,
|
|
2211
2221
|
bufferContexts: update.liveLogContexts,
|
|
2222
|
+
bufferIds: update.liveLogIds,
|
|
2212
2223
|
bufferTotalCount: update.liveLogTotalCount ?? update.liveLogs.length,
|
|
2213
2224
|
sentCount,
|
|
2214
2225
|
})
|
|
@@ -2220,6 +2231,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
|
|
|
2220
2231
|
? {
|
|
2221
2232
|
lines: update.liveLogs,
|
|
2222
2233
|
lineContexts: update.liveLogContexts,
|
|
2234
|
+
lineIds: update.liveLogIds,
|
|
2223
2235
|
totalCount: update.liveLogTotalCount,
|
|
2224
2236
|
producerAttempt: update.liveLogProducerAttempt,
|
|
2225
2237
|
}
|
|
@@ -2239,7 +2251,13 @@ async function updateRunStatusViaAppRuntimeUnlocked(
|
|
|
2239
2251
|
0,
|
|
2240
2252
|
Math.max(0, replayOffset - logSlice.channelOffset),
|
|
2241
2253
|
);
|
|
2242
|
-
if (prefix.length)
|
|
2254
|
+
if (prefix.length)
|
|
2255
|
+
statusPatchLogSlice = {
|
|
2256
|
+
...logSlice,
|
|
2257
|
+
lines: prefix,
|
|
2258
|
+
lineContexts: logSlice.lineContexts?.slice(0, prefix.length),
|
|
2259
|
+
lineIds: logSlice.lineIds?.slice(0, prefix.length),
|
|
2260
|
+
};
|
|
2243
2261
|
}
|
|
2244
2262
|
const events = buildPlayRunLedgerEventsFromStatusPatch({
|
|
2245
2263
|
patch: {
|
|
@@ -2250,6 +2268,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
|
|
|
2250
2268
|
lastCheckpointAt: update.lastCheckpointAt ?? null,
|
|
2251
2269
|
liveLogs: statusPatchLogSlice?.lines ?? null,
|
|
2252
2270
|
liveLogContexts: statusPatchLogSlice?.lineContexts ?? null,
|
|
2271
|
+
liveLogIds: statusPatchLogSlice?.lineIds ?? null,
|
|
2253
2272
|
liveLogsChannelOffset: statusPatchLogSlice?.channelOffset ?? null,
|
|
2254
2273
|
liveLogsProducerAttempt: update.liveLogProducerAttempt ?? null,
|
|
2255
2274
|
liveNodeProgress: update.liveNodeProgress ?? null,
|
|
@@ -2335,6 +2354,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
|
|
|
2335
2354
|
occurredAt: update.lastCheckpointAt ?? Date.now(),
|
|
2336
2355
|
lines: terminalLogReplay.lines,
|
|
2337
2356
|
lineContexts: terminalLogReplay.lineContexts,
|
|
2357
|
+
lineIds: terminalLogReplay.lineIds,
|
|
2338
2358
|
liveLogTotalCount: terminalLogReplay.totalCount,
|
|
2339
2359
|
dedupeMode: terminalLogReplay.dedupeMode,
|
|
2340
2360
|
producerAttempt: terminalLogReplay.producerAttempt,
|
|
@@ -2364,7 +2384,7 @@ async function updateRunStatusViaAppRuntimeUnlocked(
|
|
|
2364
2384
|
events: terminalLogEvents,
|
|
2365
2385
|
});
|
|
2366
2386
|
}
|
|
2367
|
-
if (statusPatchLogSlice) {
|
|
2387
|
+
if (statusPatchLogSlice && update.liveLogBatchOffset === undefined) {
|
|
2368
2388
|
runLogChannelSentCounts.set(
|
|
2369
2389
|
runLogChannelKey(update.playId, update.liveLogProducerAttempt),
|
|
2370
2390
|
statusPatchLogSlice.channelOffset + statusPatchLogSlice.lines.length,
|
|
@@ -1806,7 +1806,27 @@ type NativeBatchBodyAttribution = {
|
|
|
1806
1806
|
release(): void;
|
|
1807
1807
|
};
|
|
1808
1808
|
|
|
1809
|
+
type ToolBatchAdmissionObserver = {
|
|
1810
|
+
pending(requests: readonly ToolCallRequest[]): void;
|
|
1811
|
+
dispatched(requests: readonly ToolCallRequest[]): void;
|
|
1812
|
+
};
|
|
1813
|
+
|
|
1814
|
+
function settleToolBatchGroups(
|
|
1815
|
+
groups: ToolCallRequest[][],
|
|
1816
|
+
execute: (requests: ToolCallRequest[]) => Promise<void>,
|
|
1817
|
+
observer?: ToolBatchAdmissionObserver,
|
|
1818
|
+
): Promise<PromiseSettledResult<void>[]> {
|
|
1819
|
+
return Promise.allSettled(
|
|
1820
|
+
groups.map((requests) =>
|
|
1821
|
+
execute(requests).finally(() => observer?.dispatched(requests)),
|
|
1822
|
+
),
|
|
1823
|
+
);
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1809
1826
|
type ToolExecutionApiOptions = {
|
|
1827
|
+
/** Runner-local scheduling signals; never part of a provider/wire payload. */
|
|
1828
|
+
onProviderAdmissionPending?: () => void;
|
|
1829
|
+
onProviderDispatched?: () => void;
|
|
1810
1830
|
timeoutMs?: number;
|
|
1811
1831
|
abortSignal?: AbortSignal;
|
|
1812
1832
|
durableCallReceiptKey?: string | null;
|
|
@@ -11829,8 +11849,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11829
11849
|
request.executionAuthScopeDigest ?? '',
|
|
11830
11850
|
String(strategy.toBucketKey(request.input)),
|
|
11831
11851
|
].join('\u0000'),
|
|
11832
|
-
readyCount:
|
|
11833
|
-
|
|
11852
|
+
readyCount: Math.min(
|
|
11853
|
+
strategy.maxBatchSize,
|
|
11854
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11855
|
+
),
|
|
11856
|
+
maxGroupSize: Math.min(
|
|
11857
|
+
strategy.maxBatchSize,
|
|
11858
|
+
this.governor.policy.concurrency.toolCalls,
|
|
11859
|
+
),
|
|
11834
11860
|
coalesceWindowMs: this.toolCoalesceWindowMs(
|
|
11835
11861
|
this.#options.toolBatchCoalesceWindowMs,
|
|
11836
11862
|
TOOL_BATCH_COALESCE_WINDOW_MS,
|
|
@@ -12030,6 +12056,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12030
12056
|
|
|
12031
12057
|
const requests: ToolCallRequest[] = [];
|
|
12032
12058
|
const remaining: ToolCallRequest[] = [];
|
|
12059
|
+
const remainingLaneKeys = new Set<string>();
|
|
12033
12060
|
const selectedByLane = new Map<string, number>();
|
|
12034
12061
|
for (const request of this.toolCallQueue) {
|
|
12035
12062
|
const lane = this.toolDispatchLane(request);
|
|
@@ -12043,12 +12070,15 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12043
12070
|
selectedByLane.set(lane.key, selectedCount + 1);
|
|
12044
12071
|
} else {
|
|
12045
12072
|
remaining.push(request);
|
|
12073
|
+
remainingLaneKeys.add(lane.key);
|
|
12046
12074
|
}
|
|
12047
12075
|
}
|
|
12048
12076
|
this.toolCallQueue = remaining;
|
|
12049
12077
|
const selectedLaneKeys = new Set(selectedByLane.keys());
|
|
12050
12078
|
for (const laneKey of selectedLaneKeys) {
|
|
12051
|
-
|
|
12079
|
+
if (!remainingLaneKeys.has(laneKey)) {
|
|
12080
|
+
this.toolDispatchQueuedAtByLane.delete(laneKey);
|
|
12081
|
+
}
|
|
12052
12082
|
}
|
|
12053
12083
|
return { requests, laneKeys: selectedLaneKeys, nextDeadlineMs };
|
|
12054
12084
|
}
|
|
@@ -12258,6 +12288,33 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12258
12288
|
// it never waits for unrelated sibling calls from the previous column.
|
|
12259
12289
|
const inFlightToolExecutions = new Map<Promise<void>, ToolCallRequest[]>();
|
|
12260
12290
|
const activeGroupsByLane = new Map<string, number>();
|
|
12291
|
+
// Keep a native batch lane together while its physical request is
|
|
12292
|
+
// preparing or waiting for provider admission. Release it at dispatch, so
|
|
12293
|
+
// provider residence does not serialize the next batch.
|
|
12294
|
+
const pendingAdmissionByLane = new Map<string, Set<string>>();
|
|
12295
|
+
const admissionObserver: ToolBatchAdmissionObserver = {
|
|
12296
|
+
pending: (requests) => {
|
|
12297
|
+
for (const request of requests) {
|
|
12298
|
+
if (!this.#options.getBatchOperationStrategy?.(request.toolId))
|
|
12299
|
+
continue;
|
|
12300
|
+
const key = this.toolDispatchLane(request).key;
|
|
12301
|
+
const pending = pendingAdmissionByLane.get(key) ?? new Set<string>();
|
|
12302
|
+
pending.add(request.callId);
|
|
12303
|
+
pendingAdmissionByLane.set(key, pending);
|
|
12304
|
+
}
|
|
12305
|
+
},
|
|
12306
|
+
dispatched: (requests) => {
|
|
12307
|
+
let changed = false;
|
|
12308
|
+
for (const request of requests) {
|
|
12309
|
+
const key = this.toolDispatchLane(request).key;
|
|
12310
|
+
const pending = pendingAdmissionByLane.get(key);
|
|
12311
|
+
if (!pending?.delete(request.callId)) continue;
|
|
12312
|
+
changed = true;
|
|
12313
|
+
if (pending.size === 0) pendingAdmissionByLane.delete(key);
|
|
12314
|
+
}
|
|
12315
|
+
if (changed) this.wakeToolDispatcher();
|
|
12316
|
+
},
|
|
12317
|
+
};
|
|
12261
12318
|
const maxInFlightGroups = Math.max(
|
|
12262
12319
|
1,
|
|
12263
12320
|
Math.floor(
|
|
@@ -12319,6 +12376,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12319
12376
|
.filter(([, count]) => count >= maxInFlightGroupsPerLane)
|
|
12320
12377
|
.map(([laneKey]) => laneKey),
|
|
12321
12378
|
);
|
|
12379
|
+
for (const key of pendingAdmissionByLane.keys()) {
|
|
12380
|
+
blockedLaneKeys.add(key);
|
|
12381
|
+
}
|
|
12322
12382
|
const dispatchable =
|
|
12323
12383
|
inFlightToolExecutions.size >= maxInFlightGroups
|
|
12324
12384
|
? {
|
|
@@ -12361,7 +12421,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12361
12421
|
(activeGroupsByLane.get(laneKey) ?? 0) + 1,
|
|
12362
12422
|
);
|
|
12363
12423
|
}
|
|
12364
|
-
|
|
12424
|
+
admissionObserver.pending(dispatchable.requests);
|
|
12425
|
+
const tracked = this.executeBatchedToolCalls(
|
|
12426
|
+
dispatchable.requests,
|
|
12427
|
+
admissionObserver,
|
|
12428
|
+
)
|
|
12365
12429
|
.catch(async (error) => {
|
|
12366
12430
|
this.toolDispatcherFailure ??= error;
|
|
12367
12431
|
this.rejectQueuedToolCalls(error);
|
|
@@ -12371,6 +12435,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12371
12435
|
);
|
|
12372
12436
|
})
|
|
12373
12437
|
.finally(() => {
|
|
12438
|
+
admissionObserver.dispatched(dispatchable.requests);
|
|
12374
12439
|
this.observeToolDispatchGroupRoundTrip(
|
|
12375
12440
|
this.toolBatchDispatcherNowMs() - launchedAtMs,
|
|
12376
12441
|
);
|
|
@@ -14231,6 +14296,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14231
14296
|
originalLineWasRetained
|
|
14232
14297
|
? (this.logBuffer.contexts.at(-1) ?? undefined)
|
|
14233
14298
|
: undefined,
|
|
14299
|
+
this.logBuffer.ids.at(-1),
|
|
14234
14300
|
);
|
|
14235
14301
|
if (this.#options.verbose) console.log(retained);
|
|
14236
14302
|
}
|
|
@@ -14930,6 +14996,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14930
14996
|
return this.logBuffer.lines;
|
|
14931
14997
|
}
|
|
14932
14998
|
|
|
14999
|
+
getLogIds(): string[] {
|
|
15000
|
+
return this.logBuffer.ids.slice();
|
|
15001
|
+
}
|
|
15002
|
+
|
|
14933
15003
|
getLogContexts(): Array<PlayLogContext | null> {
|
|
14934
15004
|
return this.logBuffer.contexts.slice();
|
|
14935
15005
|
}
|
|
@@ -15676,6 +15746,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15676
15746
|
toolId: string;
|
|
15677
15747
|
requests: ToolCallRequest[];
|
|
15678
15748
|
strategy: AnyBatchOperationStrategy;
|
|
15749
|
+
admissionObserver?: ToolBatchAdmissionObserver;
|
|
15679
15750
|
}): Promise<ToolCallRequest[]> {
|
|
15680
15751
|
const { toolId, requests, strategy } = input;
|
|
15681
15752
|
if (requests.length === 0) return [];
|
|
@@ -15952,6 +16023,12 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
15952
16023
|
),
|
|
15953
16024
|
executionAuthScopeDigest:
|
|
15954
16025
|
detail.members[0]?.executionAuthScopeDigest ?? null,
|
|
16026
|
+
onProviderAdmissionPending: () =>
|
|
16027
|
+
input.admissionObserver?.pending(batch.memberRequests),
|
|
16028
|
+
onProviderDispatched: () =>
|
|
16029
|
+
input.admissionObserver?.dispatched(
|
|
16030
|
+
batch.memberRequests,
|
|
16031
|
+
),
|
|
15955
16032
|
providerIdempotencyKey: detail.providerOperationKey,
|
|
15956
16033
|
providerIdempotencyReceiptKey:
|
|
15957
16034
|
compatibility.resultReceiptKey,
|
|
@@ -16199,6 +16276,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16199
16276
|
});
|
|
16200
16277
|
return { batch, result };
|
|
16201
16278
|
} finally {
|
|
16279
|
+
input.admissionObserver?.dispatched(batch.memberRequests);
|
|
16202
16280
|
// A dispatch that never reached its split — a provider failure, a
|
|
16203
16281
|
// cohort throw — must not leave the body registered against
|
|
16204
16282
|
// nobody for the rest of the run.
|
|
@@ -16290,16 +16368,23 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16290
16368
|
|
|
16291
16369
|
private async executeBatchedToolCalls(
|
|
16292
16370
|
queuedToolCalls: ToolCallRequest[],
|
|
16371
|
+
admissionObserver?: ToolBatchAdmissionObserver,
|
|
16293
16372
|
): Promise<void> {
|
|
16294
|
-
//
|
|
16373
|
+
// Preserve authentication partitions through physical batch compilation.
|
|
16295
16374
|
const byTool = new Map<string, ToolCallRequest[]>();
|
|
16296
16375
|
for (const req of queuedToolCalls) {
|
|
16297
|
-
|
|
16298
|
-
|
|
16299
|
-
|
|
16300
|
-
|
|
16301
|
-
|
|
16302
|
-
|
|
16376
|
+
const key = JSON.stringify([
|
|
16377
|
+
req.toolId,
|
|
16378
|
+
req.executionAuthScopeDigest ?? '',
|
|
16379
|
+
]);
|
|
16380
|
+
if (!byTool.has(key)) byTool.set(key, []);
|
|
16381
|
+
byTool.get(key)!.push(req);
|
|
16382
|
+
}
|
|
16383
|
+
const toolGroups = [...byTool.values()];
|
|
16384
|
+
const toolSettlements = await settleToolBatchGroups(
|
|
16385
|
+
toolGroups,
|
|
16386
|
+
async (requests) => {
|
|
16387
|
+
const toolId = requests[0]!.toolId;
|
|
16303
16388
|
this.runtimeLog(
|
|
16304
16389
|
`Executing tool batch ${toolId}: ${requests.length} calls`,
|
|
16305
16390
|
);
|
|
@@ -16430,6 +16515,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16430
16515
|
toolId,
|
|
16431
16516
|
requests: cohortRequests,
|
|
16432
16517
|
strategy: operationStrategy,
|
|
16518
|
+
admissionObserver,
|
|
16433
16519
|
})
|
|
16434
16520
|
: await this.executeCacheableNonNativeMapToolCallCohort({
|
|
16435
16521
|
toolId,
|
|
@@ -16524,6 +16610,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16524
16610
|
providerOperationKey: aggregateProviderOperationKey,
|
|
16525
16611
|
apiOptions: {
|
|
16526
16612
|
durableCallReceiptKey: aggregateReceiptKey,
|
|
16613
|
+
onProviderAdmissionPending: () =>
|
|
16614
|
+
admissionObserver?.pending(batch.memberRequests),
|
|
16615
|
+
onProviderDispatched: () =>
|
|
16616
|
+
admissionObserver?.dispatched(batch.memberRequests),
|
|
16527
16617
|
legacyProviderRetryIdentities: [
|
|
16528
16618
|
{
|
|
16529
16619
|
transportReceiptKey:
|
|
@@ -16785,7 +16875,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
16785
16875
|
this.#options.onBatchComplete?.(this.checkpoint);
|
|
16786
16876
|
}
|
|
16787
16877
|
}
|
|
16788
|
-
}
|
|
16878
|
+
},
|
|
16879
|
+
admissionObserver,
|
|
16789
16880
|
);
|
|
16790
16881
|
const rejected = toolSettlements.find(
|
|
16791
16882
|
(settlement): settlement is PromiseRejectedResult =>
|
|
@@ -17944,6 +18035,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
17944
18035
|
let providerTelemetryKey: string | null = null;
|
|
17945
18036
|
let providerRunInFlightAtDispatch: number | null = null;
|
|
17946
18037
|
const acquireProviderPermit = async (signal?: AbortSignal) => {
|
|
18038
|
+
options?.onProviderAdmissionPending?.();
|
|
17947
18039
|
logProvider429RetryPhase('provider_permit_wait_started');
|
|
17948
18040
|
const permit = await this.resourceGovernor.acquireProviderPermit({
|
|
17949
18041
|
toolId,
|
|
@@ -18094,6 +18186,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18094
18186
|
acquireProviderPermit: async ({ signal }) =>
|
|
18095
18187
|
await acquireProviderPermit(signal),
|
|
18096
18188
|
tryAcquireProviderPermit: () => {
|
|
18189
|
+
options?.onProviderAdmissionPending?.();
|
|
18097
18190
|
logProvider429RetryPhase(
|
|
18098
18191
|
'provider_permit_acquire_attempted',
|
|
18099
18192
|
);
|
|
@@ -18146,6 +18239,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
18146
18239
|
> | null;
|
|
18147
18240
|
} = { current: null };
|
|
18148
18241
|
const beginProviderExecution = () => {
|
|
18242
|
+
if (providerExecution.current === null) {
|
|
18243
|
+
options?.onProviderDispatched?.();
|
|
18244
|
+
}
|
|
18149
18245
|
const execution =
|
|
18150
18246
|
providerExecution.current ??
|
|
18151
18247
|
this.resourceGovernor.beginProviderExecution({
|
|
@@ -851,7 +851,7 @@ export interface ContextOptions {
|
|
|
851
851
|
/** Called after each durable batch completes for scheduler checkpointing. */
|
|
852
852
|
onBatchComplete?: (checkpoint: PlayCheckpoint) => void;
|
|
853
853
|
/** Called when the runtime emits a new execution log line. */
|
|
854
|
-
onLog?: (line: string, context?: PlayLogContext) => void;
|
|
854
|
+
onLog?: (line: string, context?: PlayLogContext, entryId?: string) => void;
|
|
855
855
|
/**
|
|
856
856
|
* Runtime-provided HTTP transport for ctx.fetch. Node runtimes should provide
|
|
857
857
|
* a transport that validates resolved IP addresses at connect time.
|
|
@@ -84,6 +84,7 @@ const FIXTURE_MODE_BYPASS_PROVIDERS = new Set([
|
|
|
84
84
|
'test',
|
|
85
85
|
'test_high',
|
|
86
86
|
'test_low',
|
|
87
|
+
'test_batching',
|
|
87
88
|
// Dedicated stateful failure fixtures must reach their local custom handlers
|
|
88
89
|
// even when the surrounding Play runs in fixture mode.
|
|
89
90
|
'test_receipt',
|
|
@@ -17,6 +17,9 @@ export const DEFAULT_ADVISORY_PROVIDER_MAX_RPS = 100;
|
|
|
17
17
|
/** Cold-start rate for an unknown provider bucket shared across a fleet key. */
|
|
18
18
|
export const DEFAULT_UNKNOWN_PROVIDER_COLD_START_RPS = 10;
|
|
19
19
|
|
|
20
|
+
/** Learned-rate floor, independent of the unknown provider's starting guess. */
|
|
21
|
+
export const DEFAULT_UNKNOWN_PROVIDER_MIN_RPS = 2;
|
|
22
|
+
|
|
20
23
|
/** AIMD state cooldown cap; provider Retry-After holds remain uncapped. */
|
|
21
24
|
export const PROVIDER_AIMD_COOLDOWN_CAP_MS = 45 * 60 * 1_000;
|
|
22
25
|
|
|
@@ -52,6 +52,7 @@ export const PLAY_LAUNCH_ENVELOPE_FIELD_ROLES: {
|
|
|
52
52
|
playId: 'detail',
|
|
53
53
|
schedulerSchema: 'detail',
|
|
54
54
|
requiresRunCreatedProjection: 'detail',
|
|
55
|
+
runtimeLogIdentity: 'detail',
|
|
55
56
|
runtimeArtifact: 'detail',
|
|
56
57
|
runtimeProtocolVersion: 'detail',
|
|
57
58
|
staticPipeline: 'detail',
|
|
@@ -117,6 +117,23 @@ export const LOG_PROVENANCE_POLICY: Record<
|
|
|
117
117
|
receipt: { watch: false, ui: false, debug: true },
|
|
118
118
|
};
|
|
119
119
|
|
|
120
|
+
/**
|
|
121
|
+
* Shared retention rule for the detached runner's positional log channel.
|
|
122
|
+
* Sender offsets and gateway persistence must count the same events. Host
|
|
123
|
+
* lifecycle output stays in worker logs; bounded performance markers also
|
|
124
|
+
* reach the durable stream for dispatcher throughput diagnostics.
|
|
125
|
+
*/
|
|
126
|
+
export function isDurableDirectRunnerLog(event: {
|
|
127
|
+
source?: unknown;
|
|
128
|
+
line?: unknown;
|
|
129
|
+
}): boolean {
|
|
130
|
+
return (
|
|
131
|
+
typeof event.line === 'string' &&
|
|
132
|
+
event.line.length > 0 &&
|
|
133
|
+
(event.source !== 'worker' || event.line.startsWith('[perf-trace] '))
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
120
137
|
/** True when a line of the given provenance should render on the surface. */
|
|
121
138
|
export function logProvenanceReaches(
|
|
122
139
|
provenance: LogProvenance,
|
|
@@ -39,10 +39,11 @@ export type PlayRunAttemptWorkerTerminalProjectionPorts = {
|
|
|
39
39
|
};
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* winner only; it never emits a second projection
|
|
42
|
+
* Request delivery of the already committed terminal before awaited
|
|
43
|
+
* reconciliation or cleanup can delay it. Progress finalization still follows
|
|
44
|
+
* durable-truth reconciliation and failed-attempt lease release. A stale
|
|
45
|
+
* terminal fence observes the winner only; it never emits a second projection
|
|
46
|
+
* sequence.
|
|
46
47
|
*/
|
|
47
48
|
export function createPlayRunAttemptWorkerTerminalProjection(
|
|
48
49
|
ports: PlayRunAttemptWorkerTerminalProjectionPorts,
|
|
@@ -50,6 +51,7 @@ export function createPlayRunAttemptWorkerTerminalProjection(
|
|
|
50
51
|
return {
|
|
51
52
|
observeFenceLost: ports.observeFenceLost,
|
|
52
53
|
async projectCommitted(input) {
|
|
54
|
+
ports.requestTerminalProjection({ terminal: input.terminal });
|
|
53
55
|
await ports.reconcileTerminalTruth(input);
|
|
54
56
|
if (input.terminal.status === 'failed') {
|
|
55
57
|
await ports.releaseFailedAttempt({
|
|
@@ -58,7 +60,6 @@ export function createPlayRunAttemptWorkerTerminalProjection(
|
|
|
58
60
|
});
|
|
59
61
|
}
|
|
60
62
|
ports.queueProgressFinalization(input);
|
|
61
|
-
ports.requestTerminalProjection({ terminal: input.terminal });
|
|
62
63
|
},
|
|
63
64
|
};
|
|
64
65
|
}
|
|
@@ -6,6 +6,7 @@ import { fullenrichBatchStrategies } from './fullenrich-batching';
|
|
|
6
6
|
import { opensosdataBatchStrategies } from './opensosdata-batching';
|
|
7
7
|
import { zerobounceBatchStrategies } from './zerobounce-batching';
|
|
8
8
|
import { testAsyncBatchStrategies } from './test-async-batching';
|
|
9
|
+
import { testAdmissionBatchStrategies } from './test-admission-batching';
|
|
9
10
|
|
|
10
11
|
export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
|
|
11
12
|
string,
|
|
@@ -17,6 +18,7 @@ export const PLAY_RUNTIME_BATCH_OPERATION_REGISTRY: Record<
|
|
|
17
18
|
...fullenrichBatchStrategies,
|
|
18
19
|
...opensosdataBatchStrategies,
|
|
19
20
|
...testAsyncBatchStrategies,
|
|
21
|
+
...testAdmissionBatchStrategies,
|
|
20
22
|
...zerobounceBatchStrategies,
|
|
21
23
|
};
|
|
22
24
|
|
|
@@ -185,6 +185,8 @@ export type PlayRunnerBudgetChargeResult = {
|
|
|
185
185
|
};
|
|
186
186
|
|
|
187
187
|
export interface PlayRunnerContextConfig extends RetainedPipelineCarrier {
|
|
188
|
+
/** Reader-first logging cutover, frozen at admission; absent stays legacy. */
|
|
189
|
+
runtimeLogIdentity?: 'origin';
|
|
188
190
|
executorToken?: string;
|
|
189
191
|
baseUrl?: string;
|
|
190
192
|
/**
|
|
@@ -320,6 +322,8 @@ export type PlayRunnerLogEvent = {
|
|
|
320
322
|
at: string;
|
|
321
323
|
source: PlayLiveEventSource;
|
|
322
324
|
line: string;
|
|
325
|
+
/** Origin identity, preserved across live and terminal transports. */
|
|
326
|
+
entryId?: string;
|
|
323
327
|
/** Search-only context; omitted before the compatible Convex projection. */
|
|
324
328
|
context?: PlayLogContext;
|
|
325
329
|
};
|
|
@@ -411,6 +415,7 @@ export type PlayRunnerResult =
|
|
|
411
415
|
outputRowCount?: number;
|
|
412
416
|
rowOutcomes?: PlayRunnerRowOutcomeSummary;
|
|
413
417
|
logs: string[];
|
|
418
|
+
logIds?: string[];
|
|
414
419
|
/** Search-only context aligned to logs; omitted when every entry is null. */
|
|
415
420
|
logContexts?: Array<PlayLogContext | null>;
|
|
416
421
|
stats: Record<string, unknown>;
|
|
@@ -426,6 +431,7 @@ export type PlayRunnerResult =
|
|
|
426
431
|
status: 'suspended';
|
|
427
432
|
suspension: PlayExecutionSuspension;
|
|
428
433
|
logs: string[];
|
|
434
|
+
logIds?: string[];
|
|
429
435
|
/** Search-only context aligned to logs; omitted when every entry is null. */
|
|
430
436
|
logContexts?: Array<PlayLogContext | null>;
|
|
431
437
|
stats: Record<string, unknown>;
|
|
@@ -442,6 +448,7 @@ export type PlayRunnerResult =
|
|
|
442
448
|
error: string;
|
|
443
449
|
errors?: PlayRunFailureDetails[];
|
|
444
450
|
logs: string[];
|
|
451
|
+
logIds?: string[];
|
|
445
452
|
/** Search-only context aligned to logs; omitted when every entry is null. */
|
|
446
453
|
logContexts?: Array<PlayLogContext | null>;
|
|
447
454
|
stats: Record<string, unknown>;
|
|
@@ -387,6 +387,7 @@ export const RUN_SCHEDULER_OUTBOX_EVENT_TYPE = {
|
|
|
387
387
|
runCancelled: RUN_EVENT_KIND.runCancelled,
|
|
388
388
|
runEvents: 'run.events',
|
|
389
389
|
runnerLogs: 'runner.logs',
|
|
390
|
+
receiptEvent: 'receipt.event',
|
|
390
391
|
runWaitTimeout: 'run.wait_timeout',
|
|
391
392
|
} as const;
|
|
392
393
|
|
|
@@ -549,6 +550,7 @@ export const RUN_SCHEDULER_OUTBOX_EVENT_TYPE_SQL_LITERAL = {
|
|
|
549
550
|
runCancelled: sqlStringLiteral(RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runCancelled),
|
|
550
551
|
runEvents: sqlStringLiteral(RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runEvents),
|
|
551
552
|
runnerLogs: sqlStringLiteral(RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runnerLogs),
|
|
553
|
+
receiptEvent: sqlStringLiteral(RUN_SCHEDULER_OUTBOX_EVENT_TYPE.receiptEvent),
|
|
552
554
|
runWaitTimeout: sqlStringLiteral(
|
|
553
555
|
RUN_SCHEDULER_OUTBOX_EVENT_TYPE.runWaitTimeout,
|
|
554
556
|
),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { normalizeRunLogIds } from './run-log-buffer';
|
|
1
2
|
import { RUN_EVENT_KIND } from './run-event-catalog';
|
|
2
3
|
import type { PlayRunLedgerIngressEvent } from './run-ledger';
|
|
3
4
|
import type { PlayLogContext } from './log-context';
|
|
@@ -48,6 +49,14 @@ function splitRunEventForAppRuntime(
|
|
|
48
49
|
const lineContexts = hasLineContexts
|
|
49
50
|
? (rawLineContexts as Array<PlayLogContext | null>)
|
|
50
51
|
: null;
|
|
52
|
+
const payloadOwnsLineIds =
|
|
53
|
+
payload !== null && Object.hasOwn(payload, 'lineIds');
|
|
54
|
+
if (payloadOwnsLineIds && Object.hasOwn(event, 'lineIds'))
|
|
55
|
+
throw new TypeError('Run log identities must have one event owner.');
|
|
56
|
+
const lineIds = normalizeRunLogIds(
|
|
57
|
+
payloadOwnsLineIds ? payload?.lineIds : event.lineIds,
|
|
58
|
+
lines,
|
|
59
|
+
);
|
|
51
60
|
const payloadChannelOffset =
|
|
52
61
|
typeof payload?.channelOffset === 'number'
|
|
53
62
|
? payload.channelOffset
|
|
@@ -58,6 +67,7 @@ function splitRunEventForAppRuntime(
|
|
|
58
67
|
const {
|
|
59
68
|
lines: _outerLines,
|
|
60
69
|
lineContexts: _outerLineContexts,
|
|
70
|
+
lineIds: _outerLineIds,
|
|
61
71
|
...eventWithoutOuterLines
|
|
62
72
|
} = event;
|
|
63
73
|
let index = 0;
|
|
@@ -68,17 +78,28 @@ function splitRunEventForAppRuntime(
|
|
|
68
78
|
const lineBytes = appRuntimeTextEncoder.encode(
|
|
69
79
|
JSON.stringify(lines[index]!),
|
|
70
80
|
).length;
|
|
81
|
+
const metadataBytes =
|
|
82
|
+
(lineContexts
|
|
83
|
+
? appRuntimeTextEncoder.encode(JSON.stringify(lineContexts[index]))
|
|
84
|
+
.length + 1
|
|
85
|
+
: 0) +
|
|
86
|
+
(lineIds
|
|
87
|
+
? appRuntimeTextEncoder.encode(JSON.stringify(lineIds[index]))
|
|
88
|
+
.length + 1
|
|
89
|
+
: 0);
|
|
71
90
|
if (
|
|
72
91
|
index > start &&
|
|
73
|
-
bytes + lineBytes + 1 >
|
|
92
|
+
bytes + lineBytes + metadataBytes + 1 >
|
|
93
|
+
APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES
|
|
74
94
|
) {
|
|
75
95
|
break;
|
|
76
96
|
}
|
|
77
|
-
bytes += lineBytes + 1;
|
|
97
|
+
bytes += lineBytes + metadataBytes + 1;
|
|
78
98
|
index += 1;
|
|
79
99
|
}
|
|
80
100
|
const batchLines = lines.slice(start, index);
|
|
81
101
|
const batchLineContexts = lineContexts?.slice(start, index);
|
|
102
|
+
const batchLineIds = lineIds?.slice(start, index);
|
|
82
103
|
split.push(
|
|
83
104
|
payload
|
|
84
105
|
? {
|
|
@@ -87,13 +108,19 @@ function splitRunEventForAppRuntime(
|
|
|
87
108
|
? { lineContexts: batchLineContexts }
|
|
88
109
|
: {}),
|
|
89
110
|
...(payloadOwnsLines ? {} : { lines: batchLines }),
|
|
111
|
+
...(!payloadOwnsLineIds && batchLineIds
|
|
112
|
+
? { lineIds: batchLineIds }
|
|
113
|
+
: {}),
|
|
90
114
|
payload: {
|
|
91
115
|
...Object.fromEntries(
|
|
92
116
|
Object.entries(payload).filter(
|
|
93
|
-
([key]) => key !== 'lineContexts',
|
|
117
|
+
([key]) => key !== 'lineContexts' && key !== 'lineIds',
|
|
94
118
|
),
|
|
95
119
|
),
|
|
96
120
|
...(payloadOwnsLines ? { lines: batchLines } : {}),
|
|
121
|
+
...(payloadOwnsLineIds && batchLineIds
|
|
122
|
+
? { lineIds: batchLineIds }
|
|
123
|
+
: {}),
|
|
97
124
|
...(payloadOwnsLineContexts && batchLineContexts
|
|
98
125
|
? { lineContexts: batchLineContexts }
|
|
99
126
|
: {}),
|
|
@@ -109,6 +136,7 @@ function splitRunEventForAppRuntime(
|
|
|
109
136
|
...eventWithoutOuterLines,
|
|
110
137
|
lines: batchLines,
|
|
111
138
|
...(batchLineContexts ? { lineContexts: batchLineContexts } : {}),
|
|
139
|
+
...(batchLineIds ? { lineIds: batchLineIds } : {}),
|
|
112
140
|
...(eventChannelOffset === undefined
|
|
113
141
|
? {}
|
|
114
142
|
: { channelOffset: eventChannelOffset + start }),
|