deepline 0.3.71 → 0.3.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +162 -62
- package/dist/bundling-sources/shared_libs/play-runtime/log-provenance.ts +116 -4
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +4 -1
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +10 -5
- package/dist/cli/index.js +324 -123
- package/dist/cli/index.mjs +324 -123
- package/dist/{compiler-manifest-IkyvsUO-.d.mts → compiler-manifest-CMNgA_JQ.d.mts} +7 -1
- package/dist/{compiler-manifest-IkyvsUO-.d.ts → compiler-manifest-CMNgA_JQ.d.ts} +7 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +32 -3
- package/dist/index.mjs +32 -3
- package/dist/install-integrity.json +2 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/package.json +1 -1
|
@@ -199,7 +199,7 @@ export const SDK_RELEASE = {
|
|
|
199
199
|
// available at toolResponse.rawV2 while toolResponse.raw and all declared
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
|
-
version: '0.3.
|
|
202
|
+
version: '0.3.72',
|
|
203
203
|
updateSummary:
|
|
204
204
|
'Automatic CLI updates are now enabled by default. To opt out, run `deepline settings autoupdate off`; use `deepline settings autoupdate on` to re-enable updates or `deepline settings autoupdate pin <version>` to hold an exact release. This release also adds raw-v2 tool responses at toolResponse.rawV2 while preserving existing toolResponse.raw and declared getters.',
|
|
205
205
|
packageCapabilities: {
|
|
@@ -195,8 +195,11 @@ import {
|
|
|
195
195
|
type PlayAuthoringRuntimeContext,
|
|
196
196
|
} from '../plays/authoring-contract';
|
|
197
197
|
import {
|
|
198
|
+
LOG_LEVELS,
|
|
198
199
|
formatCtxFetchHttpFailureDiagnostic,
|
|
199
|
-
|
|
200
|
+
tagLogEntry,
|
|
201
|
+
tagLogLevel,
|
|
202
|
+
type LogLevel,
|
|
200
203
|
} from './log-provenance';
|
|
201
204
|
import {
|
|
202
205
|
DURABLE_RECEIPT_WAIT_DELAY_MS,
|
|
@@ -1432,6 +1435,22 @@ const IN_MEMORY_STEP_RESULT_PREVIEW_LIMIT = 25;
|
|
|
1432
1435
|
*/
|
|
1433
1436
|
const MAX_CELL_PRODUCER_ATTEMPTS = 12;
|
|
1434
1437
|
const BATCH_SIZE_LOG_SAMPLE_LIMIT = 10;
|
|
1438
|
+
|
|
1439
|
+
/**
|
|
1440
|
+
* A 100k-row Play must not turn `ctx.log` into a second data plane. Keep an
|
|
1441
|
+
* initial useful window, then deterministic samples and explicit durable
|
|
1442
|
+
* warnings. Warn/error are never sampled: losing the actual failure is worse
|
|
1443
|
+
* than spending additional log capacity.
|
|
1444
|
+
*/
|
|
1445
|
+
const LOG_RATE_LIMIT_POLICY: Readonly<
|
|
1446
|
+
Record<Exclude<LogLevel, 'warn' | 'error'>, { first: number; every: number }>
|
|
1447
|
+
> = {
|
|
1448
|
+
debug: { first: 200, every: 100 },
|
|
1449
|
+
info: { first: 1_000, every: 100 },
|
|
1450
|
+
};
|
|
1451
|
+
const LOG_RATE_LIMIT_MARKER_EVERY = 1_000;
|
|
1452
|
+
|
|
1453
|
+
type LogRateLimitState = { seen: number; sampled: number; suppressed: number };
|
|
1435
1454
|
const STEP_PROGRAM_MAP_DEFINITION = Symbol('deepline.stepProgramMapDefinition');
|
|
1436
1455
|
|
|
1437
1456
|
function shouldPersistMapCellField(fieldName: string): boolean {
|
|
@@ -2095,6 +2114,10 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
2095
2114
|
#options: ContextOptions;
|
|
2096
2115
|
private readonly executionScope: RunExecutionScope;
|
|
2097
2116
|
private logBuffer: string[] = [];
|
|
2117
|
+
private readonly logRateLimitState = new Map<
|
|
2118
|
+
Exclude<LogLevel, 'warn' | 'error'>,
|
|
2119
|
+
LogRateLimitState
|
|
2120
|
+
>();
|
|
2098
2121
|
private readonly ctxFetchHttpFailureDiagnosticIdentities = new Set<string>();
|
|
2099
2122
|
private checkpoint: PlayCheckpoint;
|
|
2100
2123
|
private readonly durableCallCacheEpochMs: number;
|
|
@@ -3022,7 +3045,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3022
3045
|
attempt,
|
|
3023
3046
|
retryAfter: null,
|
|
3024
3047
|
});
|
|
3025
|
-
this.
|
|
3048
|
+
this.runtimeLog(
|
|
3026
3049
|
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
3027
3050
|
stage: 'transport',
|
|
3028
3051
|
secret_name: secret.name,
|
|
@@ -3077,7 +3100,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3077
3100
|
retry_after_exceeds_cap: retry.retryAfterExceedsCap,
|
|
3078
3101
|
};
|
|
3079
3102
|
if (!response.ok) {
|
|
3080
|
-
this.
|
|
3103
|
+
this.runtimeLog(
|
|
3081
3104
|
`[runtime.secret_resolution_failure] ${JSON.stringify(responseDiagnostic)}`,
|
|
3082
3105
|
);
|
|
3083
3106
|
cancelRuntimeResponseBody(response);
|
|
@@ -3094,7 +3117,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3094
3117
|
try {
|
|
3095
3118
|
payload = await response.json();
|
|
3096
3119
|
} catch (error) {
|
|
3097
|
-
this.
|
|
3120
|
+
this.runtimeLog(
|
|
3098
3121
|
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
3099
3122
|
...responseDiagnostic,
|
|
3100
3123
|
response_kind: 'invalid_json',
|
|
@@ -3115,7 +3138,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3115
3138
|
? (payload as { value: string }).value
|
|
3116
3139
|
: null;
|
|
3117
3140
|
if (!value) {
|
|
3118
|
-
this.
|
|
3141
|
+
this.runtimeLog(
|
|
3119
3142
|
`[runtime.secret_resolution_failure] ${JSON.stringify({
|
|
3120
3143
|
...responseDiagnostic,
|
|
3121
3144
|
response_kind:
|
|
@@ -3131,7 +3154,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3131
3154
|
);
|
|
3132
3155
|
}
|
|
3133
3156
|
if (attempt > 1) {
|
|
3134
|
-
this.
|
|
3157
|
+
this.runtimeLog(
|
|
3135
3158
|
`[runtime.secret_resolution_recovered] ${JSON.stringify({
|
|
3136
3159
|
secret_name: secret.name,
|
|
3137
3160
|
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
@@ -3739,7 +3762,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3739
3762
|
if (normalized) byKey.set(normalized.key, normalized);
|
|
3740
3763
|
}
|
|
3741
3764
|
if (runtimeReceiptReadTraceEnabled) {
|
|
3742
|
-
this.
|
|
3765
|
+
this.runtimeLog(
|
|
3743
3766
|
`[runtime-receipt-normalize] ${runtimeReceiptReadSummary({
|
|
3744
3767
|
requested: uniqueKeys,
|
|
3745
3768
|
receipts,
|
|
@@ -3751,7 +3774,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3751
3774
|
receipts,
|
|
3752
3775
|
byKey,
|
|
3753
3776
|
});
|
|
3754
|
-
if (trace) this.
|
|
3777
|
+
if (trace) this.runtimeLog(trace);
|
|
3755
3778
|
}
|
|
3756
3779
|
return byKey;
|
|
3757
3780
|
}
|
|
@@ -3806,7 +3829,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3806
3829
|
if (normalized) byKey.set(normalized.key, normalized);
|
|
3807
3830
|
}
|
|
3808
3831
|
if (runtimeReceiptReadTraceEnabled) {
|
|
3809
|
-
this.
|
|
3832
|
+
this.runtimeLog(
|
|
3810
3833
|
`[runtime-receipt-claim-normalize] reclaim_running=${reclaimRunning} force_refresh=${forceRefresh} force_failed_refresh=${forceFailedRefresh} ` +
|
|
3811
3834
|
runtimeReceiptReadSummary({
|
|
3812
3835
|
requested: uniqueKeys,
|
|
@@ -4039,7 +4062,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4039
4062
|
completedByKey.has(key),
|
|
4040
4063
|
).length;
|
|
4041
4064
|
if (recoveredCount > 0) {
|
|
4042
|
-
this.
|
|
4065
|
+
this.runtimeLog(
|
|
4043
4066
|
`Runtime tool receipt completion response reconciled ${recoveredCount}/${missingKeys.length} missing receipt(s) from durable store read-back.`,
|
|
4044
4067
|
);
|
|
4045
4068
|
}
|
|
@@ -4326,7 +4349,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4326
4349
|
receiptKeys: keys,
|
|
4327
4350
|
store: this.durableReceiptExecutionStore(),
|
|
4328
4351
|
maxAttempts,
|
|
4329
|
-
log: (message) => this.
|
|
4352
|
+
log: (message) => this.runtimeLog(message),
|
|
4330
4353
|
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
4331
4354
|
});
|
|
4332
4355
|
}
|
|
@@ -4430,7 +4453,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
4430
4453
|
executionLockTtlMs: opts.executionLockTtlMs,
|
|
4431
4454
|
toolErrorSchemaVersion: this.currentToolErrorSchemaVersion,
|
|
4432
4455
|
formatError: (error) => this.formatRuntimeError(error),
|
|
4433
|
-
log: (message) => this.
|
|
4456
|
+
log: (message) => this.runtimeLog(message),
|
|
4434
4457
|
execute: ({ leaseId }) => execute(leaseId),
|
|
4435
4458
|
});
|
|
4436
4459
|
} finally {
|
|
@@ -5781,7 +5804,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
5781
5804
|
completed?.status !== 'completed' &&
|
|
5782
5805
|
completed?.status !== 'skipped'
|
|
5783
5806
|
) {
|
|
5784
|
-
this.
|
|
5807
|
+
this.runtimeLog(
|
|
5785
5808
|
`Durable tool call ${receiptKey} completed live after its receipt lease moved; using live result without overwriting the receipt.`,
|
|
5786
5809
|
);
|
|
5787
5810
|
}
|
|
@@ -6018,7 +6041,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6018
6041
|
'ctx.runSteps.options.description',
|
|
6019
6042
|
options.description,
|
|
6020
6043
|
);
|
|
6021
|
-
this.
|
|
6044
|
+
this.runtimeLog(options.description);
|
|
6022
6045
|
}
|
|
6023
6046
|
return (await this.executeStepProgram(program, input, 0, [], {
|
|
6024
6047
|
checkpointSteps: true,
|
|
@@ -6249,7 +6272,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6249
6272
|
immediateMaterializedRows.length = 0;
|
|
6250
6273
|
immediateMaterializedResidentBytes = 0;
|
|
6251
6274
|
immediateMaterializedCacheEnabled = false;
|
|
6252
|
-
this.
|
|
6275
|
+
this.runtimeLog(
|
|
6253
6276
|
`Dataset ${normalizedMapNamespace} result cache disabled (${reason}); using the durable runtime sheet.`,
|
|
6254
6277
|
);
|
|
6255
6278
|
};
|
|
@@ -6862,7 +6885,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
6862
6885
|
: { rows: rawMaterializedItems, droppedCount: 0, duplicateKeys: [] };
|
|
6863
6886
|
if (dedupedMaterialized.droppedCount > 0) {
|
|
6864
6887
|
const keySample = dedupedMaterialized.duplicateKeys.join(', ');
|
|
6865
|
-
this.
|
|
6888
|
+
this.runtimeLog(
|
|
6866
6889
|
`deduped ${dedupedMaterialized.droppedCount} duplicate dataset key(s) for ctx.dataset("${normalizedMapNamespace}"); keeping first occurrence` +
|
|
6867
6890
|
(keySample ? ` (e.g. ${keySample})` : ''),
|
|
6868
6891
|
);
|
|
@@ -7159,7 +7182,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7159
7182
|
const persistStartedAt = Date.now();
|
|
7160
7183
|
await persistMapRows(unpersistedRows);
|
|
7161
7184
|
if (unpersistedRows.length > 0) {
|
|
7162
|
-
this.
|
|
7185
|
+
this.runtimeLog(
|
|
7163
7186
|
`Persisted ${unpersistedRows.length} fail-fast rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
|
|
7164
7187
|
);
|
|
7165
7188
|
}
|
|
@@ -7227,7 +7250,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7227
7250
|
const persistStartedAt = Date.now();
|
|
7228
7251
|
await persistMapRows(persistRows);
|
|
7229
7252
|
if (persistRows.length > 0) {
|
|
7230
|
-
this.
|
|
7253
|
+
this.runtimeLog(
|
|
7231
7254
|
`Persisted ${persistRows.length} executed rows to sheet ${resolvedTableNamespace} in ${Date.now() - persistStartedAt}ms`,
|
|
7232
7255
|
);
|
|
7233
7256
|
}
|
|
@@ -7285,7 +7308,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7285
7308
|
expectedRows: totalInputCount,
|
|
7286
7309
|
currentRows: results,
|
|
7287
7310
|
failedRowCount: mapResult.failedRows.length,
|
|
7288
|
-
log: (line) => this.
|
|
7311
|
+
log: (line) => this.runtimeLog(line),
|
|
7289
7312
|
readPersistedRows: async (readInput) => {
|
|
7290
7313
|
if (
|
|
7291
7314
|
!this.#options.baseUrl ||
|
|
@@ -7548,7 +7571,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7548
7571
|
completedRowsToPersist.length = 0;
|
|
7549
7572
|
failedRowsToPersist.length = 0;
|
|
7550
7573
|
retainedRowsComplete = false;
|
|
7551
|
-
this.
|
|
7574
|
+
this.runtimeLog(
|
|
7552
7575
|
`Dataset ${normalizedTableNamespace} released its in-memory result cache after ${completedRowCount + failedRowCount} row(s); using the durable Runtime Sheet.`,
|
|
7553
7576
|
);
|
|
7554
7577
|
}
|
|
@@ -7583,11 +7606,11 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7583
7606
|
};
|
|
7584
7607
|
|
|
7585
7608
|
if (completedRows > 0 || pendingRows !== totalRows) {
|
|
7586
|
-
this.
|
|
7609
|
+
this.runtimeLog(
|
|
7587
7610
|
`Starting map over ${totalRows} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace}; ${completedRows} duplicate keys skipped; ${pendingRows} pending)`,
|
|
7588
7611
|
);
|
|
7589
7612
|
} else {
|
|
7590
|
-
this.
|
|
7613
|
+
this.runtimeLog(
|
|
7591
7614
|
`Starting map over ${items.length} items with ${visibleFields.length} fields (key: ${normalizedTableNamespace})`,
|
|
7592
7615
|
);
|
|
7593
7616
|
}
|
|
@@ -7764,7 +7787,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7764
7787
|
}
|
|
7765
7788
|
this.lastDatasetStep = datasetStep;
|
|
7766
7789
|
this.activeDatasetStep = null;
|
|
7767
|
-
this.
|
|
7790
|
+
this.runtimeLog(
|
|
7768
7791
|
`Map completed: ${results.length + completedRows} results (${results.length} succeeded, 0 failed, ${completedRows} duplicate keys skipped)`,
|
|
7769
7792
|
);
|
|
7770
7793
|
return {
|
|
@@ -7829,7 +7852,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
7829
7852
|
});
|
|
7830
7853
|
mapStallObserver = new RuntimeMapStallObserver({
|
|
7831
7854
|
getWriterDiagnostics: () => incrementalPersistence?.diagnostics() ?? null,
|
|
7832
|
-
log: (line) => this.
|
|
7855
|
+
log: (line) => this.runtimeLog(line),
|
|
7833
7856
|
mapName: normalizedTableNamespace,
|
|
7834
7857
|
pageOffset:
|
|
7835
7858
|
runtimeOptions?.executionRowIndexes?.[0] ??
|
|
@@ -8281,17 +8304,20 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
8281
8304
|
});
|
|
8282
8305
|
}
|
|
8283
8306
|
for (const failedRow of failedRowsToPersist.slice(0, 3)) {
|
|
8284
|
-
this.
|
|
8307
|
+
this.runtimeLog(
|
|
8285
8308
|
`row ${failedMapRowLogLabel(failedRow)} failed: ${failedRow.error ?? 'unknown error'}`,
|
|
8309
|
+
{ level: 'error' },
|
|
8286
8310
|
);
|
|
8287
8311
|
}
|
|
8288
8312
|
if (failedRowsToPersist.length > 3) {
|
|
8289
|
-
this.
|
|
8313
|
+
this.runtimeLog(
|
|
8290
8314
|
`${failedRowsToPersist.length - 3} additional row failure(s) omitted from map log`,
|
|
8315
|
+
{ level: 'warn' },
|
|
8291
8316
|
);
|
|
8292
8317
|
}
|
|
8293
|
-
this.
|
|
8318
|
+
this.runtimeLog(
|
|
8294
8319
|
`Map completed: ${succeededRows + completedRows} results (${succeededRows} succeeded, ${failedRowCount} failed, ${completedRows} duplicate keys skipped)`,
|
|
8320
|
+
failedRowCount > 0 ? { level: 'warn' } : undefined,
|
|
8295
8321
|
);
|
|
8296
8322
|
return {
|
|
8297
8323
|
completedRows: [...completedRowsToPersist].sort(
|
|
@@ -9055,8 +9081,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9055
9081
|
);
|
|
9056
9082
|
if (dispatchable.requests.length > 0) {
|
|
9057
9083
|
pass += 1;
|
|
9058
|
-
this.
|
|
9059
|
-
this.
|
|
9084
|
+
this.runtimeLog(` Batch pass ${pass}`);
|
|
9085
|
+
this.runtimeLog(
|
|
9060
9086
|
` Dispatcher launch: ready=${dispatchable.requests.length} ` +
|
|
9061
9087
|
`queued=${this.toolCallQueue.length} ` +
|
|
9062
9088
|
`in_flight_groups=${inFlightToolExecutions.size}`,
|
|
@@ -9294,7 +9320,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9294
9320
|
'direct',
|
|
9295
9321
|
);
|
|
9296
9322
|
if (cached) {
|
|
9297
|
-
this.
|
|
9323
|
+
this.runtimeLog(`Calling tool: ${toolId} recovered from checkpoint`);
|
|
9298
9324
|
// Started and settled are paired in a finally: an uncovered throw
|
|
9299
9325
|
// between them strands the tool node `running` forever in the
|
|
9300
9326
|
// snapshot, which is exactly the lossy state ADR 0018 removed.
|
|
@@ -9355,7 +9381,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9355
9381
|
);
|
|
9356
9382
|
}
|
|
9357
9383
|
}
|
|
9358
|
-
this.
|
|
9384
|
+
this.runtimeLog(
|
|
9359
9385
|
toolCachePolicy.force
|
|
9360
9386
|
? `Calling tool: ${toolId} (force)`
|
|
9361
9387
|
: `Calling tool: ${toolId}`,
|
|
@@ -9610,7 +9636,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9610
9636
|
? null
|
|
9611
9637
|
: this.getCachedToolResultCandidate(toolId, checkpointCacheKeys);
|
|
9612
9638
|
if (cached) {
|
|
9613
|
-
this.
|
|
9639
|
+
this.runtimeLog(` Row ${rowId} ${toolId}: recovered from checkpoint`);
|
|
9614
9640
|
return await this.wrapToolExecutionResult({
|
|
9615
9641
|
toolId,
|
|
9616
9642
|
status: cached.result.result == null ? 'no_result' : 'completed',
|
|
@@ -9730,7 +9756,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9730
9756
|
// of a silent continuation: the credential identity changed after the
|
|
9731
9757
|
// receipt key was prepared, so we evict the cached digest, re-resolve,
|
|
9732
9758
|
// and re-claim a fresh receipt under the new scope before retrying once.
|
|
9733
|
-
this.
|
|
9759
|
+
this.runtimeLog(
|
|
9734
9760
|
`ctx.tools.execute(${toolId}): auth_scope_changed_reclaim ` +
|
|
9735
9761
|
`(label: ${normalizedKey}); credential identity changed mid-run, ` +
|
|
9736
9762
|
`re-resolving auth scope and re-claiming a fresh receipt under the ` +
|
|
@@ -9801,7 +9827,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9801
9827
|
const existing =
|
|
9802
9828
|
this.checkpoint.resolvedBoundaries?.[preparedBoundary.boundaryId];
|
|
9803
9829
|
if (existing?.kind === 'integration_event' && 'output' in existing) {
|
|
9804
|
-
this.
|
|
9830
|
+
this.runtimeLog(
|
|
9805
9831
|
`Integration event ${preparedBoundary.boundaryId}: recovered response from checkpoint`,
|
|
9806
9832
|
);
|
|
9807
9833
|
// Wrapped like every other tool result (toolOutput.raw, getters) — the
|
|
@@ -9824,7 +9850,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9824
9850
|
boundary: preparedBoundary,
|
|
9825
9851
|
});
|
|
9826
9852
|
|
|
9827
|
-
this.
|
|
9853
|
+
this.runtimeLog(
|
|
9828
9854
|
`Armed ${handler.provider} integration event wait: ${boundary.eventKey}`,
|
|
9829
9855
|
);
|
|
9830
9856
|
this.checkpoint.resolvedBoundaries = {
|
|
@@ -9936,7 +9962,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
9936
9962
|
execution: options?.execution,
|
|
9937
9963
|
childPlayName: resolvedName,
|
|
9938
9964
|
});
|
|
9939
|
-
this.
|
|
9965
|
+
this.runtimeLog(
|
|
9940
9966
|
`ctx.runPlay(${normalizedKey}): ${childExecutionDecision.strategy} (${childExecutionDecision.reason})`,
|
|
9941
9967
|
);
|
|
9942
9968
|
|
|
@@ -10182,22 +10208,89 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10182
10208
|
});
|
|
10183
10209
|
}
|
|
10184
10210
|
|
|
10185
|
-
log(msg: string): void {
|
|
10211
|
+
log(msg: string, options?: { level?: LogLevel }): void {
|
|
10186
10212
|
assertNoSecretTaint(msg, 'ctx.log');
|
|
10187
|
-
const
|
|
10213
|
+
const requestedLevel = options?.level;
|
|
10214
|
+
if (requestedLevel !== undefined && !LOG_LEVELS.includes(requestedLevel)) {
|
|
10215
|
+
throw new Error(
|
|
10216
|
+
`ctx.log level must be one of: ${LOG_LEVELS.join(', ')}.`,
|
|
10217
|
+
);
|
|
10218
|
+
}
|
|
10219
|
+
const level = requestedLevel ?? 'info';
|
|
10220
|
+
const timestamped = `[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(msg)}`;
|
|
10221
|
+
// Stamp the default too: a customer message that happens to begin with
|
|
10222
|
+
// "[error]" is still info unless the author selected a higher severity.
|
|
10223
|
+
const line = tagLogEntry('user', level, timestamped);
|
|
10224
|
+
if (!this.shouldKeepRateLimitedLog(level)) return;
|
|
10225
|
+
this.logBuffer.push(line);
|
|
10226
|
+
this.#options.onLog?.(line);
|
|
10227
|
+
if (this.#options.verbose) console.log(line);
|
|
10228
|
+
}
|
|
10229
|
+
|
|
10230
|
+
/**
|
|
10231
|
+
* Runtime-owned progress and failure narration must stay durable even when
|
|
10232
|
+
* authored per-row logs are being sampled. Preserve the untagged historical
|
|
10233
|
+
* wire shape by default: several runtime diagnostics contain trailing JSON
|
|
10234
|
+
* consumed by older readers. Callers may opt into an explicit severity.
|
|
10235
|
+
*/
|
|
10236
|
+
private runtimeLog(msg: string, options?: { level?: LogLevel }): void {
|
|
10237
|
+
assertNoSecretTaint(msg, 'runtime log');
|
|
10238
|
+
const timestamped = `[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(msg)}`;
|
|
10239
|
+
const line = options?.level
|
|
10240
|
+
? tagLogLevel(timestamped, options.level)
|
|
10241
|
+
: timestamped;
|
|
10188
10242
|
this.logBuffer.push(line);
|
|
10189
10243
|
this.#options.onLog?.(line);
|
|
10190
10244
|
if (this.#options.verbose) console.log(line);
|
|
10191
10245
|
}
|
|
10192
10246
|
|
|
10247
|
+
/** Rate-limit chatty authored logs before they consume the live stream or durable retention. */
|
|
10248
|
+
private shouldKeepRateLimitedLog(level: LogLevel): boolean {
|
|
10249
|
+
if (level === 'warn' || level === 'error') return true;
|
|
10250
|
+
const policy = LOG_RATE_LIMIT_POLICY[level];
|
|
10251
|
+
const state = this.logRateLimitState.get(level) ?? {
|
|
10252
|
+
seen: 0,
|
|
10253
|
+
sampled: 0,
|
|
10254
|
+
suppressed: 0,
|
|
10255
|
+
};
|
|
10256
|
+
state.seen += 1;
|
|
10257
|
+
const keep = state.seen <= policy.first || state.seen % policy.every === 0;
|
|
10258
|
+
if (keep) {
|
|
10259
|
+
if (state.seen > policy.first) state.sampled += 1;
|
|
10260
|
+
this.logRateLimitState.set(level, state);
|
|
10261
|
+
return true;
|
|
10262
|
+
}
|
|
10263
|
+
state.suppressed += 1;
|
|
10264
|
+
this.logRateLimitState.set(level, state);
|
|
10265
|
+
if (
|
|
10266
|
+
state.suppressed === 1 ||
|
|
10267
|
+
state.suppressed % LOG_RATE_LIMIT_MARKER_EVERY === 0
|
|
10268
|
+
) {
|
|
10269
|
+
const marker = `[log rate limited: level=${level} retained=${policy.first + state.sampled} sampled=${state.sampled} suppressed=${state.suppressed}; showing first ${policy.first}, then 1 every ${policy.every}]`;
|
|
10270
|
+
const line = tagLogEntry(
|
|
10271
|
+
'diagnostic',
|
|
10272
|
+
'warn',
|
|
10273
|
+
`[${new Date().toISOString()}] ${marker}`,
|
|
10274
|
+
);
|
|
10275
|
+
this.logBuffer.push(line);
|
|
10276
|
+
this.#options.onLog?.(line);
|
|
10277
|
+
if (this.#options.verbose) console.log(line);
|
|
10278
|
+
}
|
|
10279
|
+
return false;
|
|
10280
|
+
}
|
|
10281
|
+
|
|
10193
10282
|
/**
|
|
10194
10283
|
* Emit a runtime-authored diagnostic through the same durable log path as
|
|
10195
10284
|
* ctx.log without changing the customer-authored log wire format.
|
|
10196
10285
|
*/
|
|
10197
|
-
private runtimeDiagnosticLog(
|
|
10286
|
+
private runtimeDiagnosticLog(
|
|
10287
|
+
message: string,
|
|
10288
|
+
level: LogLevel = 'warn',
|
|
10289
|
+
): void {
|
|
10198
10290
|
assertNoSecretTaint(message, 'runtime diagnostic log');
|
|
10199
|
-
const line =
|
|
10291
|
+
const line = tagLogEntry(
|
|
10200
10292
|
'diagnostic',
|
|
10293
|
+
level,
|
|
10201
10294
|
`[${new Date().toISOString()}] ${this.secretRedactor.redactRegisteredSecrets(message)}`,
|
|
10202
10295
|
);
|
|
10203
10296
|
this.logBuffer.push(line);
|
|
@@ -10438,7 +10531,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10438
10531
|
existing?.kind === 'fetch' &&
|
|
10439
10532
|
'output' in existing
|
|
10440
10533
|
) {
|
|
10441
|
-
this.
|
|
10534
|
+
this.runtimeLog(
|
|
10535
|
+
`ctx.fetch(${url}): recovered response from checkpoint`,
|
|
10536
|
+
);
|
|
10442
10537
|
const checkpointOutput = existing.output as PlayFetchResponse;
|
|
10443
10538
|
if (!checkpointOutput.ok) {
|
|
10444
10539
|
this.logCtxFetchHttpFailure({
|
|
@@ -10556,7 +10651,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10556
10651
|
canRetryTransport &&
|
|
10557
10652
|
attempt < FETCH_TRANSPORT_MAX_ATTEMPTS
|
|
10558
10653
|
) {
|
|
10559
|
-
this.
|
|
10654
|
+
this.runtimeLog(
|
|
10560
10655
|
`ctx.fetch(${method} ${url}) transport failed on attempt ${attempt}/${FETCH_TRANSPORT_MAX_ATTEMPTS}; retrying: ${message}`,
|
|
10561
10656
|
);
|
|
10562
10657
|
await sleepWithinCtxFetchDeadline({
|
|
@@ -10665,7 +10760,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10665
10760
|
const executeStep = async (): Promise<T> => {
|
|
10666
10761
|
const existing = this.checkpoint.resolvedBoundaries?.[boundaryId];
|
|
10667
10762
|
if (existing?.kind === 'step' && 'output' in existing) {
|
|
10668
|
-
this.
|
|
10763
|
+
this.runtimeLog(
|
|
10669
10764
|
`ctx.step(${normalizedKey}): recovered result from checkpoint`,
|
|
10670
10765
|
);
|
|
10671
10766
|
return existing.output as T;
|
|
@@ -10869,7 +10964,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
10869
10964
|
leaseLost ??= error;
|
|
10870
10965
|
},
|
|
10871
10966
|
onTransientFailure: (error) => {
|
|
10872
|
-
this.
|
|
10967
|
+
this.runtimeLog(
|
|
10873
10968
|
`Pending tool receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
|
|
10874
10969
|
);
|
|
10875
10970
|
},
|
|
@@ -11211,7 +11306,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11211
11306
|
cohortLeaseLost ??= error;
|
|
11212
11307
|
},
|
|
11213
11308
|
onTransientFailure: (error) => {
|
|
11214
|
-
this.
|
|
11309
|
+
this.runtimeLog(
|
|
11215
11310
|
`Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
11216
11311
|
error instanceof Error ? error.message : String(error)
|
|
11217
11312
|
}`,
|
|
@@ -11829,7 +11924,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11829
11924
|
),
|
|
11830
11925
|
),
|
|
11831
11926
|
onTransientFailure: (error) => {
|
|
11832
|
-
this.
|
|
11927
|
+
this.runtimeLog(
|
|
11833
11928
|
`Native Tool Call Receipt Cohort heartbeat transport failed; retrying: ${
|
|
11834
11929
|
error instanceof Error ? error.message : String(error)
|
|
11835
11930
|
}`,
|
|
@@ -11918,7 +12013,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11918
12013
|
|
|
11919
12014
|
const toolSettlements = await Promise.allSettled(
|
|
11920
12015
|
[...byTool.entries()].map(async ([toolId, requests]) => {
|
|
11921
|
-
this.
|
|
12016
|
+
this.runtimeLog(
|
|
12017
|
+
`Executing tool batch ${toolId}: ${requests.length} calls`,
|
|
12018
|
+
);
|
|
11922
12019
|
const successfulLiveStepCallIds = new Set<string>();
|
|
11923
12020
|
|
|
11924
12021
|
const recordToolStep = (stepRequests: ToolCallRequest[]): void => {
|
|
@@ -11973,7 +12070,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
11973
12070
|
? undefined
|
|
11974
12071
|
: this.getCachedToolResult(toolId, req.cacheKey);
|
|
11975
12072
|
if (cached?.done) {
|
|
11976
|
-
this.
|
|
12073
|
+
this.runtimeLog(
|
|
12074
|
+
` Row ${req.rowId} ${toolId}: recovered from checkpoint`,
|
|
12075
|
+
);
|
|
11977
12076
|
const resolver = this.toolCallResolvers.get(req.callId);
|
|
11978
12077
|
if (resolver) {
|
|
11979
12078
|
resolver.resolve(cached.result);
|
|
@@ -12553,7 +12652,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12553
12652
|
),
|
|
12554
12653
|
) || DURABLE_RECEIPT_WAIT_MAX_ATTEMPTS;
|
|
12555
12654
|
if (runtimeReceiptReadTraceEnabled) {
|
|
12556
|
-
this.
|
|
12655
|
+
this.runtimeLog(
|
|
12557
12656
|
`[runtime-receipt-wait] phase=start requested=${pendingWaits.size} ` +
|
|
12558
12657
|
`request_digests=${[...pendingWaits.keys()]
|
|
12559
12658
|
.map(runtimeReceiptKeyDigest)
|
|
@@ -12650,7 +12749,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
12650
12749
|
: ['unknown'];
|
|
12651
12750
|
},
|
|
12652
12751
|
);
|
|
12653
|
-
this.
|
|
12752
|
+
this.runtimeLog(
|
|
12654
12753
|
`[runtime-receipt-wait] phase=delivery settled=${deliverySettlements.length} ` +
|
|
12655
12754
|
`rejected=${rejected.length} rejected_digests=${rejected.join(',')}`,
|
|
12656
12755
|
);
|
|
@@ -13540,7 +13639,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13540
13639
|
);
|
|
13541
13640
|
}
|
|
13542
13641
|
if (runtimeReceiptReadTraceEnabled) {
|
|
13543
|
-
this.
|
|
13642
|
+
this.runtimeLog(
|
|
13544
13643
|
`[perf] tool call id=${toolId} phase=governor_admission elapsed_ms=${Date.now() - admissionStartedAt}`,
|
|
13545
13644
|
);
|
|
13546
13645
|
}
|
|
@@ -13693,7 +13792,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13693
13792
|
}): Promise<void> => {
|
|
13694
13793
|
transportAttempt += 1;
|
|
13695
13794
|
const diagnostic = describeTransportError(input.error);
|
|
13696
|
-
this.
|
|
13795
|
+
this.runtimeLog(
|
|
13697
13796
|
`[runtime.transport_failure] ${JSON.stringify({
|
|
13698
13797
|
tool_id: toolId,
|
|
13699
13798
|
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
@@ -13713,7 +13812,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13713
13812
|
'plays.transport_retry_attempt',
|
|
13714
13813
|
transportAttempt,
|
|
13715
13814
|
);
|
|
13716
|
-
this.
|
|
13815
|
+
this.runtimeLog(
|
|
13717
13816
|
`Tool ${toolId} transport failed calling ${url} on attempt ${transportAttempt}/${TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS}; retrying after ${retryAfterMs}ms: ${diagnostic.message ?? 'unknown transport error'}`,
|
|
13718
13817
|
);
|
|
13719
13818
|
await options?.parkProviderCall?.();
|
|
@@ -13770,7 +13869,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13770
13869
|
const ownershipStartedAt = Date.now();
|
|
13771
13870
|
await options?.beforeProviderCall?.();
|
|
13772
13871
|
if (runtimeReceiptReadTraceEnabled) {
|
|
13773
|
-
this.
|
|
13872
|
+
this.runtimeLog(
|
|
13774
13873
|
`[perf] tool call id=${toolId} phase=before_provider_ownership elapsed_ms=${Date.now() - ownershipStartedAt}`,
|
|
13775
13874
|
);
|
|
13776
13875
|
}
|
|
@@ -13795,7 +13894,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13795
13894
|
abortController?.abort(error);
|
|
13796
13895
|
},
|
|
13797
13896
|
onTransientFailure: (error) => {
|
|
13798
|
-
this.
|
|
13897
|
+
this.runtimeLog(
|
|
13799
13898
|
`Tool ${toolId} receipt heartbeat transport failed; retrying: ${error instanceof Error ? error.message : String(error)}`,
|
|
13800
13899
|
);
|
|
13801
13900
|
},
|
|
@@ -13874,7 +13973,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13874
13973
|
signal: abortController?.signal,
|
|
13875
13974
|
onSelected: (value) => {
|
|
13876
13975
|
selected = value;
|
|
13877
|
-
this.
|
|
13976
|
+
this.runtimeLog(
|
|
13878
13977
|
`[fixture.response_delay] ${JSON.stringify({
|
|
13879
13978
|
outcome: 'scheduled',
|
|
13880
13979
|
provider: canonicalProvider,
|
|
@@ -13889,7 +13988,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13889
13988
|
},
|
|
13890
13989
|
});
|
|
13891
13990
|
} catch (error) {
|
|
13892
|
-
this.
|
|
13991
|
+
this.runtimeLog(
|
|
13893
13992
|
`[fixture.response_delay] ${JSON.stringify({
|
|
13894
13993
|
outcome: 'aborted',
|
|
13895
13994
|
provider: canonicalProvider,
|
|
@@ -13904,7 +14003,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13904
14003
|
);
|
|
13905
14004
|
throw error;
|
|
13906
14005
|
}
|
|
13907
|
-
this.
|
|
14006
|
+
this.runtimeLog(
|
|
13908
14007
|
`[fixture.response_delay] ${JSON.stringify({
|
|
13909
14008
|
outcome: 'completed',
|
|
13910
14009
|
provider: canonicalProvider,
|
|
@@ -13994,7 +14093,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
13994
14093
|
runtimeReceiptReadTraceEnabled &&
|
|
13995
14094
|
integrationFetchStartedAt !== null
|
|
13996
14095
|
) {
|
|
13997
|
-
this.
|
|
14096
|
+
this.runtimeLog(
|
|
13998
14097
|
`[perf] tool call id=${toolId} phase=integration_fetch_body elapsed_ms=${Date.now() - integrationFetchStartedAt} status=${response.status}`,
|
|
13999
14098
|
);
|
|
14000
14099
|
}
|
|
@@ -14083,7 +14182,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14083
14182
|
fixtureReplaySafe;
|
|
14084
14183
|
if (!transportReplaySafe) {
|
|
14085
14184
|
const diagnostic = describeTransportError(transportError);
|
|
14086
|
-
this.
|
|
14185
|
+
this.runtimeLog(
|
|
14087
14186
|
`[runtime.transport_failure] ${JSON.stringify({
|
|
14088
14187
|
tool_id: toolId,
|
|
14089
14188
|
gateway_origin: transportGatewayOriginForDiagnostic(url),
|
|
@@ -14230,8 +14329,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14230
14329
|
`plays.${retryAttributePrefix}_attempt`,
|
|
14231
14330
|
httpFailureAttempt,
|
|
14232
14331
|
);
|
|
14233
|
-
this.
|
|
14332
|
+
this.runtimeLog(
|
|
14234
14333
|
`Tool ${toolId} returned ${response.status}; retrying after ${failure.retryDelayMs}ms`,
|
|
14334
|
+
{ level: 'warn' },
|
|
14235
14335
|
);
|
|
14236
14336
|
retryActivityEmitted = true;
|
|
14237
14337
|
this.emitExecutionEvent({
|
|
@@ -14275,7 +14375,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
14275
14375
|
});
|
|
14276
14376
|
continue;
|
|
14277
14377
|
}
|
|
14278
|
-
this.
|
|
14378
|
+
this.runtimeLog(failure.error.message, { level: 'error' });
|
|
14279
14379
|
throw failure.error;
|
|
14280
14380
|
}
|
|
14281
14381
|
|