@granular-software/sdk 0.4.45 → 0.4.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-evals.d.mts +10 -3
- package/dist/agent-evals.d.ts +10 -3
- package/dist/agent-evals.js +1067 -408
- package/dist/agent-evals.js.map +1 -1
- package/dist/agent-evals.mjs +1067 -408
- package/dist/agent-evals.mjs.map +1 -1
- package/dist/agent-harness.d.mts +1 -0
- package/dist/agent-harness.d.ts +1 -0
- package/dist/agent-harness.js +87 -11
- package/dist/agent-harness.js.map +1 -1
- package/dist/agent-harness.mjs +87 -11
- package/dist/agent-harness.mjs.map +1 -1
- package/dist/cli/index.js +114 -18
- package/dist/{client-BZ8NuQ_e.d.ts → client-ButG6ePW.d.ts} +7 -1
- package/dist/{client-CBQFvuKf.d.mts → client-C1UqPDwe.d.mts} +7 -1
- package/dist/index.d.mts +3 -4
- package/dist/index.d.ts +3 -4
- package/dist/index.js +201 -29
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +201 -29
- package/dist/index.mjs.map +1 -1
- package/dist/{spend-tAz2a16I.d.mts → spend-D2Vy3N1D.d.mts} +113 -4
- package/dist/{spend-tAz2a16I.d.ts → spend-D2Vy3N1D.d.ts} +113 -4
- package/dist/spend.d.mts +1 -2
- package/dist/spend.d.ts +1 -2
- package/package.json +2 -2
package/dist/agent-evals.mjs
CHANGED
|
@@ -4003,6 +4003,9 @@ var MAX_TIMER_DELAY_MS = 2147483647;
|
|
|
4003
4003
|
var DEBUG_WS = process.env.GRANULAR_DEBUG_WS === "1";
|
|
4004
4004
|
var DEFAULT_RPC_TIMEOUT_MS = 3e4;
|
|
4005
4005
|
var DOMAIN_PACKAGE_RPC_TIMEOUT_MS = 12e4;
|
|
4006
|
+
var EFFECT_CONTROL_RPC_TIMEOUT_MS = 12e4;
|
|
4007
|
+
var DEFAULT_RECONNECT_DELAY_MS = 3e3;
|
|
4008
|
+
var DEFAULT_MAX_RECONNECT_ATTEMPTS = 5;
|
|
4006
4009
|
function debugWs(...args) {
|
|
4007
4010
|
if (DEBUG_WS) {
|
|
4008
4011
|
console.log(...args);
|
|
@@ -4013,6 +4016,10 @@ function rpcTimeoutMsForMethod(method) {
|
|
|
4013
4016
|
case "domain.fetchPackagePart":
|
|
4014
4017
|
case "domain.getSummary":
|
|
4015
4018
|
return DOMAIN_PACKAGE_RPC_TIMEOUT_MS;
|
|
4019
|
+
case "client.heartbeat":
|
|
4020
|
+
case "effects.publishCatalog":
|
|
4021
|
+
case "effects.refresh":
|
|
4022
|
+
return EFFECT_CONTROL_RPC_TIMEOUT_MS;
|
|
4016
4023
|
default:
|
|
4017
4024
|
return DEFAULT_RPC_TIMEOUT_MS;
|
|
4018
4025
|
}
|
|
@@ -4032,6 +4039,7 @@ var WSClient = class {
|
|
|
4032
4039
|
reconnectTimer = null;
|
|
4033
4040
|
tokenRefreshTimer = null;
|
|
4034
4041
|
isExplicitlyDisconnected = false;
|
|
4042
|
+
reconnectAttempts = 0;
|
|
4035
4043
|
options;
|
|
4036
4044
|
constructor(options) {
|
|
4037
4045
|
this.options = options;
|
|
@@ -4186,6 +4194,7 @@ var WSClient = class {
|
|
|
4186
4194
|
clearTimeout(this.reconnectTimer);
|
|
4187
4195
|
this.reconnectTimer = null;
|
|
4188
4196
|
}
|
|
4197
|
+
this.reconnectAttempts = 0;
|
|
4189
4198
|
this.emit("open", {});
|
|
4190
4199
|
resolve();
|
|
4191
4200
|
});
|
|
@@ -4217,6 +4226,7 @@ var WSClient = class {
|
|
|
4217
4226
|
clearTimeout(this.reconnectTimer);
|
|
4218
4227
|
this.reconnectTimer = null;
|
|
4219
4228
|
}
|
|
4229
|
+
this.reconnectAttempts = 0;
|
|
4220
4230
|
this.emit("open", {});
|
|
4221
4231
|
resolve();
|
|
4222
4232
|
};
|
|
@@ -4276,7 +4286,8 @@ var WSClient = class {
|
|
|
4276
4286
|
return new Error(`WebSocket disconnected${suffix}`);
|
|
4277
4287
|
}
|
|
4278
4288
|
handleDisconnect(close = {}) {
|
|
4279
|
-
const
|
|
4289
|
+
const baseReconnectDelayMs = typeof this.options.reconnectDelayMs === "number" && Number.isFinite(this.options.reconnectDelayMs) && this.options.reconnectDelayMs > 0 ? this.options.reconnectDelayMs : DEFAULT_RECONNECT_DELAY_MS;
|
|
4290
|
+
const maxReconnectAttempts = typeof this.options.maxReconnectAttempts === "number" && Number.isFinite(this.options.maxReconnectAttempts) && this.options.maxReconnectAttempts >= 0 ? Math.floor(this.options.maxReconnectAttempts) : DEFAULT_MAX_RECONNECT_ATTEMPTS;
|
|
4280
4291
|
const unexpected = !this.isExplicitlyDisconnected;
|
|
4281
4292
|
const info = {
|
|
4282
4293
|
code: close.code,
|
|
@@ -4296,6 +4307,30 @@ var WSClient = class {
|
|
|
4296
4307
|
const disconnectError = this.buildDisconnectError(info);
|
|
4297
4308
|
this.rejectPending(disconnectError);
|
|
4298
4309
|
this.emit("disconnect", info);
|
|
4310
|
+
if (this.reconnectAttempts >= maxReconnectAttempts) {
|
|
4311
|
+
const reconnectInfo = {
|
|
4312
|
+
error: `WebSocket reconnect attempts exhausted after ${maxReconnectAttempts} attempt(s).`,
|
|
4313
|
+
sessionId: this.sessionId,
|
|
4314
|
+
timestamp: Date.now()
|
|
4315
|
+
};
|
|
4316
|
+
this.emit("reconnect_error", reconnectInfo);
|
|
4317
|
+
if (this.options.onReconnectError) {
|
|
4318
|
+
try {
|
|
4319
|
+
this.options.onReconnectError(reconnectInfo);
|
|
4320
|
+
} catch (callbackError) {
|
|
4321
|
+
console.error(
|
|
4322
|
+
"[Granular] onReconnectError callback failed:",
|
|
4323
|
+
callbackError
|
|
4324
|
+
);
|
|
4325
|
+
}
|
|
4326
|
+
}
|
|
4327
|
+
return;
|
|
4328
|
+
}
|
|
4329
|
+
this.reconnectAttempts += 1;
|
|
4330
|
+
const reconnectDelayMs = Math.min(
|
|
4331
|
+
3e4,
|
|
4332
|
+
baseReconnectDelayMs * 2 ** Math.max(0, this.reconnectAttempts - 1)
|
|
4333
|
+
);
|
|
4299
4334
|
info.reconnectScheduled = true;
|
|
4300
4335
|
info.reconnectDelayMs = reconnectDelayMs;
|
|
4301
4336
|
if (this.options.onUnexpectedClose) {
|
|
@@ -4750,6 +4785,9 @@ var Session = class {
|
|
|
4750
4785
|
promptCache = /* @__PURE__ */ new Map();
|
|
4751
4786
|
/** Prompt ids locally answered before the document sync catches up. */
|
|
4752
4787
|
hiddenPromptIds = /* @__PURE__ */ new Set();
|
|
4788
|
+
domainPackagePartCache = /* @__PURE__ */ new Map();
|
|
4789
|
+
domainPackagePartPromises = /* @__PURE__ */ new Map();
|
|
4790
|
+
domainPackageFetchQueue = Promise.resolve();
|
|
4753
4791
|
constructor(client, clientId, options = {}) {
|
|
4754
4792
|
this.client = client;
|
|
4755
4793
|
this.clientId = clientId || `client_${Date.now()}`;
|
|
@@ -4957,12 +4995,18 @@ var Session = class {
|
|
|
4957
4995
|
const resolvedAnswer = resolvePromptAnswer(prompt, answer);
|
|
4958
4996
|
this.promptCache.delete(promptId);
|
|
4959
4997
|
this.hiddenPromptIds.add(promptId);
|
|
4998
|
+
this.emit("prompt", { id: promptId, status: "answered" });
|
|
4960
4999
|
try {
|
|
4961
|
-
await this.client.call("prompt.answer", {
|
|
5000
|
+
const response = await this.client.call("prompt.answer", {
|
|
4962
5001
|
promptId,
|
|
4963
5002
|
answer: resolvedAnswer,
|
|
4964
5003
|
value: resolvedAnswer
|
|
4965
5004
|
});
|
|
5005
|
+
if (response && typeof response === "object" && "ok" in response && response.ok === false) {
|
|
5006
|
+
const rejected = response;
|
|
5007
|
+
const errorMessage = typeof rejected.error === "string" ? rejected.error : "Prompt answer was rejected.";
|
|
5008
|
+
throw new Error(errorMessage);
|
|
5009
|
+
}
|
|
4966
5010
|
} catch (error) {
|
|
4967
5011
|
this.hiddenPromptIds.delete(promptId);
|
|
4968
5012
|
if (prompt) {
|
|
@@ -5190,11 +5234,33 @@ var Session = class {
|
|
|
5190
5234
|
* Fetch a domain package part from the backend (no fallback).
|
|
5191
5235
|
*/
|
|
5192
5236
|
async fetchDomainPart(part) {
|
|
5193
|
-
const
|
|
5194
|
-
|
|
5195
|
-
|
|
5237
|
+
const cached = this.domainPackagePartCache.get(part);
|
|
5238
|
+
if (cached !== void 0) {
|
|
5239
|
+
return cached;
|
|
5240
|
+
}
|
|
5241
|
+
const inFlight = this.domainPackagePartPromises.get(part);
|
|
5242
|
+
if (inFlight) {
|
|
5243
|
+
return inFlight;
|
|
5244
|
+
}
|
|
5245
|
+
const fetchPromise = this.domainPackageFetchQueue.then(async () => {
|
|
5246
|
+
const result = await this.client.call("domain.fetchPackagePart", {
|
|
5247
|
+
moduleSpecifier: "@sandbox/domain",
|
|
5248
|
+
part
|
|
5249
|
+
});
|
|
5250
|
+
const content = result?.content ?? "";
|
|
5251
|
+
this.domainPackagePartCache.set(part, content);
|
|
5252
|
+
return content;
|
|
5196
5253
|
});
|
|
5197
|
-
|
|
5254
|
+
this.domainPackagePartPromises.set(part, fetchPromise);
|
|
5255
|
+
this.domainPackageFetchQueue = fetchPromise.then(
|
|
5256
|
+
() => void 0,
|
|
5257
|
+
() => void 0
|
|
5258
|
+
);
|
|
5259
|
+
try {
|
|
5260
|
+
return await fetchPromise;
|
|
5261
|
+
} finally {
|
|
5262
|
+
this.domainPackagePartPromises.delete(part);
|
|
5263
|
+
}
|
|
5198
5264
|
}
|
|
5199
5265
|
/**
|
|
5200
5266
|
* Get TypeScript class declarations for the current domain (for LLM/code gen).
|
|
@@ -5444,7 +5510,10 @@ import { ${allImports} } from "./sandbox-tools";
|
|
|
5444
5510
|
const emitPrompt = (payload) => {
|
|
5445
5511
|
const prompt = normalizePrompt(payload);
|
|
5446
5512
|
if (!prompt) return;
|
|
5447
|
-
this.hiddenPromptIds.
|
|
5513
|
+
if (this.hiddenPromptIds.has(prompt.id)) {
|
|
5514
|
+
this.emit("prompt", { ...prompt, status: "answered" });
|
|
5515
|
+
return;
|
|
5516
|
+
}
|
|
5448
5517
|
this.promptCache.set(prompt.id, prompt);
|
|
5449
5518
|
this.emit("prompt", prompt);
|
|
5450
5519
|
};
|
|
@@ -13055,6 +13124,7 @@ var LOCAL_CONTROL_REQUEST_RETRY_DELAY_MS = 500;
|
|
|
13055
13124
|
var SESSION_DATA_REQUEST_RETRY_COUNT = 4;
|
|
13056
13125
|
var SESSION_DATA_REQUEST_RETRY_DELAY_MS = 500;
|
|
13057
13126
|
var EFFECT_HOST_CONNECT_TIMEOUT_MS = 15e3;
|
|
13127
|
+
var SESSION_CONNECT_TIMEOUT_MS = 15e3;
|
|
13058
13128
|
function filenameFromUploadBody(body) {
|
|
13059
13129
|
const maybe = body;
|
|
13060
13130
|
return typeof maybe.name === "string" && maybe.name.trim() ? maybe.name.trim() : null;
|
|
@@ -13074,7 +13144,7 @@ function bodyInitFromSessionFileUpload(body) {
|
|
|
13074
13144
|
}
|
|
13075
13145
|
return body;
|
|
13076
13146
|
}
|
|
13077
|
-
var EFFECT_CATALOG_SYNC_TIMEOUT_MS =
|
|
13147
|
+
var EFFECT_CATALOG_SYNC_TIMEOUT_MS = 12e4;
|
|
13078
13148
|
var EFFECT_CATALOG_SYNC_RETRY_COUNT = 3;
|
|
13079
13149
|
var EFFECT_CATALOG_SYNC_RETRY_DELAY_MS = 1e3;
|
|
13080
13150
|
function planRecordObjectsChunks(records, batchSize) {
|
|
@@ -15030,7 +15100,14 @@ var Granular = class _Granular {
|
|
|
15030
15100
|
return tag;
|
|
15031
15101
|
}
|
|
15032
15102
|
buildManagedEnvironmentName(tag, versionId) {
|
|
15033
|
-
return `__sdk__${tag}__${versionId}`;
|
|
15103
|
+
return `__sdk__${tag}__${versionId}__pinned`;
|
|
15104
|
+
}
|
|
15105
|
+
isManagedEnvironmentName(environment, tagName) {
|
|
15106
|
+
const name = environment.environment || environment.envName || "";
|
|
15107
|
+
return name.startsWith(`__sdk__${tagName}__`);
|
|
15108
|
+
}
|
|
15109
|
+
isPinnedToVersion(environment, versionId) {
|
|
15110
|
+
return environment.buildPolicy.mode === "pinned" && (environment.versionId === versionId || environment.buildPolicy.versionId === versionId || environment.buildPolicy.buildId === versionId);
|
|
15034
15111
|
}
|
|
15035
15112
|
matchesTagTrackedEnvironment(environment, tagName, tagId) {
|
|
15036
15113
|
const environmentTagName = environment.tag?.name || environment.buildPolicy.tagName || null;
|
|
@@ -15091,7 +15168,7 @@ var Granular = class _Granular {
|
|
|
15091
15168
|
);
|
|
15092
15169
|
const currentMatches = this.sortEnvironmentsByRecency(
|
|
15093
15170
|
userEnvironments.filter(
|
|
15094
|
-
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId) && environment.versionId === targetVersionId
|
|
15171
|
+
(environment) => this.matchesTagTrackedEnvironment(environment, tagName, tag.tagId) && environment.versionId === targetVersionId && (!this.isManagedEnvironmentName(environment, tagName) || this.isPinnedToVersion(environment, targetVersionId))
|
|
15095
15172
|
)
|
|
15096
15173
|
);
|
|
15097
15174
|
if (currentMatches.length > 0) {
|
|
@@ -15120,6 +15197,7 @@ var Granular = class _Granular {
|
|
|
15120
15197
|
subjectId: user.granularId,
|
|
15121
15198
|
environment: this.buildManagedEnvironmentName(tagName, targetVersionId),
|
|
15122
15199
|
tagId: tag.tagId,
|
|
15200
|
+
versionId: targetVersionId,
|
|
15123
15201
|
permissionProfileId: null
|
|
15124
15202
|
}),
|
|
15125
15203
|
requestedOntology: ontology,
|
|
@@ -15165,6 +15243,7 @@ var Granular = class _Granular {
|
|
|
15165
15243
|
row.summaryUpdatedAt ?? row.summary_updated_at
|
|
15166
15244
|
) : null,
|
|
15167
15245
|
subjectId: row.subjectId != null ? String(row.subjectId) : null,
|
|
15246
|
+
sessionScope: row.sessionScope != null || row.session_scope != null ? String(row.sessionScope ?? row.session_scope) : null,
|
|
15168
15247
|
jobCount: typeof row.jobCount === "number" ? row.jobCount : void 0,
|
|
15169
15248
|
toolCallCount: typeof row.toolCallCount === "number" ? row.toolCallCount : void 0
|
|
15170
15249
|
};
|
|
@@ -15372,7 +15451,11 @@ var Granular = class _Granular {
|
|
|
15372
15451
|
onUnexpectedClose: this.onUnexpectedClose,
|
|
15373
15452
|
onReconnectError: this.onReconnectError
|
|
15374
15453
|
});
|
|
15375
|
-
await
|
|
15454
|
+
await withTimeout(
|
|
15455
|
+
client.connect(),
|
|
15456
|
+
SESSION_CONNECT_TIMEOUT_MS,
|
|
15457
|
+
`session WebSocket connect for ${session.sessionId}`
|
|
15458
|
+
);
|
|
15376
15459
|
const environmentSession = new EnvironmentSession(
|
|
15377
15460
|
client,
|
|
15378
15461
|
environment,
|
|
@@ -15543,12 +15626,24 @@ var Granular = class _Granular {
|
|
|
15543
15626
|
host.heartbeatInFlight = false;
|
|
15544
15627
|
}
|
|
15545
15628
|
async synchronizeEffectHost(host) {
|
|
15546
|
-
|
|
15547
|
-
|
|
15548
|
-
|
|
15549
|
-
|
|
15550
|
-
|
|
15551
|
-
|
|
15629
|
+
if (host.syncPromise) {
|
|
15630
|
+
return host.syncPromise;
|
|
15631
|
+
}
|
|
15632
|
+
host.syncPromise = (async () => {
|
|
15633
|
+
await host.wsClient.call("client.hello", {
|
|
15634
|
+
clientId: host.clientId,
|
|
15635
|
+
protocolVersion: "2.0"
|
|
15636
|
+
});
|
|
15637
|
+
await this.publishSandboxEffectCatalog(host);
|
|
15638
|
+
this.startEffectHostHeartbeat(host);
|
|
15639
|
+
})();
|
|
15640
|
+
try {
|
|
15641
|
+
await host.syncPromise;
|
|
15642
|
+
} finally {
|
|
15643
|
+
if (host.syncPromise) {
|
|
15644
|
+
host.syncPromise = null;
|
|
15645
|
+
}
|
|
15646
|
+
}
|
|
15552
15647
|
}
|
|
15553
15648
|
async ensureSandboxEffectHost(sandboxId) {
|
|
15554
15649
|
const existing = this.sandboxEffectHosts.get(sandboxId);
|
|
@@ -15584,7 +15679,8 @@ var Granular = class _Granular {
|
|
|
15584
15679
|
wsClient,
|
|
15585
15680
|
heartbeatTimer: null,
|
|
15586
15681
|
heartbeatInFlight: false,
|
|
15587
|
-
recovering: false
|
|
15682
|
+
recovering: false,
|
|
15683
|
+
syncPromise: null
|
|
15588
15684
|
};
|
|
15589
15685
|
wsClient.registerRpcHandler("effect.invoke", async (params) => {
|
|
15590
15686
|
const request = params;
|
|
@@ -16402,6 +16498,79 @@ function validateHarnessTemplateManifest(value, context = "HarnessTemplateManife
|
|
|
16402
16498
|
function defineHarnessTemplateManifest(value, context) {
|
|
16403
16499
|
return validateHarnessTemplateManifest(value, context);
|
|
16404
16500
|
}
|
|
16501
|
+
var DEFAULT_IGNORED_REASONING_COMMENT_DIRECTIVES = [
|
|
16502
|
+
/^@ts-ignore\b/i,
|
|
16503
|
+
/^@ts-expect-error\b/i,
|
|
16504
|
+
/^eslint-[\w-]+\b/i,
|
|
16505
|
+
/^biome-ignore\b/i,
|
|
16506
|
+
/^prettier-ignore\b/i,
|
|
16507
|
+
/^istanbul ignore\b/i
|
|
16508
|
+
];
|
|
16509
|
+
var DEFAULT_LOW_SIGNAL_REASONING_LINES = [
|
|
16510
|
+
/^running\.?$/i,
|
|
16511
|
+
/^working\.?$/i,
|
|
16512
|
+
/^thinking\.?$/i,
|
|
16513
|
+
/^generating(?: code)?\.?$/i,
|
|
16514
|
+
/^starting(?: execution)?\.?$/i
|
|
16515
|
+
];
|
|
16516
|
+
function parseReasoningCommentLine(line, options = {}) {
|
|
16517
|
+
const trimmed = line.trimStart();
|
|
16518
|
+
if (!trimmed.startsWith("//")) return null;
|
|
16519
|
+
const text = trimmed.replace(/^\/\/\s?/, "").trim();
|
|
16520
|
+
if (!text) return { kind: "ignored" };
|
|
16521
|
+
const ignoredDirectives = options.ignoredCommentDirectives || DEFAULT_IGNORED_REASONING_COMMENT_DIRECTIVES;
|
|
16522
|
+
if (ignoredDirectives.some((pattern) => pattern.test(text))) {
|
|
16523
|
+
return { kind: "ignored" };
|
|
16524
|
+
}
|
|
16525
|
+
const lowSignalLines = options.lowSignalReasoningLines || DEFAULT_LOW_SIGNAL_REASONING_LINES;
|
|
16526
|
+
if (lowSignalLines.some((pattern) => pattern.test(text))) {
|
|
16527
|
+
return { kind: "ignored" };
|
|
16528
|
+
}
|
|
16529
|
+
return { kind: "reasoning", text };
|
|
16530
|
+
}
|
|
16531
|
+
function consumeGranularReasoningTraceChunk(buffer, chunk, options = {}) {
|
|
16532
|
+
let text = buffer + chunk;
|
|
16533
|
+
let visibleText = "";
|
|
16534
|
+
const reasoningLines = [];
|
|
16535
|
+
while (true) {
|
|
16536
|
+
const newlineIndex = text.indexOf("\n");
|
|
16537
|
+
if (newlineIndex === -1) break;
|
|
16538
|
+
const rawLine = text.slice(0, newlineIndex);
|
|
16539
|
+
text = text.slice(newlineIndex + 1);
|
|
16540
|
+
const comment = parseReasoningCommentLine(
|
|
16541
|
+
rawLine.replace(/\r$/, ""),
|
|
16542
|
+
options
|
|
16543
|
+
);
|
|
16544
|
+
if (comment?.kind === "reasoning") {
|
|
16545
|
+
reasoningLines.push(comment.text);
|
|
16546
|
+
} else if (comment?.kind === "ignored") {
|
|
16547
|
+
continue;
|
|
16548
|
+
} else {
|
|
16549
|
+
visibleText += `${rawLine}
|
|
16550
|
+
`;
|
|
16551
|
+
}
|
|
16552
|
+
}
|
|
16553
|
+
if (options.final && text.length > 0) {
|
|
16554
|
+
const comment = parseReasoningCommentLine(text.replace(/\r$/, ""), options);
|
|
16555
|
+
if (comment?.kind === "reasoning") {
|
|
16556
|
+
reasoningLines.push(comment.text);
|
|
16557
|
+
text = "";
|
|
16558
|
+
} else if (comment?.kind === "ignored") {
|
|
16559
|
+
text = "";
|
|
16560
|
+
} else {
|
|
16561
|
+
visibleText += text;
|
|
16562
|
+
text = "";
|
|
16563
|
+
}
|
|
16564
|
+
}
|
|
16565
|
+
return { buffer: text, visibleText, reasoningLines };
|
|
16566
|
+
}
|
|
16567
|
+
function consumeGranularReasoningOnlyChunk(buffer, chunk, options = {}) {
|
|
16568
|
+
const result = consumeGranularReasoningTraceChunk(buffer, chunk, options);
|
|
16569
|
+
return {
|
|
16570
|
+
buffer: result.buffer,
|
|
16571
|
+
reasoningLines: result.reasoningLines
|
|
16572
|
+
};
|
|
16573
|
+
}
|
|
16405
16574
|
function asRecord4(value) {
|
|
16406
16575
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
16407
16576
|
return value;
|
|
@@ -17607,7 +17776,8 @@ function buildGranularAgentSessionBlock(sessionContext) {
|
|
|
17607
17776
|
runtimeId: sessionContext?.sandboxId || null,
|
|
17608
17777
|
environmentId: sessionContext?.environmentId || null,
|
|
17609
17778
|
userName: sessionContext?.userName || null,
|
|
17610
|
-
domainRevision: sessionContext?.domainRevision || null
|
|
17779
|
+
domainRevision: sessionContext?.domainRevision || null,
|
|
17780
|
+
uiContext: sessionContext?.uiContext || null
|
|
17611
17781
|
});
|
|
17612
17782
|
}
|
|
17613
17783
|
function buildGranularAgentHeapBlock(heapSummary) {
|
|
@@ -17854,7 +18024,7 @@ function buildGranularAgentToolBlock(tools, capabilityOverrides) {
|
|
|
17854
18024
|
const rightScope = `${right.className || "global"}:${right.static ? "static" : "instance"}`;
|
|
17855
18025
|
return leftScope.localeCompare(rightScope) || left.name.localeCompare(right.name);
|
|
17856
18026
|
});
|
|
17857
|
-
const
|
|
18027
|
+
const availableActions = normalizedTools.filter((tool) => tool.ready !== false).map((tool) => {
|
|
17858
18028
|
const scope = tool.className ? `${tool.static ? "class" : "record"}:${tool.className}` : "global";
|
|
17859
18029
|
return {
|
|
17860
18030
|
name: tool.name,
|
|
@@ -17865,7 +18035,8 @@ function buildGranularAgentToolBlock(tools, capabilityOverrides) {
|
|
|
17865
18035
|
const capabilities = {
|
|
17866
18036
|
executeCode: resolvedCapabilities.executeCode,
|
|
17867
18037
|
readEntities: resolvedCapabilities.readEntities,
|
|
17868
|
-
|
|
18038
|
+
availableActions,
|
|
18039
|
+
writeActions: availableActions,
|
|
17869
18040
|
workflowHelpers: resolvedCapabilities.workflowHelpers,
|
|
17870
18041
|
savedData: resolvedCapabilities.savedData,
|
|
17871
18042
|
showRecords: resolvedCapabilities.showRecords
|
|
@@ -17879,7 +18050,7 @@ function buildGranularAgentActionIndex(tools) {
|
|
|
17879
18050
|
return leftScope.localeCompare(rightScope) || left.name.localeCompare(right.name);
|
|
17880
18051
|
});
|
|
17881
18052
|
if (normalizedTools.length === 0) {
|
|
17882
|
-
return "No
|
|
18053
|
+
return "No executable actions are available.";
|
|
17883
18054
|
}
|
|
17884
18055
|
const globalTools = normalizedTools.filter((tool) => !tool.className);
|
|
17885
18056
|
const staticTools = normalizedTools.filter(
|
|
@@ -17973,6 +18144,76 @@ function splitDomainDocumentation(domainDocumentation) {
|
|
|
17973
18144
|
}
|
|
17974
18145
|
return { types: normalized, docs: "" };
|
|
17975
18146
|
}
|
|
18147
|
+
var DOMAIN_HELPER_FUNCTION_NAMES = /* @__PURE__ */ new Set([
|
|
18148
|
+
"agent_heap_objects",
|
|
18149
|
+
"agent_message",
|
|
18150
|
+
"agent_text_message"
|
|
18151
|
+
]);
|
|
18152
|
+
function inferGlobalActionToolsFromDomainTypes(domainTypes) {
|
|
18153
|
+
const inferred = [];
|
|
18154
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18155
|
+
const declarationPattern = /(?:export\s+)?declare\s+function\s+([A-Za-z_$][\w$]*)\s*\(/g;
|
|
18156
|
+
let match;
|
|
18157
|
+
while (match = declarationPattern.exec(domainTypes)) {
|
|
18158
|
+
const name = match[1];
|
|
18159
|
+
if (!name || DOMAIN_HELPER_FUNCTION_NAMES.has(name) || seen.has(name)) {
|
|
18160
|
+
continue;
|
|
18161
|
+
}
|
|
18162
|
+
seen.add(name);
|
|
18163
|
+
inferred.push({
|
|
18164
|
+
name,
|
|
18165
|
+
description: "Executable global action declared by the domain runtime."
|
|
18166
|
+
});
|
|
18167
|
+
}
|
|
18168
|
+
const actionLinePattern = /^-\s*([A-Za-z_$][\w$]*)\s+\(global\):\s*(.+)$/gm;
|
|
18169
|
+
while (match = actionLinePattern.exec(domainTypes)) {
|
|
18170
|
+
const name = match[1];
|
|
18171
|
+
if (!name || DOMAIN_HELPER_FUNCTION_NAMES.has(name) || seen.has(name)) {
|
|
18172
|
+
continue;
|
|
18173
|
+
}
|
|
18174
|
+
seen.add(name);
|
|
18175
|
+
inferred.push({
|
|
18176
|
+
name,
|
|
18177
|
+
description: match[2]?.trim() || "Executable global action declared by the domain runtime."
|
|
18178
|
+
});
|
|
18179
|
+
}
|
|
18180
|
+
const scopedActionLinePattern = /^-\s*([A-Za-z_$][\w$]*)\.([A-Za-z_$][\w$]*)\s+\((record|class)\):\s*(.+)$/gm;
|
|
18181
|
+
while (match = scopedActionLinePattern.exec(domainTypes)) {
|
|
18182
|
+
const className = match[1]?.toLowerCase();
|
|
18183
|
+
const name = match[2];
|
|
18184
|
+
const scope = match[3];
|
|
18185
|
+
if (!className || !name || DOMAIN_HELPER_FUNCTION_NAMES.has(name)) {
|
|
18186
|
+
continue;
|
|
18187
|
+
}
|
|
18188
|
+
const key = `${className}:${scope}:${name}`;
|
|
18189
|
+
if (seen.has(key)) {
|
|
18190
|
+
continue;
|
|
18191
|
+
}
|
|
18192
|
+
seen.add(key);
|
|
18193
|
+
inferred.push({
|
|
18194
|
+
name,
|
|
18195
|
+
className,
|
|
18196
|
+
static: scope === "class",
|
|
18197
|
+
description: match[4]?.trim() || "Executable action declared by the domain runtime."
|
|
18198
|
+
});
|
|
18199
|
+
}
|
|
18200
|
+
return inferred;
|
|
18201
|
+
}
|
|
18202
|
+
function resolvePromptTools(tools, domainTypes) {
|
|
18203
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
18204
|
+
for (const tool of tools || []) {
|
|
18205
|
+
if (!tool?.name) continue;
|
|
18206
|
+
const key = `${tool.className || "global"}:${tool.static ? "static" : "instance"}:${tool.name}`;
|
|
18207
|
+
byKey.set(key, tool);
|
|
18208
|
+
}
|
|
18209
|
+
for (const tool of inferGlobalActionToolsFromDomainTypes(domainTypes)) {
|
|
18210
|
+
const key = `global:instance:${tool.name}`;
|
|
18211
|
+
if (!byKey.has(key)) {
|
|
18212
|
+
byKey.set(key, tool);
|
|
18213
|
+
}
|
|
18214
|
+
}
|
|
18215
|
+
return [...byKey.values()];
|
|
18216
|
+
}
|
|
17976
18217
|
function buildGranularAgentCheckpointBlock(checkpoint) {
|
|
17977
18218
|
if (!checkpoint) {
|
|
17978
18219
|
return renderConstBlock("previousCodeResult", null);
|
|
@@ -18045,12 +18286,13 @@ function buildGranularAgentSystemPrompt(input) {
|
|
|
18045
18286
|
const outputMode = input.outputMode || "agentMessages";
|
|
18046
18287
|
const promptCapabilities = resolvePromptCapabilities(input.capabilities);
|
|
18047
18288
|
const domainSections = splitDomainDocumentation(input.domainDocumentation);
|
|
18289
|
+
const promptTools = resolvePromptTools(input.tools, domainSections.types);
|
|
18048
18290
|
const sessionBlock = buildGranularAgentSessionBlock(input.sessionContext);
|
|
18049
18291
|
const toolBlock = buildGranularAgentToolBlock(
|
|
18050
|
-
|
|
18292
|
+
promptTools,
|
|
18051
18293
|
input.capabilities
|
|
18052
18294
|
);
|
|
18053
|
-
const actionIndex = buildGranularAgentActionIndex(
|
|
18295
|
+
const actionIndex = buildGranularAgentActionIndex(promptTools);
|
|
18054
18296
|
const domainBlock = buildGranularAgentDomainBlock(domainSections.types);
|
|
18055
18297
|
const workflowBlock = buildGranularAgentWorkflowBlock(input.workflowSummary);
|
|
18056
18298
|
const checkpointBlock = buildGranularAgentCheckpointBlock(input.checkpoint);
|
|
@@ -18078,7 +18320,7 @@ function buildGranularAgentSystemPrompt(input) {
|
|
|
18078
18320
|
- When \`agent_text_message(...)\` mentions a grounded record that should remain clickable/referable, wrap only the visible record label in a self-closing inline reference tag: \`<granular-object class="class_name" id="stable_id_or_path" label="Visible label" />\`. Use the actual class name and stable id/path from the runtime record or effect result; do not invent ids, field names, or snake/camel-case aliases that are not present in the type declarations or returned object.
|
|
18079
18321
|
- Treat \`agent_heap_objects(...)\` as the UI display call for user-visible records, not as a general storage helper. Do not wrap records under an \`items\` key.
|
|
18080
18322
|
- When records should remain reusable for follow-ups, first save the runtime record or ordered record array with \`await heap.setVar("stable_selection_name", value)\`, then display that saved selection exactly once with \`await agent_heap_objects({ variableNames: ["stable_selection_name"] })\`.
|
|
18081
|
-
- \`heap.setVar(...)\` only accepts scalar values, runtime records/sandbox instances, or arrays of runtime records/sandbox instances. Do not save plain action/effect result objects. If an action returns
|
|
18323
|
+
- \`heap.setVar(...)\` only accepts scalar values, runtime records/sandbox instances, or arrays of runtime records/sandbox instances from one class. Do not save plain action/effect result objects or arrays of JSON summaries returned by actions. If an action returns ids/paths for records that should remain referable or displayed as records, fetch the matching runtime records first with the generated class \`.get(...)\`/query API, then save/display those fetched records. If the action returned only structured summaries, answer from those summaries with \`agent_text_message(...)\`.
|
|
18082
18324
|
- Do not use \`agent_heap_objects({ entries: [...] })\` or \`agent_heap_objects({ saveAs, entries })\` as a shortcut for ordered pages, queues, search results, or ranked lists; those forms can create duplicate or poorly labelled displays. Save the selection with \`heap.setVar(...)\` and display it via \`variableNames\` instead.
|
|
18083
18325
|
- Use \`entryPaths\` only for a few already-known individual records and \`listNames\` only for a host-created list that you intentionally want to show. Do not display both an entry/list selection and a heap variable for the same records.
|
|
18084
18326
|
- When the user asks to show, list, display, open, or "show them" for records you found, call \`agent_heap_objects(...)\`; do not answer only with a count or text summary.
|
|
@@ -18128,10 +18370,11 @@ ${outputRules}` : `Code:
|
|
|
18128
18370
|
- Use choice only for 2 to 5 short grounded options.
|
|
18129
18371
|
- For record choices, set each option value to a stable scalar such as the record \`_graphPath\` or \`id\`, not a label-only value.
|
|
18130
18372
|
- After \`await loop.ask_user(...)\` returns from a choice prompt, tolerate either the option value, the option object, or a human-readable label by matching against value, id/path, label, and description before failing. If a returned label is a prefix or substring of exactly one option label, treat it as that option.
|
|
18131
|
-
- Use \`loop.confirm(...)\` for yes/no confirmation only when the user explicitly asks for
|
|
18373
|
+
- Use \`loop.confirm(...)\` for yes/no confirmation only when the user explicitly asks for a separate confirmation step, policy requires confirmation outside the action runtime, or material uncertainty remains after grounding.
|
|
18374
|
+
- If action, effect, tool, or permission metadata already marks the invoked action as confirmation-gated, do not call \`loop.confirm(...)\` before invoking it. Ground the target and input, then call the action once; the runtime action policy will surface the confirmation prompt and resume the same invocation after approval.
|
|
18132
18375
|
- Do not add a generic yes/no confirmation after the user has already made a grounded choice, unless one of those confirmation conditions still applies.
|
|
18133
|
-
- Do not add confirmation only because an allowed mutation is visible to other people, customer-facing, or consequential. If the user clearly requested the mutation and the grounded target, action, and condition are unique, perform the mutation unless confirmation is required
|
|
18134
|
-
- A conditional request such as "if this is true, do that" is authorization to perform the requested action after you verify the condition. Once the condition, target, and action are grounded uniquely, call the action directly; do not ask "should I perform/post/send this?" unless the user, policy
|
|
18376
|
+
- Do not add confirmation only because an allowed mutation is visible to other people, customer-facing, or consequential. If the user clearly requested the mutation and the grounded target, action, and condition are unique, perform the mutation unless confirmation is required outside the action runtime or remaining material uncertainty exists.
|
|
18377
|
+
- A conditional request such as "if this is true, do that" is authorization to perform the requested action after you verify the condition. Once the condition, target, and action are grounded uniquely, call the action directly; do not ask "should I perform/post/send this?" unless the user, policy outside the action runtime, or unresolved material uncertainty requires confirmation. The visibility or impact of an allowed action is not by itself unresolved uncertainty.
|
|
18135
18378
|
- If the user explicitly asks you to stop for confirmation, natural-language text such as "please confirm" is not enough: call \`await loop.confirm(...)\` before the mutation, then perform the approved mutation in the same resumed job when it returns true.
|
|
18136
18379
|
- Reuse existing task, decision, and closure ids from [State].
|
|
18137
18380
|
- If a user request matches both a domain record/action and a workflow helper, prefer the domain capability.` : "";
|
|
@@ -18277,7 +18520,8 @@ Query policy:
|
|
|
18277
18520
|
- For operational blocker, risk, status, or "what is happening" questions, inspect the relevant record's scalar fields such as status, priority, blocker, summary, latest update/message, due date, amount, and other domain-specific descriptive fields before answering.
|
|
18278
18521
|
- For read-only readiness, risk, health, or status summaries, call any visible read-only assessment/status action on the grounded primary record before ad-hoc aggregation when such an action semantically matches the request. Use the returned fields in the reply and supplement with counts or record reads only when useful.
|
|
18279
18522
|
- Do not hide required visible read-only assessment/status actions inside broad try/catch blocks. The runtime action surface should show that the assessment action ran.
|
|
18280
|
-
- Treat action/effect results as structured values, not necessarily arrays. Before indexing, iterating, checking \`.length\`, or calling array methods, normalize the result first: use the result itself only when \`Array.isArray(result)\`; otherwise read the exact array field shown in the output schema, or a documented array field such as \`items\`, \`matches\`, \`results\`, \`records\`, \`entries\`, \`candidates\`, \`options\`, or
|
|
18523
|
+
- Treat action/effect results as structured values, not necessarily arrays. Before indexing, iterating, checking \`.length\`, or calling array methods, normalize the result first: use the result itself only when \`Array.isArray(result)\`; otherwise read the exact array field shown in the output schema, or a documented array field such as \`items\`, \`matches\`, \`results\`, \`records\`, \`entries\`, \`candidates\`, \`options\`, \`requests\`, \`vendors\`, \`transactions\`, \`approvals\`, \`receipts\`, or another domain-specific array field. If a structured result has \`count > 0\`, never conclude there are no matches until you inspect every array-valued field on that result object, especially fields named by the output schema. Never convert a non-array object result to \`[]\` before checking its documented fields.
|
|
18524
|
+
- Plain JSON objects returned by actions are not sandbox record instances, even when they contain ids, titles, labels, or status fields. Use them for reasoning and text responses. Do not pass action-returned JSON objects or arrays directly to \`heap.setVar(...)\` or \`agent_heap_objects(...)\`; fetch corresponding runtime records first when the user needs record display or follow-up references.
|
|
18281
18525
|
- When a visible search, lookup, availability, or assessment action returns candidates or matches, treat those returned records as already scoped by the action inputs unless the output schema gives reliable fields for further narrowing. When matching returned candidates to grounded records, use the output schema's actual identifier fields, including \`id\`, \`path\`, or fields ending in \`Id\`; do not assume candidates have \`_graphPath\`. Do not discard all returned candidates by re-filtering on guessed property names.
|
|
18282
18526
|
- For scheduling actions, convert relative wording into concrete ISO timestamps before mutating records.
|
|
18283
18527
|
- When a decision depends on fresh external state and a visible read-only status/lookup action exists on the grounded record, call it before deciding, mutating, or refusing based on stale stored fields.
|
|
@@ -18339,6 +18583,7 @@ ${domainSections.docs}
|
|
|
18339
18583
|
|
|
18340
18584
|
Actions:
|
|
18341
18585
|
${actionIndex}
|
|
18586
|
+
- Global actions are executable functions exported by "./sandbox-tools"; import each global action you call, e.g. \`import { some_action } from "./sandbox-tools"; await some_action(...)\`. This includes frontend actions such as opening, focusing, or navigating the host UI.
|
|
18342
18587
|
- Actions listed under "Record-level" are instance methods. First fetch or find the specific record, then call the action on that instance, e.g. \`const item = await Item.get({ path }); await item.action_name(...)\`.
|
|
18343
18588
|
- Actions listed under "Class-level" are class/static methods. Call them on the imported class, e.g. \`await Item.action_name(...)\`.
|
|
18344
18589
|
- The action index is the visibility contract. If an action is listed for a class, call it directly on fetched/listed instances of that class; do not use \`typeof record.action_name === "function"\` as a discovery gate. If an action is not listed, do not call it.
|
|
@@ -19114,7 +19359,7 @@ function extractJsonStringField(source, fieldName) {
|
|
|
19114
19359
|
function modelOutputInstruction() {
|
|
19115
19360
|
return [
|
|
19116
19361
|
"Return only a JSON object with this shape:",
|
|
19117
|
-
'{ "action": "reply" | "job", "
|
|
19362
|
+
'{ "action": "reply" | "job", "code": string, "reply": string }',
|
|
19118
19363
|
'Use "action":"reply" only when a plain conversational answer is enough and no live session state should change.',
|
|
19119
19364
|
'Do not use "action":"reply" to promise future tool work; if the user asks to check, find, look up, inspect, read, reopen, summarize, transform, update, post, send, approve, schedule, reschedule, calculate, or confirm around session state, session files, generated files, tools, or a domain action, use "action":"job".',
|
|
19120
19365
|
'If the user asks to use an attached file, uploaded file, generated file, previous output file, or "the summary/workbook/file you just created", choose "action":"job" and read it through [Runtime Imports] instead of answering from memory.',
|
|
@@ -19122,7 +19367,8 @@ function modelOutputInstruction() {
|
|
|
19122
19367
|
'Before claiming you lack access, inspect the visible action list. If a visible read-only search, lookup, list, guidance, note, policy, or knowledge action can satisfy a "check", "find", "look up", or "whether we have guidance" request, choose "action":"job" and call it.',
|
|
19123
19368
|
"Generated code must not report no matches for the primary human-described anchor after a single zero-result list/find/page call. Before that primary no-match return, retry the primary anchor with fewer text constraints or a distinct fallback such as owner/container grounding, relationship traversal, exact-id/path lookup, or shorter target-local search.",
|
|
19124
19369
|
'Use "action":"job" when the next step should run code or mutate workflow state.',
|
|
19125
|
-
'When action is "job", include runnable code in "code".',
|
|
19370
|
+
'When action is "job", include runnable code in "code" and emit the "code" field before any non-empty "reply" field so generated code comments can stream as progress.',
|
|
19371
|
+
"Generated job code must use plain ASCII punctuation in string literals and comments. Do not use curly quotes, smart apostrophes, en dashes, em dashes, or other typographic punctuation in code.",
|
|
19126
19372
|
"Generated code must not reference prompt-only symbols such as runtimeImports, savedData, sessionFileManifest, recentReferences, workflowContext, workflowState, or capabilities. Copy concrete paths/ids from the prompt into strings, fetch records with documented imports, or use documented runtime globals.",
|
|
19127
19373
|
"Generated code must follow [Runtime Imports]: import module exports from their listed module, use listed globals directly without importing them, and do not leave undeclared identifiers in the job.",
|
|
19128
19374
|
"Generated action calls must use the exact input property names from the visible action schema. Do not invent synonym keys for required inputs.",
|
|
@@ -19207,13 +19453,36 @@ ${modelOutputInstruction()}`
|
|
|
19207
19453
|
let text = "";
|
|
19208
19454
|
let usage = null;
|
|
19209
19455
|
let requestId = null;
|
|
19210
|
-
if (input.onTextDelta) {
|
|
19211
|
-
const onTextDelta = input.onTextDelta;
|
|
19456
|
+
if (input.onTextDelta || input.onReplyDelta || input.onCodeDelta) {
|
|
19212
19457
|
const stream = await client.chat.completions.create({
|
|
19213
19458
|
...payload,
|
|
19214
19459
|
stream: true,
|
|
19215
19460
|
stream_options: { include_usage: true }
|
|
19216
19461
|
});
|
|
19462
|
+
let streamedReply = "";
|
|
19463
|
+
let streamedCode = "";
|
|
19464
|
+
const emitReplyDelta = async () => {
|
|
19465
|
+
if (!input.onReplyDelta) return;
|
|
19466
|
+
const replyField = extractJsonStringField(text, "reply");
|
|
19467
|
+
if (!replyField) return;
|
|
19468
|
+
const nextReply = replyField.value;
|
|
19469
|
+
if (!nextReply.startsWith(streamedReply)) return;
|
|
19470
|
+
const delta = nextReply.slice(streamedReply.length);
|
|
19471
|
+
if (!delta) return;
|
|
19472
|
+
streamedReply = nextReply;
|
|
19473
|
+
await input.onReplyDelta(delta);
|
|
19474
|
+
};
|
|
19475
|
+
const emitCodeDelta = async () => {
|
|
19476
|
+
if (!input.onCodeDelta) return;
|
|
19477
|
+
const codeField = extractJsonStringField(text, "code");
|
|
19478
|
+
if (!codeField) return;
|
|
19479
|
+
const nextCode = codeField.value;
|
|
19480
|
+
if (!nextCode.startsWith(streamedCode)) return;
|
|
19481
|
+
const delta = nextCode.slice(streamedCode.length);
|
|
19482
|
+
if (!delta) return;
|
|
19483
|
+
streamedCode = nextCode;
|
|
19484
|
+
await input.onCodeDelta(delta);
|
|
19485
|
+
};
|
|
19217
19486
|
for await (const event of stream) {
|
|
19218
19487
|
requestId = requestId || event.id || event._request_id || null;
|
|
19219
19488
|
usage = event.usage || usage;
|
|
@@ -19221,8 +19490,12 @@ ${modelOutputInstruction()}`
|
|
|
19221
19490
|
const deltaText = typeof delta === "string" ? delta : Array.isArray(delta) ? delta.map((part) => asRecord6(part)?.text || "").join("") : "";
|
|
19222
19491
|
if (!deltaText) continue;
|
|
19223
19492
|
text += deltaText;
|
|
19224
|
-
await onTextDelta(deltaText);
|
|
19493
|
+
await input.onTextDelta?.(deltaText);
|
|
19494
|
+
await emitReplyDelta();
|
|
19495
|
+
await emitCodeDelta();
|
|
19225
19496
|
}
|
|
19497
|
+
await emitReplyDelta();
|
|
19498
|
+
await emitCodeDelta();
|
|
19226
19499
|
raw = { streamed: true, model, usage, request_id: requestId };
|
|
19227
19500
|
} else {
|
|
19228
19501
|
const completion = await client.chat.completions.create(
|
|
@@ -19324,6 +19597,64 @@ function normalizeHeapSnapshot2(heap) {
|
|
|
19324
19597
|
updatedAt: typeof heap?.updatedAt === "number" ? heap.updatedAt : Date.now()
|
|
19325
19598
|
};
|
|
19326
19599
|
}
|
|
19600
|
+
function targetFromUiContext(context) {
|
|
19601
|
+
const target = asRecord6(context?.target) || asRecord6(context?.currentPageObject) || asRecord6(context?.commentaryTarget);
|
|
19602
|
+
const className = typeof target?.className === "string" ? target.className : "";
|
|
19603
|
+
const id = typeof target?.id === "string" ? target.id : "";
|
|
19604
|
+
if (!className || !id) return null;
|
|
19605
|
+
return {
|
|
19606
|
+
className,
|
|
19607
|
+
id,
|
|
19608
|
+
label: typeof target?.label === "string" ? target.label : void 0
|
|
19609
|
+
};
|
|
19610
|
+
}
|
|
19611
|
+
function heapEntryMatchesTarget(entry, target) {
|
|
19612
|
+
if (entry.className !== target.className) return false;
|
|
19613
|
+
if (entry.id === target.id) return true;
|
|
19614
|
+
const fields = asRecord6(entry.fields);
|
|
19615
|
+
return fields?.real_id === target.id || fields?._realId === target.id;
|
|
19616
|
+
}
|
|
19617
|
+
function focusedHeapEntryPathsFromUiContext(heap, context) {
|
|
19618
|
+
const target = targetFromUiContext(context);
|
|
19619
|
+
if (!target) return [];
|
|
19620
|
+
return Object.entries(heap.entriesByPath).filter(([, entry]) => heapEntryMatchesTarget(entry, target)).map(([path2]) => path2);
|
|
19621
|
+
}
|
|
19622
|
+
function hasSessionDocumentContext(document) {
|
|
19623
|
+
const doc = asRecord6(document);
|
|
19624
|
+
if (!doc) return false;
|
|
19625
|
+
const heap = normalizeHeapSnapshot2(asRecord6(doc.heap));
|
|
19626
|
+
if (Object.keys(heap.entriesByPath).length > 0 || Object.keys(heap.listsByName).length > 0 || Object.keys(heap.variablesByName).length > 0) {
|
|
19627
|
+
return true;
|
|
19628
|
+
}
|
|
19629
|
+
const domain = asRecord6(doc.domain);
|
|
19630
|
+
const packages = asRecord6(domain?.packages);
|
|
19631
|
+
if (packages && Object.keys(packages).length > 0) return true;
|
|
19632
|
+
const prompts = asRecord6(doc.prompts);
|
|
19633
|
+
if (prompts && Object.keys(prompts).length > 0) return true;
|
|
19634
|
+
const workflows = asRecord6(doc.workflows);
|
|
19635
|
+
if (workflows && Object.keys(workflows).length > 0) return true;
|
|
19636
|
+
return false;
|
|
19637
|
+
}
|
|
19638
|
+
async function waitForSessionDocumentContext(environment, timeoutMs = 3e3) {
|
|
19639
|
+
if (hasSessionDocumentContext(environment.document)) return;
|
|
19640
|
+
await new Promise((resolve) => {
|
|
19641
|
+
let settled = false;
|
|
19642
|
+
let unsubscribe = null;
|
|
19643
|
+
const settle = () => {
|
|
19644
|
+
if (settled) return;
|
|
19645
|
+
settled = true;
|
|
19646
|
+
if (unsubscribe) unsubscribe();
|
|
19647
|
+
clearTimeout(timer);
|
|
19648
|
+
resolve();
|
|
19649
|
+
};
|
|
19650
|
+
const timer = setTimeout(settle, timeoutMs);
|
|
19651
|
+
unsubscribe = environment.on("sync", (document) => {
|
|
19652
|
+
if (hasSessionDocumentContext(document)) {
|
|
19653
|
+
settle();
|
|
19654
|
+
}
|
|
19655
|
+
});
|
|
19656
|
+
});
|
|
19657
|
+
}
|
|
19327
19658
|
function buildContinuationPreview(checkpoint, noProgressCount) {
|
|
19328
19659
|
const lines = [
|
|
19329
19660
|
`Controller no-progress count: ${noProgressCount}`,
|
|
@@ -19345,7 +19676,7 @@ function readableAgentMessage(message) {
|
|
|
19345
19676
|
const show = asRecord6(record.show);
|
|
19346
19677
|
const variableNames = asArray3(show?.variableNames).map((value) => String(value)).filter(Boolean);
|
|
19347
19678
|
if (variableNames.length) {
|
|
19348
|
-
return
|
|
19679
|
+
return variableNames.length === 1 ? "Displayed the selected record" : "Displayed the selected records";
|
|
19349
19680
|
}
|
|
19350
19681
|
if (typeof record.kind === "string") {
|
|
19351
19682
|
return `Agent ${record.kind} message`;
|
|
@@ -19531,6 +19862,202 @@ async function writeJson(filePath, value) {
|
|
|
19531
19862
|
await writeFile(filePath, `${JSON.stringify(value, null, 2)}
|
|
19532
19863
|
`);
|
|
19533
19864
|
}
|
|
19865
|
+
function safePathSegment(value, fallback) {
|
|
19866
|
+
const normalized = (value || "").trim().toLowerCase();
|
|
19867
|
+
const sanitized = normalized.replace(/[^a-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^-+|-+$/g, "").slice(0, 128);
|
|
19868
|
+
return sanitized || fallback;
|
|
19869
|
+
}
|
|
19870
|
+
function extractToolCalls(rawGeneration) {
|
|
19871
|
+
const raw = asRecord6(rawGeneration);
|
|
19872
|
+
if (!raw) return null;
|
|
19873
|
+
const choices = asArray3(raw.choices);
|
|
19874
|
+
const firstChoiceMessage = asRecord6(choices[0]);
|
|
19875
|
+
const message = asRecord6(firstChoiceMessage?.message);
|
|
19876
|
+
const toolCalls = asArray3(message?.tool_calls);
|
|
19877
|
+
if (toolCalls.length > 0) return toolCalls;
|
|
19878
|
+
return asArray3(raw.tool_calls).length > 0 ? asArray3(raw.tool_calls) : null;
|
|
19879
|
+
}
|
|
19880
|
+
function buildTurnMdxReport(input) {
|
|
19881
|
+
const {
|
|
19882
|
+
status,
|
|
19883
|
+
conversation,
|
|
19884
|
+
request,
|
|
19885
|
+
requestTimestamp,
|
|
19886
|
+
turnLog,
|
|
19887
|
+
responseText,
|
|
19888
|
+
terminalKind,
|
|
19889
|
+
actionSummary = [],
|
|
19890
|
+
promptInteractions = [],
|
|
19891
|
+
result,
|
|
19892
|
+
prompts = [],
|
|
19893
|
+
verification,
|
|
19894
|
+
error
|
|
19895
|
+
} = input;
|
|
19896
|
+
const iterationLines = [];
|
|
19897
|
+
for (const iteration of turnLog.iterations) {
|
|
19898
|
+
iterationLines.push(`### Iteration ${iteration.iteration}`);
|
|
19899
|
+
iterationLines.push(
|
|
19900
|
+
`- Request: ${iteration.request}`,
|
|
19901
|
+
`- Generation duration: ${iteration.generationDurationMs !== void 0 ? `${iteration.generationDurationMs}ms` : "unknown"}`
|
|
19902
|
+
);
|
|
19903
|
+
if (iteration.templateId) {
|
|
19904
|
+
iterationLines.push(
|
|
19905
|
+
`- Template: ${iteration.templateId}@${iteration.templateVersion || "unknown"}`
|
|
19906
|
+
);
|
|
19907
|
+
}
|
|
19908
|
+
if (iteration.templateHash) {
|
|
19909
|
+
iterationLines.push(`- Template hash: ${iteration.templateHash}`);
|
|
19910
|
+
}
|
|
19911
|
+
if (iteration.promptInstanceHash) {
|
|
19912
|
+
iterationLines.push(`- Prompt hash: ${iteration.promptInstanceHash}`);
|
|
19913
|
+
}
|
|
19914
|
+
iterationLines.push("", "#### Full prompt sent to LLM", "");
|
|
19915
|
+
iterationLines.push(fenced(iteration.systemPrompt, "text"));
|
|
19916
|
+
if (iteration.generationReply?.trim()) {
|
|
19917
|
+
iterationLines.push(
|
|
19918
|
+
"",
|
|
19919
|
+
"#### LLM reply text",
|
|
19920
|
+
iteration.generationReply.trim(),
|
|
19921
|
+
""
|
|
19922
|
+
);
|
|
19923
|
+
}
|
|
19924
|
+
if (iteration.generatedCode?.trim()) {
|
|
19925
|
+
iterationLines.push("#### Generated code", "", fenced(iteration.generatedCode.trim(), "ts"));
|
|
19926
|
+
}
|
|
19927
|
+
const toolCalls = extractToolCalls(iteration.rawGeneration);
|
|
19928
|
+
iterationLines.push("#### Tool calls / raw generation", "");
|
|
19929
|
+
if (toolCalls) {
|
|
19930
|
+
iterationLines.push(fenced(JSON.stringify(toolCalls, null, 2), "json"));
|
|
19931
|
+
} else if (iteration.rawGeneration) {
|
|
19932
|
+
iterationLines.push(fenced(JSON.stringify(iteration.rawGeneration, null, 2), "json"));
|
|
19933
|
+
} else {
|
|
19934
|
+
iterationLines.push("_No tool call information._");
|
|
19935
|
+
}
|
|
19936
|
+
if (iteration.tokenUsage) {
|
|
19937
|
+
iterationLines.push("", "#### Token usage", ...formatTokenUsage(iteration.tokenUsage));
|
|
19938
|
+
}
|
|
19939
|
+
if (iteration.responseText?.trim()) {
|
|
19940
|
+
iterationLines.push("", "#### Runtime/prompt outcome", iteration.responseText);
|
|
19941
|
+
}
|
|
19942
|
+
if (iteration.actionSummary?.length) {
|
|
19943
|
+
iterationLines.push("", "#### Action summary", "");
|
|
19944
|
+
iterationLines.push(...iteration.actionSummary.map((line) => `- ${line}`));
|
|
19945
|
+
}
|
|
19946
|
+
if (iteration.continuation) {
|
|
19947
|
+
iterationLines.push("", "#### Continuation", "", jsonBlock(iteration.continuation));
|
|
19948
|
+
}
|
|
19949
|
+
if (iteration.result !== void 0) {
|
|
19950
|
+
iterationLines.push("", "#### Result", "", jsonBlock(iteration.result));
|
|
19951
|
+
}
|
|
19952
|
+
if (iteration.error) {
|
|
19953
|
+
iterationLines.push("", `#### Error`, "", iteration.error);
|
|
19954
|
+
}
|
|
19955
|
+
iterationLines.push("", "---", "");
|
|
19956
|
+
}
|
|
19957
|
+
const promptEventLines = conversation.promptEvents.filter((event) => event.receivedAt >= requestTimestamp).map((event, index) => {
|
|
19958
|
+
const prompt = event.prompt;
|
|
19959
|
+
return `${index + 1}. ${prompt.type} ${prompt.title || ""} ${prompt.message ? `\u2014 ${prompt.message}` : ""}`;
|
|
19960
|
+
});
|
|
19961
|
+
const historyLines = conversation.history.map((entry, index) => {
|
|
19962
|
+
const label = `${index + 1}. ${entry.role}`;
|
|
19963
|
+
const detail = entry.content || (entry.code ? "(code)" : "");
|
|
19964
|
+
return `${label}: ${detail || "(no text)"}`;
|
|
19965
|
+
});
|
|
19966
|
+
const lines = [
|
|
19967
|
+
"# Local Agent Query Report",
|
|
19968
|
+
"",
|
|
19969
|
+
`- timestamp: ${new Date(requestTimestamp).toISOString()}`,
|
|
19970
|
+
`- turn: ${turnLog.turnNumber} (${turnLog.turnId})`,
|
|
19971
|
+
`- status: ${status}`,
|
|
19972
|
+
"",
|
|
19973
|
+
"## Table of contents",
|
|
19974
|
+
"- [Metadata](#metadata)",
|
|
19975
|
+
"- [User query and context](#user-query-and-context)",
|
|
19976
|
+
"- [Conversation history](#conversation-history)",
|
|
19977
|
+
"- [Harness loop iterations](#harness-loop-iterations)",
|
|
19978
|
+
"- [Result and interactions](#result-and-interactions)",
|
|
19979
|
+
"",
|
|
19980
|
+
"## Metadata",
|
|
19981
|
+
`- Ontology: ${conversation.environment.ontologyId}`,
|
|
19982
|
+
`- Subject: ${conversation.environment.subjectId}`,
|
|
19983
|
+
`- Session: ${conversation.environment.sessionId}`,
|
|
19984
|
+
`- Environment: ${conversation.environment.environmentId}`,
|
|
19985
|
+
`- Sandbox: ${conversation.environment.sandboxId}`,
|
|
19986
|
+
`- Permission profile: ${conversation.environment.permissionProfileId}`,
|
|
19987
|
+
`- Conversation artifacts: ${turnLog.turnDir}`,
|
|
19988
|
+
"",
|
|
19989
|
+
"## User query and context",
|
|
19990
|
+
"",
|
|
19991
|
+
"### Request",
|
|
19992
|
+
fenced(request, "text"),
|
|
19993
|
+
"",
|
|
19994
|
+
"### Open prompts at/after request time"
|
|
19995
|
+
];
|
|
19996
|
+
if (promptEventLines.length) {
|
|
19997
|
+
lines.push(...promptEventLines.map((line) => `- ${line}`));
|
|
19998
|
+
} else {
|
|
19999
|
+
lines.push("- _None");
|
|
20000
|
+
}
|
|
20001
|
+
lines.push(
|
|
20002
|
+
"",
|
|
20003
|
+
"## Conversation history",
|
|
20004
|
+
...historyLines.map((line) => `- ${line}`),
|
|
20005
|
+
"",
|
|
20006
|
+
"## Harness loop iterations",
|
|
20007
|
+
"",
|
|
20008
|
+
...iterationLines,
|
|
20009
|
+
"## Result and interactions",
|
|
20010
|
+
""
|
|
20011
|
+
);
|
|
20012
|
+
if (responseText) {
|
|
20013
|
+
lines.push("### Final response", responseText, "");
|
|
20014
|
+
}
|
|
20015
|
+
if (terminalKind) {
|
|
20016
|
+
lines.push(`### Terminal kind`, terminalKind, "");
|
|
20017
|
+
}
|
|
20018
|
+
lines.push("### Action summary");
|
|
20019
|
+
if (actionSummary.length) {
|
|
20020
|
+
lines.push(...actionSummary.map((line) => `- ${line}`));
|
|
20021
|
+
} else {
|
|
20022
|
+
lines.push("- None");
|
|
20023
|
+
}
|
|
20024
|
+
lines.push("", "### User interactions");
|
|
20025
|
+
if (promptInteractions.length) {
|
|
20026
|
+
for (const interaction of promptInteractions) {
|
|
20027
|
+
lines.push(
|
|
20028
|
+
`- ${interaction.type} ${interaction.message || interaction.title} -> ${JSON.stringify(interaction.answer)}`
|
|
20029
|
+
);
|
|
20030
|
+
}
|
|
20031
|
+
} else {
|
|
20032
|
+
lines.push("- None");
|
|
20033
|
+
}
|
|
20034
|
+
lines.push("", "### Pending prompts");
|
|
20035
|
+
if (prompts.length) {
|
|
20036
|
+
for (const prompt of prompts) {
|
|
20037
|
+
lines.push(`- ${prompt.type} ${prompt.title || ""} ${prompt.message || ""}`);
|
|
20038
|
+
}
|
|
20039
|
+
} else {
|
|
20040
|
+
lines.push("- None");
|
|
20041
|
+
}
|
|
20042
|
+
if (result !== void 0) {
|
|
20043
|
+
lines.push("", "### Result payload", "", jsonBlock(result));
|
|
20044
|
+
}
|
|
20045
|
+
if (verification !== void 0) {
|
|
20046
|
+
lines.push("", "### Verification", "", jsonBlock(verification));
|
|
20047
|
+
}
|
|
20048
|
+
if (error) {
|
|
20049
|
+
lines.push("", `### Error`, "", error);
|
|
20050
|
+
}
|
|
20051
|
+
if (turnLog.error) {
|
|
20052
|
+
lines.push("", "### Turn log error", "", turnLog.error);
|
|
20053
|
+
}
|
|
20054
|
+
lines.push("");
|
|
20055
|
+
if (iterationLines.length === 0) {
|
|
20056
|
+
lines.splice(lines.indexOf("## Harness loop iterations") + 1, 0, "- _No iterations recorded._");
|
|
20057
|
+
}
|
|
20058
|
+
return `${lines.join("\n")}
|
|
20059
|
+
`;
|
|
20060
|
+
}
|
|
19534
20061
|
function describeScenarioBehavior(result) {
|
|
19535
20062
|
if (result.scenario.description?.trim()) {
|
|
19536
20063
|
return result.scenario.description.trim();
|
|
@@ -20311,6 +20838,16 @@ function createAgentEvalHarness(options) {
|
|
|
20311
20838
|
const chatTimeoutMs = options.chatTimeoutMs ?? 12e4;
|
|
20312
20839
|
const jobTimeoutMs = options.jobTimeoutMs ?? 9e4;
|
|
20313
20840
|
const pollIntervalMs = options.pollIntervalMs ?? 250;
|
|
20841
|
+
const isTruthyEnv = (value) => {
|
|
20842
|
+
return value?.trim().toLowerCase() === "1" || value?.trim().toLowerCase() === "true" || value?.trim().toLowerCase() === "yes" || value?.trim().toLowerCase() === "on";
|
|
20843
|
+
};
|
|
20844
|
+
const localTurnReportsEnabled = (() => {
|
|
20845
|
+
if (typeof options.local === "boolean") return options.local;
|
|
20846
|
+
return process.env.NODE_ENV === "development" || isTruthyEnv(process.env.GRANULAR_LOCAL) || isTruthyEnv(process.env.GRANULAR_USE_LOCAL_ENDPOINTS) || isTruthyEnv(process.env.GRANULAR_USE_LOCAL) || process.env.GRANULAR_ENDPOINT_MODE?.toLowerCase() === "local";
|
|
20847
|
+
})();
|
|
20848
|
+
const localTurnReportBaseDir = path.resolve(
|
|
20849
|
+
options.localTurnReportBaseDir || process.cwd()
|
|
20850
|
+
);
|
|
20314
20851
|
const resolvedTemplate = resolveHarnessTemplate(
|
|
20315
20852
|
options.harnessTemplateId || process.env.GRANULAR_AGENT_HARNESS_TEMPLATE || "stable");
|
|
20316
20853
|
const promptRenderer = options.promptRenderer || resolvedTemplate.renderPrompt;
|
|
@@ -20338,6 +20875,7 @@ function createAgentEvalHarness(options) {
|
|
|
20338
20875
|
if (environment.getEffects().length > 0) break;
|
|
20339
20876
|
await sleep2(250);
|
|
20340
20877
|
}
|
|
20878
|
+
await waitForSessionDocumentContext(environment);
|
|
20341
20879
|
await ensureDir(path.join(artifactDir, slugify(label)));
|
|
20342
20880
|
return {
|
|
20343
20881
|
label,
|
|
@@ -20358,6 +20896,41 @@ function createAgentEvalHarness(options) {
|
|
|
20358
20896
|
} catch {
|
|
20359
20897
|
}
|
|
20360
20898
|
}
|
|
20899
|
+
async function writeLocalTurnReport(input) {
|
|
20900
|
+
if (!localTurnReportsEnabled) return;
|
|
20901
|
+
const { status, conversation, turnLog, request, requestTimestamp, error } = input;
|
|
20902
|
+
const requestId = safePathSegment(
|
|
20903
|
+
new Date(requestTimestamp).toISOString().replace(/[:.]/g, "-"),
|
|
20904
|
+
"turn"
|
|
20905
|
+
);
|
|
20906
|
+
const reportDir = path.join(
|
|
20907
|
+
localTurnReportBaseDir,
|
|
20908
|
+
safePathSegment(conversation.environment.ontologyId, "ontology"),
|
|
20909
|
+
safePathSegment(conversation.environment.subjectId, "subject"),
|
|
20910
|
+
safePathSegment(conversation.environment.sessionId, "session")
|
|
20911
|
+
);
|
|
20912
|
+
await ensureDir(reportDir);
|
|
20913
|
+
const completed = input.completed;
|
|
20914
|
+
const pending = input.pending;
|
|
20915
|
+
const report = buildTurnMdxReport({
|
|
20916
|
+
status,
|
|
20917
|
+
conversation,
|
|
20918
|
+
request,
|
|
20919
|
+
requestTimestamp,
|
|
20920
|
+
turnLog,
|
|
20921
|
+
responseText: completed?.responseText,
|
|
20922
|
+
terminalKind: completed?.terminalKind,
|
|
20923
|
+
actionSummary: completed?.actionSummary,
|
|
20924
|
+
promptInteractions: completed?.promptInteractions || pending?.promptInteractions,
|
|
20925
|
+
result: completed?.result,
|
|
20926
|
+
prompts: pending?.prompts,
|
|
20927
|
+
verification: completed?.verification,
|
|
20928
|
+
error
|
|
20929
|
+
});
|
|
20930
|
+
const reportPath = path.join(reportDir, `${requestId}.mdx`);
|
|
20931
|
+
await writeFile(reportPath, report);
|
|
20932
|
+
console.log(`[agent][local] saved query report: ${reportPath}`);
|
|
20933
|
+
}
|
|
20361
20934
|
async function runCheckJob(code, session) {
|
|
20362
20935
|
const job = await session.submitJob(code);
|
|
20363
20936
|
return withTimeout2(job.result, jobTimeoutMs, `check job ${job.id}`);
|
|
@@ -20617,237 +21190,329 @@ function createAgentEvalHarness(options) {
|
|
|
20617
21190
|
const baselineClosureId = getCurrentClosureId(
|
|
20618
21191
|
cloneJson(conversation.environment.document)
|
|
20619
21192
|
);
|
|
20620
|
-
|
|
20621
|
-
|
|
20622
|
-
|
|
20623
|
-
|
|
20624
|
-
|
|
20625
|
-
|
|
20626
|
-
|
|
20627
|
-
|
|
20628
|
-
|
|
21193
|
+
const writeTurnReport = async (status) => {
|
|
21194
|
+
await writeLocalTurnReport({
|
|
21195
|
+
status: status.type,
|
|
21196
|
+
conversation,
|
|
21197
|
+
turnLog,
|
|
21198
|
+
request: input.request,
|
|
21199
|
+
requestTimestamp: boundaryTimestamp,
|
|
21200
|
+
completed: status.completed,
|
|
21201
|
+
pending: status.pending,
|
|
21202
|
+
error: status.error
|
|
20629
21203
|
});
|
|
20630
|
-
|
|
20631
|
-
|
|
20632
|
-
|
|
20633
|
-
|
|
20634
|
-
|
|
20635
|
-
|
|
20636
|
-
|
|
20637
|
-
|
|
20638
|
-
};
|
|
20639
|
-
const tools = conversation.environment.getEffects().map((tool) => ({
|
|
20640
|
-
name: tool.name,
|
|
20641
|
-
description: tool.description,
|
|
20642
|
-
className: tool.className,
|
|
20643
|
-
static: tool.static,
|
|
20644
|
-
ready: tool.ready,
|
|
20645
|
-
inputSchema: tool.inputSchema,
|
|
20646
|
-
outputSchema: tool.outputSchema
|
|
20647
|
-
}));
|
|
20648
|
-
const renderedPrompt = promptRenderer({
|
|
20649
|
-
domainDocumentation: await conversation.environment.getDomainDocumentation(),
|
|
20650
|
-
sessionContext: {
|
|
20651
|
-
sandboxId: conversation.environment.sandboxId,
|
|
20652
|
-
environmentId: conversation.environment.environmentId,
|
|
20653
|
-
domainRevision: conversation.environment.domainRevision
|
|
20654
|
-
},
|
|
20655
|
-
heapSummary: projectHeapSummary(asRecord6(liveDoc?.heap), {
|
|
20656
|
-
focus: heapFocus
|
|
20657
|
-
}),
|
|
20658
|
-
fileSummary: projectSessionFileSummary(liveDoc),
|
|
20659
|
-
referentSummary: projectConversationReferentSummary(liveDoc),
|
|
20660
|
-
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
21204
|
+
};
|
|
21205
|
+
try {
|
|
21206
|
+
while (iteration < maxIterations) {
|
|
21207
|
+
const liveDoc = cloneJson(conversation.environment.document);
|
|
21208
|
+
const promptHeap = normalizeHeapSnapshot2(asRecord6(liveDoc?.heap));
|
|
21209
|
+
const pendingPrompts = filterPromptsByBoundary(
|
|
21210
|
+
liveDoc,
|
|
21211
|
+
getOpenPromptsFromDoc(liveDoc),
|
|
20661
21212
|
boundaryTimestamp
|
|
20662
|
-
|
|
20663
|
-
|
|
21213
|
+
);
|
|
21214
|
+
const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
|
|
20664
21215
|
boundaryTimestamp
|
|
20665
|
-
})
|
|
20666
|
-
|
|
20667
|
-
|
|
20668
|
-
|
|
20669
|
-
|
|
20670
|
-
|
|
20671
|
-
|
|
20672
|
-
|
|
20673
|
-
|
|
20674
|
-
|
|
20675
|
-
|
|
20676
|
-
|
|
20677
|
-
|
|
20678
|
-
|
|
20679
|
-
|
|
20680
|
-
|
|
20681
|
-
|
|
20682
|
-
|
|
20683
|
-
|
|
20684
|
-
|
|
20685
|
-
|
|
20686
|
-
|
|
20687
|
-
|
|
20688
|
-
|
|
20689
|
-
|
|
20690
|
-
|
|
20691
|
-
await emitProgress(onProgress, {
|
|
20692
|
-
phase: "generation",
|
|
20693
|
-
status: "running",
|
|
20694
|
-
scenarioId: conversation.label,
|
|
20695
|
-
stepId: turnId,
|
|
20696
|
-
iteration: iteration + 1,
|
|
20697
|
-
templateId: renderedPrompt.templateId,
|
|
20698
|
-
templateVersion: renderedPrompt.templateVersion,
|
|
20699
|
-
title: iteration === 0 ? "Generating agent response" : "Generating continuation",
|
|
20700
|
-
message: request
|
|
20701
|
-
});
|
|
20702
|
-
const generation = await withTimeout2(
|
|
20703
|
-
generateTurnWithRepair(options.generator, {
|
|
20704
|
-
systemPrompt,
|
|
20705
|
-
history: buildHistory(conversation.history),
|
|
20706
|
-
request,
|
|
20707
|
-
attempt: 1,
|
|
20708
|
-
tools,
|
|
20709
|
-
usageContext: {
|
|
21216
|
+
});
|
|
21217
|
+
const referentFocus = projectConversationReferentFocus(liveDoc);
|
|
21218
|
+
const heapFocus = {
|
|
21219
|
+
variableNames: [
|
|
21220
|
+
...workflowFocus.variableNames,
|
|
21221
|
+
...referentFocus.variableNames
|
|
21222
|
+
],
|
|
21223
|
+
listNames: [...workflowFocus.listNames, ...referentFocus.listNames],
|
|
21224
|
+
entryPaths: [
|
|
21225
|
+
...workflowFocus.entryPaths,
|
|
21226
|
+
...referentFocus.entryPaths,
|
|
21227
|
+
...focusedHeapEntryPathsFromUiContext(promptHeap, input.uiContext)
|
|
21228
|
+
]
|
|
21229
|
+
};
|
|
21230
|
+
const tools = conversation.environment.getEffects().map((tool) => ({
|
|
21231
|
+
name: tool.name,
|
|
21232
|
+
description: tool.description,
|
|
21233
|
+
className: tool.className,
|
|
21234
|
+
static: tool.static,
|
|
21235
|
+
ready: tool.ready,
|
|
21236
|
+
inputSchema: tool.inputSchema,
|
|
21237
|
+
outputSchema: tool.outputSchema
|
|
21238
|
+
}));
|
|
21239
|
+
const renderedPrompt = promptRenderer({
|
|
21240
|
+
domainDocumentation: await conversation.environment.getDomainDocumentation(),
|
|
21241
|
+
sessionContext: {
|
|
20710
21242
|
sandboxId: conversation.environment.sandboxId,
|
|
20711
21243
|
environmentId: conversation.environment.environmentId,
|
|
20712
|
-
|
|
20713
|
-
|
|
20714
|
-
|
|
21244
|
+
domainRevision: conversation.environment.domainRevision,
|
|
21245
|
+
uiContext: input.uiContext || null
|
|
21246
|
+
},
|
|
21247
|
+
heapSummary: projectHeapSummary(asRecord6(liveDoc?.heap), {
|
|
21248
|
+
focus: heapFocus
|
|
21249
|
+
}),
|
|
21250
|
+
fileSummary: projectSessionFileSummary(liveDoc),
|
|
21251
|
+
referentSummary: projectConversationReferentSummary(liveDoc),
|
|
21252
|
+
loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
|
|
21253
|
+
boundaryTimestamp
|
|
21254
|
+
}),
|
|
21255
|
+
workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
|
|
21256
|
+
boundaryTimestamp
|
|
21257
|
+
}),
|
|
21258
|
+
tools,
|
|
21259
|
+
checkpoint: latestCheckpoint
|
|
21260
|
+
});
|
|
21261
|
+
const systemPrompt = renderedPrompt.prompt;
|
|
21262
|
+
await emitProgress(onProgress, {
|
|
21263
|
+
phase: "prompt",
|
|
21264
|
+
status: "passed",
|
|
21265
|
+
scenarioId: conversation.label,
|
|
21266
|
+
stepId: turnId,
|
|
21267
|
+
iteration: iteration + 1,
|
|
21268
|
+
templateId: renderedPrompt.templateId,
|
|
21269
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21270
|
+
title: "Rendered harness prompt",
|
|
21271
|
+
message: `${systemPrompt.split("\n").length} lines`,
|
|
21272
|
+
data: {
|
|
21273
|
+
templateId: renderedPrompt.templateId,
|
|
21274
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21275
|
+
templateHash: renderedPrompt.templateHash,
|
|
21276
|
+
promptInstanceHash: renderedPrompt.promptInstanceHash,
|
|
21277
|
+
prompt: systemPrompt
|
|
20715
21278
|
}
|
|
20716
|
-
})
|
|
20717
|
-
|
|
20718
|
-
|
|
20719
|
-
|
|
20720
|
-
|
|
20721
|
-
|
|
20722
|
-
|
|
20723
|
-
|
|
20724
|
-
|
|
20725
|
-
|
|
20726
|
-
|
|
20727
|
-
|
|
20728
|
-
|
|
20729
|
-
|
|
20730
|
-
|
|
20731
|
-
|
|
20732
|
-
|
|
20733
|
-
|
|
20734
|
-
|
|
20735
|
-
|
|
20736
|
-
|
|
20737
|
-
|
|
20738
|
-
|
|
20739
|
-
|
|
20740
|
-
|
|
20741
|
-
|
|
20742
|
-
|
|
20743
|
-
|
|
20744
|
-
|
|
20745
|
-
|
|
20746
|
-
|
|
20747
|
-
|
|
20748
|
-
generationAttempts: generation.generationAttempts,
|
|
20749
|
-
tokenUsage: tokenUsageForGenerationOutput(generation)
|
|
20750
|
-
};
|
|
20751
|
-
turnLog.iterations.push(iterationLog);
|
|
20752
|
-
await writeJson(
|
|
20753
|
-
path.join(turnDir, `iteration-${iteration + 1}-generation.json`),
|
|
20754
|
-
generation
|
|
20755
|
-
);
|
|
20756
|
-
if (!generation.code) {
|
|
20757
|
-
const responseText2 = generation.reply?.trim() || "Done.";
|
|
20758
|
-
conversation.history.push({ role: "assistant", content: responseText2 });
|
|
20759
|
-
const completed = {
|
|
20760
|
-
conversation,
|
|
20761
|
-
request: input.request,
|
|
20762
|
-
turnDir,
|
|
20763
|
-
responseText: responseText2,
|
|
20764
|
-
terminalKind: "reply",
|
|
20765
|
-
actionSummary: [],
|
|
20766
|
-
promptInteractions: [],
|
|
20767
|
-
verification: null,
|
|
20768
|
-
result: generation.reply?.trim() || responseText2
|
|
20769
|
-
};
|
|
20770
|
-
if (input.verification) {
|
|
20771
|
-
completed.verification = await runInspection(
|
|
20772
|
-
conversation,
|
|
20773
|
-
input.verification,
|
|
20774
|
-
completed,
|
|
20775
|
-
turnDir
|
|
20776
|
-
);
|
|
20777
|
-
}
|
|
20778
|
-
iterationLog.responseText = responseText2;
|
|
20779
|
-
iterationLog.terminalKind = "reply";
|
|
20780
|
-
iterationLog.actionSummary = [];
|
|
20781
|
-
iterationLog.promptInteractions = [];
|
|
20782
|
-
iterationLog.result = completed.result;
|
|
20783
|
-
turnLog.completed = {
|
|
20784
|
-
responseText: responseText2,
|
|
20785
|
-
terminalKind: "reply",
|
|
20786
|
-
actionSummary: [],
|
|
20787
|
-
promptInteractions: [],
|
|
20788
|
-
result: completed.result
|
|
21279
|
+
});
|
|
21280
|
+
const request = iteration === 0 ? input.request : continuationRenderer(
|
|
21281
|
+
buildContinuationPreview(latestCheckpoint, noProgressCount)
|
|
21282
|
+
).instruction;
|
|
21283
|
+
await emitProgress(onProgress, {
|
|
21284
|
+
phase: "generation",
|
|
21285
|
+
status: "running",
|
|
21286
|
+
scenarioId: conversation.label,
|
|
21287
|
+
stepId: turnId,
|
|
21288
|
+
iteration: iteration + 1,
|
|
21289
|
+
templateId: renderedPrompt.templateId,
|
|
21290
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21291
|
+
title: iteration === 0 ? "Generating agent response" : "Generating continuation",
|
|
21292
|
+
message: request
|
|
21293
|
+
});
|
|
21294
|
+
const emittedGeneratedReasoningLines = /* @__PURE__ */ new Set();
|
|
21295
|
+
let generatedReasoningBuffer = "";
|
|
21296
|
+
const emitGeneratedReasoningLine = async (line) => {
|
|
21297
|
+
if (emittedGeneratedReasoningLines.has(line)) return;
|
|
21298
|
+
emittedGeneratedReasoningLines.add(line);
|
|
21299
|
+
await emitProgress(onProgress, {
|
|
21300
|
+
phase: "generation",
|
|
21301
|
+
status: "running",
|
|
21302
|
+
scenarioId: conversation.label,
|
|
21303
|
+
stepId: turnId,
|
|
21304
|
+
iteration: iteration + 1,
|
|
21305
|
+
templateId: renderedPrompt.templateId,
|
|
21306
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21307
|
+
title: "Generated reasoning comment",
|
|
21308
|
+
message: line,
|
|
21309
|
+
data: { source: "generated_code_comment" }
|
|
21310
|
+
});
|
|
20789
21311
|
};
|
|
20790
|
-
|
|
21312
|
+
const generationStartedAt = Date.now();
|
|
21313
|
+
const generation = await withTimeout2(
|
|
21314
|
+
generateTurnWithRepair(options.generator, {
|
|
21315
|
+
systemPrompt,
|
|
21316
|
+
history: buildHistory(conversation.history),
|
|
21317
|
+
request,
|
|
21318
|
+
attempt: 1,
|
|
21319
|
+
tools,
|
|
21320
|
+
onReplyDelta: async (delta) => {
|
|
21321
|
+
await emitProgress(onProgress, {
|
|
21322
|
+
phase: "generation",
|
|
21323
|
+
status: "running",
|
|
21324
|
+
scenarioId: conversation.label,
|
|
21325
|
+
stepId: turnId,
|
|
21326
|
+
iteration: iteration + 1,
|
|
21327
|
+
templateId: renderedPrompt.templateId,
|
|
21328
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21329
|
+
title: "Generated text reply delta",
|
|
21330
|
+
message: delta,
|
|
21331
|
+
data: { delta }
|
|
21332
|
+
});
|
|
21333
|
+
},
|
|
21334
|
+
onCodeDelta: async (delta) => {
|
|
21335
|
+
const parsed = consumeGranularReasoningOnlyChunk(
|
|
21336
|
+
generatedReasoningBuffer,
|
|
21337
|
+
delta
|
|
21338
|
+
);
|
|
21339
|
+
generatedReasoningBuffer = parsed.buffer;
|
|
21340
|
+
for (const line of parsed.reasoningLines) {
|
|
21341
|
+
await emitGeneratedReasoningLine(line);
|
|
21342
|
+
}
|
|
21343
|
+
},
|
|
21344
|
+
usageContext: {
|
|
21345
|
+
sandboxId: conversation.environment.sandboxId,
|
|
21346
|
+
environmentId: conversation.environment.environmentId,
|
|
21347
|
+
sessionId: conversation.environment.sessionId,
|
|
21348
|
+
subjectId: conversation.environment.subjectId,
|
|
21349
|
+
permissionProfileId: conversation.environment.permissionProfileId
|
|
21350
|
+
}
|
|
21351
|
+
}),
|
|
21352
|
+
chatTimeoutMs,
|
|
21353
|
+
`chat generation for ${conversation.label} iteration ${iteration + 1}`
|
|
21354
|
+
);
|
|
21355
|
+
const generationDurationMs = Date.now() - generationStartedAt;
|
|
20791
21356
|
await emitProgress(onProgress, {
|
|
20792
|
-
phase: "
|
|
21357
|
+
phase: "generation",
|
|
20793
21358
|
status: "passed",
|
|
20794
21359
|
scenarioId: conversation.label,
|
|
20795
21360
|
stepId: turnId,
|
|
20796
|
-
|
|
20797
|
-
|
|
20798
|
-
|
|
21361
|
+
iteration: iteration + 1,
|
|
21362
|
+
templateId: renderedPrompt.templateId,
|
|
21363
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21364
|
+
title: generation.code ? "Generated job code" : "Generated text reply",
|
|
21365
|
+
message: generation.code || generation.reply || "",
|
|
21366
|
+
data: {
|
|
21367
|
+
reply: generation.reply,
|
|
21368
|
+
code: generation.code,
|
|
21369
|
+
attempts: generation.generationAttempts,
|
|
21370
|
+
usage: tokenUsageForGenerationOutput(generation)
|
|
21371
|
+
}
|
|
20799
21372
|
});
|
|
20800
|
-
|
|
20801
|
-
|
|
20802
|
-
|
|
20803
|
-
|
|
20804
|
-
|
|
20805
|
-
|
|
20806
|
-
|
|
20807
|
-
systemPrompt,
|
|
20808
|
-
history: buildHistory(conversation.history),
|
|
20809
|
-
scenarioLabel: conversation.label,
|
|
20810
|
-
turnId,
|
|
21373
|
+
const generatedReasoningLines = generation.code ? consumeGranularReasoningOnlyChunk("", generation.code, {
|
|
21374
|
+
final: true
|
|
21375
|
+
}).reasoningLines : [];
|
|
21376
|
+
for (const line of generatedReasoningLines) {
|
|
21377
|
+
await emitGeneratedReasoningLine(line);
|
|
21378
|
+
}
|
|
21379
|
+
const iterationLog = {
|
|
20811
21380
|
iteration: iteration + 1,
|
|
20812
|
-
|
|
21381
|
+
request,
|
|
21382
|
+
generationDurationMs,
|
|
21383
|
+
systemPrompt,
|
|
21384
|
+
templateId: renderedPrompt.templateId,
|
|
21385
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21386
|
+
templateHash: renderedPrompt.templateHash,
|
|
21387
|
+
promptInstanceHash: renderedPrompt.promptInstanceHash,
|
|
20813
21388
|
generationReply: generation.reply,
|
|
21389
|
+
generatedCode: generation.code,
|
|
20814
21390
|
rawGeneration: generation.raw,
|
|
20815
|
-
|
|
20816
|
-
|
|
20817
|
-
|
|
20818
|
-
|
|
20819
|
-
|
|
20820
|
-
|
|
20821
|
-
|
|
20822
|
-
|
|
20823
|
-
|
|
20824
|
-
|
|
20825
|
-
|
|
20826
|
-
|
|
20827
|
-
|
|
20828
|
-
|
|
20829
|
-
|
|
20830
|
-
|
|
20831
|
-
|
|
20832
|
-
|
|
20833
|
-
|
|
20834
|
-
|
|
20835
|
-
|
|
20836
|
-
|
|
20837
|
-
|
|
20838
|
-
|
|
20839
|
-
|
|
20840
|
-
|
|
21391
|
+
generationAttempts: generation.generationAttempts,
|
|
21392
|
+
tokenUsage: tokenUsageForGenerationOutput(generation)
|
|
21393
|
+
};
|
|
21394
|
+
turnLog.iterations.push(iterationLog);
|
|
21395
|
+
await writeJson(
|
|
21396
|
+
path.join(turnDir, `iteration-${iteration + 1}-generation.json`),
|
|
21397
|
+
generation
|
|
21398
|
+
);
|
|
21399
|
+
if (!generation.code) {
|
|
21400
|
+
const responseText2 = generation.reply?.trim() || "Done.";
|
|
21401
|
+
conversation.history.push({ role: "assistant", content: responseText2 });
|
|
21402
|
+
const completed = {
|
|
21403
|
+
conversation,
|
|
21404
|
+
request: input.request,
|
|
21405
|
+
turnDir,
|
|
21406
|
+
responseText: responseText2,
|
|
21407
|
+
terminalKind: "reply",
|
|
21408
|
+
actionSummary: [],
|
|
21409
|
+
promptInteractions: [],
|
|
21410
|
+
verification: null,
|
|
21411
|
+
result: generation.reply?.trim() || responseText2
|
|
21412
|
+
};
|
|
21413
|
+
if (input.verification) {
|
|
21414
|
+
completed.verification = await runInspection(
|
|
21415
|
+
conversation,
|
|
21416
|
+
input.verification,
|
|
21417
|
+
completed,
|
|
21418
|
+
turnDir
|
|
21419
|
+
);
|
|
21420
|
+
}
|
|
21421
|
+
iterationLog.responseText = responseText2;
|
|
21422
|
+
iterationLog.terminalKind = "reply";
|
|
21423
|
+
iterationLog.actionSummary = [];
|
|
21424
|
+
iterationLog.promptInteractions = [];
|
|
21425
|
+
iterationLog.result = completed.result;
|
|
21426
|
+
turnLog.completed = {
|
|
21427
|
+
responseText: responseText2,
|
|
21428
|
+
terminalKind: "reply",
|
|
21429
|
+
actionSummary: [],
|
|
21430
|
+
promptInteractions: [],
|
|
21431
|
+
result: completed.result
|
|
21432
|
+
};
|
|
21433
|
+
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
21434
|
+
await writeTurnReport({ type: "completed", completed });
|
|
21435
|
+
await emitProgress(onProgress, {
|
|
21436
|
+
phase: "step",
|
|
21437
|
+
status: "passed",
|
|
21438
|
+
scenarioId: conversation.label,
|
|
21439
|
+
stepId: turnId,
|
|
21440
|
+
title: "Step completed with text reply",
|
|
21441
|
+
message: responseText2,
|
|
21442
|
+
data: completed
|
|
21443
|
+
});
|
|
21444
|
+
return completed;
|
|
21445
|
+
}
|
|
21446
|
+
const session = conversation.environment;
|
|
21447
|
+
const job = await session.submitJob(generation.code, {
|
|
21448
|
+
agent: {
|
|
21449
|
+
userRequest: input.request,
|
|
21450
|
+
generationRequest: request,
|
|
21451
|
+
systemPrompt,
|
|
21452
|
+
history: buildHistory(conversation.history),
|
|
21453
|
+
scenarioLabel: conversation.label,
|
|
21454
|
+
turnId,
|
|
21455
|
+
iteration: iteration + 1,
|
|
21456
|
+
tools,
|
|
21457
|
+
generationReply: generation.reply,
|
|
21458
|
+
rawGeneration: generation.raw,
|
|
21459
|
+
repairIssues: generation.generationAttempts?.flatMap(
|
|
21460
|
+
(attempt) => attempt.repairIssues || []
|
|
21461
|
+
)
|
|
21462
|
+
}
|
|
21463
|
+
});
|
|
21464
|
+
await emitProgress(onProgress, {
|
|
21465
|
+
phase: "job",
|
|
21466
|
+
status: "running",
|
|
20841
21467
|
scenarioId: conversation.label,
|
|
20842
21468
|
stepId: turnId,
|
|
20843
21469
|
iteration: iteration + 1,
|
|
21470
|
+
jobId: job.id,
|
|
20844
21471
|
templateId: renderedPrompt.templateId,
|
|
20845
|
-
templateVersion: renderedPrompt.templateVersion
|
|
20846
|
-
|
|
20847
|
-
|
|
20848
|
-
|
|
20849
|
-
|
|
20850
|
-
|
|
21472
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21473
|
+
title: "Submitted Granular job",
|
|
21474
|
+
message: job.id,
|
|
21475
|
+
data: { code: generation.code }
|
|
21476
|
+
});
|
|
21477
|
+
const outcome = await waitForJobOutcome({
|
|
21478
|
+
environment: conversation.environment,
|
|
21479
|
+
job,
|
|
21480
|
+
boundaryTimestamp,
|
|
21481
|
+
timeoutMs: jobTimeoutMs,
|
|
21482
|
+
pollIntervalMs,
|
|
21483
|
+
onProgress: (event) => void onProgress?.(event),
|
|
21484
|
+
progressContext: {
|
|
21485
|
+
scenarioId: conversation.label,
|
|
21486
|
+
stepId: turnId,
|
|
21487
|
+
iteration: iteration + 1,
|
|
21488
|
+
templateId: renderedPrompt.templateId,
|
|
21489
|
+
templateVersion: renderedPrompt.templateVersion
|
|
21490
|
+
}
|
|
21491
|
+
});
|
|
21492
|
+
if (outcome.kind === "prompt") {
|
|
21493
|
+
if (!autoAnswerPrompts) {
|
|
21494
|
+
const pending2 = {
|
|
21495
|
+
conversation,
|
|
21496
|
+
request: input.request,
|
|
21497
|
+
turnDir,
|
|
21498
|
+
boundaryTimestamp,
|
|
21499
|
+
finalCode: generation.code,
|
|
21500
|
+
finalReply: generation.reply?.trim() || "",
|
|
21501
|
+
stdout: outcome.stdout,
|
|
21502
|
+
stderr: outcome.stderr,
|
|
21503
|
+
prompts: outcome.prompts,
|
|
21504
|
+
promptInteractions: [],
|
|
21505
|
+
job
|
|
21506
|
+
};
|
|
21507
|
+
await writeTurnReport({ type: "pending", pending: pending2 });
|
|
21508
|
+
return pending2;
|
|
21509
|
+
}
|
|
21510
|
+
if (!input.human) {
|
|
21511
|
+
throw new Error(
|
|
21512
|
+
"This turn reached a human prompt but no responder was provided"
|
|
21513
|
+
);
|
|
21514
|
+
}
|
|
21515
|
+
let pending = {
|
|
20851
21516
|
conversation,
|
|
20852
21517
|
request: input.request,
|
|
20853
21518
|
turnDir,
|
|
@@ -20860,187 +21525,180 @@ function createAgentEvalHarness(options) {
|
|
|
20860
21525
|
promptInteractions: [],
|
|
20861
21526
|
job
|
|
20862
21527
|
};
|
|
21528
|
+
while ("prompts" in pending) {
|
|
21529
|
+
const resumed = await resumePendingTurn(pending, input.human);
|
|
21530
|
+
if ("prompts" in resumed) {
|
|
21531
|
+
pending = resumed;
|
|
21532
|
+
continue;
|
|
21533
|
+
}
|
|
21534
|
+
if (input.verification) {
|
|
21535
|
+
resumed.verification = await runInspection(
|
|
21536
|
+
conversation,
|
|
21537
|
+
input.verification,
|
|
21538
|
+
resumed,
|
|
21539
|
+
turnDir
|
|
21540
|
+
);
|
|
21541
|
+
}
|
|
21542
|
+
turnLog.completed = {
|
|
21543
|
+
responseText: resumed.responseText,
|
|
21544
|
+
terminalKind: resumed.terminalKind,
|
|
21545
|
+
actionSummary: resumed.actionSummary,
|
|
21546
|
+
promptInteractions: resumed.promptInteractions,
|
|
21547
|
+
result: resumed.result
|
|
21548
|
+
};
|
|
21549
|
+
await writeTurnReport({ type: "completed", completed: resumed });
|
|
21550
|
+
return resumed;
|
|
21551
|
+
}
|
|
20863
21552
|
}
|
|
20864
|
-
if (
|
|
21553
|
+
if (outcome.kind !== "completed") {
|
|
20865
21554
|
throw new Error(
|
|
20866
|
-
"
|
|
21555
|
+
"Unexpected non-completed outcome after prompt handling"
|
|
20867
21556
|
);
|
|
20868
21557
|
}
|
|
20869
|
-
|
|
20870
|
-
|
|
20871
|
-
|
|
20872
|
-
|
|
20873
|
-
|
|
20874
|
-
|
|
20875
|
-
|
|
21558
|
+
await sleep2(350);
|
|
21559
|
+
const settledLiveDoc = cloneJson(
|
|
21560
|
+
conversation.environment.document
|
|
21561
|
+
);
|
|
21562
|
+
const sessionHeap = normalizeHeapSnapshot2(asRecord6(settledLiveDoc?.heap));
|
|
21563
|
+
const presentation = resolveJobPresentation({
|
|
21564
|
+
jobId: job.id,
|
|
21565
|
+
result: outcome.result,
|
|
20876
21566
|
stdout: outcome.stdout,
|
|
20877
|
-
|
|
20878
|
-
|
|
20879
|
-
|
|
20880
|
-
|
|
21567
|
+
agentMessages: getJobAgentMessages(settledLiveDoc, job.id),
|
|
21568
|
+
sessionHeap
|
|
21569
|
+
});
|
|
21570
|
+
const responseText = presentation.responseText || generation.reply?.trim() || "Done.";
|
|
21571
|
+
const verifierSnapshot = createHarnessVerifierSnapshot({
|
|
21572
|
+
finalCode: generation.code,
|
|
21573
|
+
resultPreview: JSON.stringify(outcome.result, null, 2),
|
|
21574
|
+
liveDoc: settledLiveDoc,
|
|
21575
|
+
projectionOptions: { boundaryTimestamp }
|
|
21576
|
+
});
|
|
21577
|
+
const continuation = evaluateContinuation({
|
|
21578
|
+
iteration,
|
|
21579
|
+
budgets: controllerBudgets,
|
|
21580
|
+
baselineClosureId,
|
|
21581
|
+
currentClosureId: getCurrentClosureId(settledLiveDoc),
|
|
21582
|
+
liveDoc: settledLiveDoc,
|
|
21583
|
+
pendingPrompts: filterPromptsByBoundary(
|
|
21584
|
+
settledLiveDoc,
|
|
21585
|
+
getOpenPromptsFromDoc(settledLiveDoc),
|
|
21586
|
+
boundaryTimestamp
|
|
21587
|
+
),
|
|
21588
|
+
projectionOptions: { boundaryTimestamp },
|
|
21589
|
+
latestResponseText: responseText,
|
|
21590
|
+
previousSnapshot,
|
|
21591
|
+
currentSnapshot: verifierSnapshot,
|
|
21592
|
+
previousNoProgressCount: noProgressCount
|
|
21593
|
+
});
|
|
21594
|
+
latestCheckpoint = {
|
|
21595
|
+
iteration: iteration + 1,
|
|
21596
|
+
latestJobStatus: "succeeded",
|
|
21597
|
+
latestJobResult: JSON.stringify(outcome.result, null, 2),
|
|
21598
|
+
latestActionSummary: getActionSummary(settledLiveDoc, job.id),
|
|
21599
|
+
controllerOutcome: continuation.outcome,
|
|
21600
|
+
controllerReason: continuation.reason,
|
|
21601
|
+
noProgressCount: continuation.nextNoProgressCount
|
|
20881
21602
|
};
|
|
20882
|
-
|
|
20883
|
-
|
|
20884
|
-
|
|
20885
|
-
|
|
20886
|
-
|
|
21603
|
+
await emitProgress(onProgress, {
|
|
21604
|
+
phase: "continuation",
|
|
21605
|
+
status: continuation.shouldContinue ? "running" : "passed",
|
|
21606
|
+
scenarioId: conversation.label,
|
|
21607
|
+
stepId: turnId,
|
|
21608
|
+
iteration: iteration + 1,
|
|
21609
|
+
jobId: job.id,
|
|
21610
|
+
templateId: renderedPrompt.templateId,
|
|
21611
|
+
templateVersion: renderedPrompt.templateVersion,
|
|
21612
|
+
title: continuation.shouldContinue ? "Harness requested another loop" : "Harness accepted completion",
|
|
21613
|
+
message: `${continuation.reason}; ${continuation.outcome}`,
|
|
21614
|
+
data: {
|
|
21615
|
+
continuation,
|
|
21616
|
+
checkpoint: latestCheckpoint,
|
|
21617
|
+
verifierSnapshot
|
|
20887
21618
|
}
|
|
21619
|
+
});
|
|
21620
|
+
previousSnapshot = verifierSnapshot;
|
|
21621
|
+
noProgressCount = continuation.nextNoProgressCount;
|
|
21622
|
+
conversation.history.push({
|
|
21623
|
+
role: "assistant",
|
|
21624
|
+
content: responseText,
|
|
21625
|
+
code: generation.code,
|
|
21626
|
+
jobStatus: "succeeded",
|
|
21627
|
+
jobResultPreview: JSON.stringify(outcome.result, null, 2)
|
|
21628
|
+
});
|
|
21629
|
+
await writeJson(
|
|
21630
|
+
path.join(turnDir, `iteration-${iteration + 1}-result.json`),
|
|
21631
|
+
{
|
|
21632
|
+
responseText,
|
|
21633
|
+
continuation,
|
|
21634
|
+
actionSummary: latestCheckpoint.latestActionSummary,
|
|
21635
|
+
result: outcome.result
|
|
21636
|
+
}
|
|
21637
|
+
);
|
|
21638
|
+
iterationLog.responseText = responseText;
|
|
21639
|
+
iterationLog.terminalKind = getCurrentClosureId(settledLiveDoc) ? "closure" : "reply";
|
|
21640
|
+
iterationLog.actionSummary = latestCheckpoint.latestActionSummary || [];
|
|
21641
|
+
iterationLog.promptInteractions = [];
|
|
21642
|
+
iterationLog.continuation = continuation;
|
|
21643
|
+
iterationLog.result = outcome.result;
|
|
21644
|
+
if (!continuation.shouldContinue) {
|
|
21645
|
+
const completed = {
|
|
21646
|
+
conversation,
|
|
21647
|
+
request: input.request,
|
|
21648
|
+
turnDir,
|
|
21649
|
+
responseText,
|
|
21650
|
+
terminalKind: getCurrentClosureId(settledLiveDoc) ? "closure" : "reply",
|
|
21651
|
+
finalCode: generation.code,
|
|
21652
|
+
actionSummary: latestCheckpoint.latestActionSummary || [],
|
|
21653
|
+
promptInteractions: [],
|
|
21654
|
+
verification: null,
|
|
21655
|
+
result: outcome.result
|
|
21656
|
+
};
|
|
20888
21657
|
if (input.verification) {
|
|
20889
|
-
|
|
21658
|
+
completed.verification = await runInspection(
|
|
20890
21659
|
conversation,
|
|
20891
21660
|
input.verification,
|
|
20892
|
-
|
|
21661
|
+
completed,
|
|
20893
21662
|
turnDir
|
|
20894
21663
|
);
|
|
20895
21664
|
}
|
|
20896
21665
|
turnLog.completed = {
|
|
20897
|
-
responseText
|
|
20898
|
-
terminalKind:
|
|
20899
|
-
actionSummary:
|
|
20900
|
-
promptInteractions:
|
|
20901
|
-
result:
|
|
21666
|
+
responseText,
|
|
21667
|
+
terminalKind: completed.terminalKind,
|
|
21668
|
+
actionSummary: completed.actionSummary,
|
|
21669
|
+
promptInteractions: [],
|
|
21670
|
+
result: outcome.result
|
|
20902
21671
|
};
|
|
20903
|
-
|
|
21672
|
+
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
21673
|
+
await writeTurnReport({ type: "completed", completed });
|
|
21674
|
+
await emitProgress(onProgress, {
|
|
21675
|
+
phase: "step",
|
|
21676
|
+
status: "passed",
|
|
21677
|
+
scenarioId: conversation.label,
|
|
21678
|
+
stepId: turnId,
|
|
21679
|
+
iteration: iteration + 1,
|
|
21680
|
+
jobId: job.id,
|
|
21681
|
+
title: "Step completed",
|
|
21682
|
+
message: responseText,
|
|
21683
|
+
data: completed
|
|
21684
|
+
});
|
|
21685
|
+
return completed;
|
|
20904
21686
|
}
|
|
21687
|
+
iteration += 1;
|
|
20905
21688
|
}
|
|
20906
|
-
|
|
20907
|
-
|
|
20908
|
-
"Unexpected non-completed outcome after prompt handling"
|
|
20909
|
-
);
|
|
20910
|
-
}
|
|
20911
|
-
await sleep2(350);
|
|
20912
|
-
const settledLiveDoc = cloneJson(
|
|
20913
|
-
conversation.environment.document
|
|
20914
|
-
);
|
|
20915
|
-
const sessionHeap = normalizeHeapSnapshot2(asRecord6(settledLiveDoc?.heap));
|
|
20916
|
-
const presentation = resolveJobPresentation({
|
|
20917
|
-
jobId: job.id,
|
|
20918
|
-
result: outcome.result,
|
|
20919
|
-
stdout: outcome.stdout,
|
|
20920
|
-
agentMessages: getJobAgentMessages(settledLiveDoc, job.id),
|
|
20921
|
-
sessionHeap
|
|
20922
|
-
});
|
|
20923
|
-
const responseText = presentation.responseText || generation.reply?.trim() || "Done.";
|
|
20924
|
-
const verifierSnapshot = createHarnessVerifierSnapshot({
|
|
20925
|
-
finalCode: generation.code,
|
|
20926
|
-
resultPreview: JSON.stringify(outcome.result, null, 2),
|
|
20927
|
-
liveDoc: settledLiveDoc,
|
|
20928
|
-
projectionOptions: { boundaryTimestamp }
|
|
20929
|
-
});
|
|
20930
|
-
const continuation = evaluateContinuation({
|
|
20931
|
-
iteration,
|
|
20932
|
-
budgets: controllerBudgets,
|
|
20933
|
-
baselineClosureId,
|
|
20934
|
-
currentClosureId: getCurrentClosureId(settledLiveDoc),
|
|
20935
|
-
liveDoc: settledLiveDoc,
|
|
20936
|
-
pendingPrompts: filterPromptsByBoundary(
|
|
20937
|
-
settledLiveDoc,
|
|
20938
|
-
getOpenPromptsFromDoc(settledLiveDoc),
|
|
20939
|
-
boundaryTimestamp
|
|
20940
|
-
),
|
|
20941
|
-
projectionOptions: { boundaryTimestamp },
|
|
20942
|
-
latestResponseText: responseText,
|
|
20943
|
-
previousSnapshot,
|
|
20944
|
-
currentSnapshot: verifierSnapshot,
|
|
20945
|
-
previousNoProgressCount: noProgressCount
|
|
20946
|
-
});
|
|
20947
|
-
latestCheckpoint = {
|
|
20948
|
-
iteration: iteration + 1,
|
|
20949
|
-
latestJobStatus: "succeeded",
|
|
20950
|
-
latestJobResult: JSON.stringify(outcome.result, null, 2),
|
|
20951
|
-
latestActionSummary: getActionSummary(settledLiveDoc, job.id),
|
|
20952
|
-
controllerOutcome: continuation.outcome,
|
|
20953
|
-
controllerReason: continuation.reason,
|
|
20954
|
-
noProgressCount: continuation.nextNoProgressCount
|
|
20955
|
-
};
|
|
20956
|
-
await emitProgress(onProgress, {
|
|
20957
|
-
phase: "continuation",
|
|
20958
|
-
status: continuation.shouldContinue ? "running" : "passed",
|
|
20959
|
-
scenarioId: conversation.label,
|
|
20960
|
-
stepId: turnId,
|
|
20961
|
-
iteration: iteration + 1,
|
|
20962
|
-
jobId: job.id,
|
|
20963
|
-
templateId: renderedPrompt.templateId,
|
|
20964
|
-
templateVersion: renderedPrompt.templateVersion,
|
|
20965
|
-
title: continuation.shouldContinue ? "Harness requested another loop" : "Harness accepted completion",
|
|
20966
|
-
message: `${continuation.reason}; ${continuation.outcome}`,
|
|
20967
|
-
data: {
|
|
20968
|
-
continuation,
|
|
20969
|
-
checkpoint: latestCheckpoint,
|
|
20970
|
-
verifierSnapshot
|
|
20971
|
-
}
|
|
20972
|
-
});
|
|
20973
|
-
previousSnapshot = verifierSnapshot;
|
|
20974
|
-
noProgressCount = continuation.nextNoProgressCount;
|
|
20975
|
-
conversation.history.push({
|
|
20976
|
-
role: "assistant",
|
|
20977
|
-
content: responseText,
|
|
20978
|
-
code: generation.code,
|
|
20979
|
-
jobStatus: "succeeded",
|
|
20980
|
-
jobResultPreview: JSON.stringify(outcome.result, null, 2)
|
|
20981
|
-
});
|
|
20982
|
-
await writeJson(
|
|
20983
|
-
path.join(turnDir, `iteration-${iteration + 1}-result.json`),
|
|
20984
|
-
{
|
|
20985
|
-
responseText,
|
|
20986
|
-
continuation,
|
|
20987
|
-
actionSummary: latestCheckpoint.latestActionSummary,
|
|
20988
|
-
result: outcome.result
|
|
20989
|
-
}
|
|
21689
|
+
throw new Error(
|
|
21690
|
+
`Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
|
|
20990
21691
|
);
|
|
20991
|
-
|
|
20992
|
-
|
|
20993
|
-
|
|
20994
|
-
|
|
20995
|
-
|
|
20996
|
-
iterationLog.result = outcome.result;
|
|
20997
|
-
if (!continuation.shouldContinue) {
|
|
20998
|
-
const completed = {
|
|
20999
|
-
conversation,
|
|
21000
|
-
request: input.request,
|
|
21001
|
-
turnDir,
|
|
21002
|
-
responseText,
|
|
21003
|
-
terminalKind: getCurrentClosureId(settledLiveDoc) ? "closure" : "reply",
|
|
21004
|
-
finalCode: generation.code,
|
|
21005
|
-
actionSummary: latestCheckpoint.latestActionSummary || [],
|
|
21006
|
-
promptInteractions: [],
|
|
21007
|
-
verification: null,
|
|
21008
|
-
result: outcome.result
|
|
21009
|
-
};
|
|
21010
|
-
if (input.verification) {
|
|
21011
|
-
completed.verification = await runInspection(
|
|
21012
|
-
conversation,
|
|
21013
|
-
input.verification,
|
|
21014
|
-
completed,
|
|
21015
|
-
turnDir
|
|
21016
|
-
);
|
|
21017
|
-
}
|
|
21018
|
-
turnLog.completed = {
|
|
21019
|
-
responseText,
|
|
21020
|
-
terminalKind: completed.terminalKind,
|
|
21021
|
-
actionSummary: completed.actionSummary,
|
|
21022
|
-
promptInteractions: [],
|
|
21023
|
-
result: outcome.result
|
|
21024
|
-
};
|
|
21025
|
-
await writeJson(path.join(turnDir, "result.json"), completed);
|
|
21026
|
-
await emitProgress(onProgress, {
|
|
21027
|
-
phase: "step",
|
|
21028
|
-
status: "passed",
|
|
21029
|
-
scenarioId: conversation.label,
|
|
21030
|
-
stepId: turnId,
|
|
21031
|
-
iteration: iteration + 1,
|
|
21032
|
-
jobId: job.id,
|
|
21033
|
-
title: "Step completed",
|
|
21034
|
-
message: responseText,
|
|
21035
|
-
data: completed
|
|
21036
|
-
});
|
|
21037
|
-
return completed;
|
|
21692
|
+
} catch (error) {
|
|
21693
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
21694
|
+
const latestIteration = latestIterationLog(turnLog);
|
|
21695
|
+
if (latestIteration) {
|
|
21696
|
+
latestIteration.error = message;
|
|
21038
21697
|
}
|
|
21039
|
-
|
|
21698
|
+
turnLog.error = message;
|
|
21699
|
+
await writeTurnReport({ type: "failed", error: message });
|
|
21700
|
+
throw error;
|
|
21040
21701
|
}
|
|
21041
|
-
throw new Error(
|
|
21042
|
-
`Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
|
|
21043
|
-
);
|
|
21044
21702
|
}
|
|
21045
21703
|
return {
|
|
21046
21704
|
artifactDir,
|
|
@@ -21075,6 +21733,7 @@ function createAgentTester(options) {
|
|
|
21075
21733
|
const harness = createAgentEvalHarness({
|
|
21076
21734
|
granular,
|
|
21077
21735
|
environmentId: resolvedEnvironmentId || void 0,
|
|
21736
|
+
local: options.local,
|
|
21078
21737
|
openEnvironment: async ({ clientId }) => {
|
|
21079
21738
|
if (resolvedEnvironmentId) {
|
|
21080
21739
|
return granular.createSession({
|