@the-open-engine/zeroshot 6.34.1 → 6.34.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/index.js +34 -8
- package/cli/json-export.js +77 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/src/agent/agent-task-executor.js +380 -99
- package/src/ledger.js +364 -31
- package/src/template-resolver.js +5 -0
- package/task-lib/watcher-output-runtime.js +128 -43
|
@@ -60,6 +60,11 @@ const {
|
|
|
60
60
|
} = require('./structured-output-error');
|
|
61
61
|
const TASK_TERMINAL_STATUSES = new Set(['completed', 'failed', 'killed', 'stale']);
|
|
62
62
|
const MAX_CONTROL_PLANE_RECORD_BYTES = 1024 * 1024;
|
|
63
|
+
const MAX_CONTROL_PLANE_OUTPUT_BYTES = 2 * 1024 * 1024;
|
|
64
|
+
const MAX_CONTROL_PLANE_OUTPUT_RECORDS = 1024;
|
|
65
|
+
const MAX_LIVE_OUTPUT_BYTES = 512 * 1024;
|
|
66
|
+
const MAX_LIVE_OUTPUT_RECORDS = 512;
|
|
67
|
+
const CONTROL_PLANE_OMISSION_MARKER_BYTES = 1024;
|
|
63
68
|
const LOG_READ_CHUNK_BYTES = 64 * 1024;
|
|
64
69
|
function runCommandWithTimeout(command, args, options = {}, callback = null) {
|
|
65
70
|
const timeout = options.timeout ?? 30000;
|
|
@@ -1222,8 +1227,8 @@ async function waitForTaskReady(agent, taskId, maxRetries = 10, delayMs = 200) {
|
|
|
1222
1227
|
const MAX_STATUS_FAILURES = 30;
|
|
1223
1228
|
|
|
1224
1229
|
function createLogFollowState() {
|
|
1225
|
-
|
|
1226
|
-
|
|
1230
|
+
const state = {
|
|
1231
|
+
controlPlaneOutput: createControlPlaneOutputState(),
|
|
1227
1232
|
logFilePath: null,
|
|
1228
1233
|
lastSize: 0,
|
|
1229
1234
|
pollInterval: null,
|
|
@@ -1233,6 +1238,175 @@ function createLogFollowState() {
|
|
|
1233
1238
|
logDecoder: new StringDecoder('utf8'),
|
|
1234
1239
|
consecutiveExecFailures: 0,
|
|
1235
1240
|
};
|
|
1241
|
+
defineControlPlaneOutputAccessor(state, 'output');
|
|
1242
|
+
return state;
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
function createControlPlaneOutputState(options = {}) {
|
|
1246
|
+
const maxBytes = options.maxBytes || MAX_CONTROL_PLANE_OUTPUT_BYTES;
|
|
1247
|
+
const maxRecords = options.maxRecords || MAX_CONTROL_PLANE_OUTPUT_RECORDS;
|
|
1248
|
+
const liveByteLimit = options.liveByteLimit || MAX_LIVE_OUTPUT_BYTES;
|
|
1249
|
+
const liveRecordLimit = options.liveRecordLimit || MAX_LIVE_OUTPUT_RECORDS;
|
|
1250
|
+
if (maxBytes <= CONTROL_PLANE_OMISSION_MARKER_BYTES || maxRecords < 1) {
|
|
1251
|
+
throw new Error('Control-plane output bounds must retain space for at least one record');
|
|
1252
|
+
}
|
|
1253
|
+
return {
|
|
1254
|
+
records: [],
|
|
1255
|
+
head: 0,
|
|
1256
|
+
byteLength: 0,
|
|
1257
|
+
maxBytes,
|
|
1258
|
+
maxRecords,
|
|
1259
|
+
liveByteLimit,
|
|
1260
|
+
liveRecordLimit,
|
|
1261
|
+
liveBytes: 0,
|
|
1262
|
+
liveRecords: 0,
|
|
1263
|
+
nextSequence: 0,
|
|
1264
|
+
lastLiveSequence: -1,
|
|
1265
|
+
liveSuppressed: false,
|
|
1266
|
+
terminalFlushed: false,
|
|
1267
|
+
omittedBytes: 0,
|
|
1268
|
+
omittedRecords: 0,
|
|
1269
|
+
omittedDigest: createHash('sha256'),
|
|
1270
|
+
prefixOmitted: false,
|
|
1271
|
+
};
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
function defineControlPlaneOutputAccessor(state, property) {
|
|
1275
|
+
Object.defineProperty(state, property, {
|
|
1276
|
+
configurable: true,
|
|
1277
|
+
enumerable: true,
|
|
1278
|
+
get: () => snapshotControlPlaneOutput(state.controlPlaneOutput),
|
|
1279
|
+
});
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
function ensureControlPlaneOutputState(state = {}) {
|
|
1283
|
+
if (state.controlPlaneOutput) return state;
|
|
1284
|
+
let existingOutput = '';
|
|
1285
|
+
if (typeof state.output === 'string') {
|
|
1286
|
+
existingOutput = state.output;
|
|
1287
|
+
} else if (typeof state.fullOutput === 'string') {
|
|
1288
|
+
existingOutput = state.fullOutput;
|
|
1289
|
+
}
|
|
1290
|
+
state.controlPlaneOutput = createControlPlaneOutputState();
|
|
1291
|
+
if (Object.hasOwn(state, 'output')) delete state.output;
|
|
1292
|
+
if (Object.hasOwn(state, 'fullOutput')) delete state.fullOutput;
|
|
1293
|
+
defineControlPlaneOutputAccessor(state, 'output');
|
|
1294
|
+
if (existingOutput) {
|
|
1295
|
+
replayCompleteLogContent(existingOutput, (content) =>
|
|
1296
|
+
appendControlPlaneRecord(state.controlPlaneOutput, {
|
|
1297
|
+
content,
|
|
1298
|
+
timestamp: Date.now(),
|
|
1299
|
+
type: isValidJsonLine(content) ? 'json' : 'text',
|
|
1300
|
+
})
|
|
1301
|
+
);
|
|
1302
|
+
}
|
|
1303
|
+
return state;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
function retainedControlPlaneRecords(outputState) {
|
|
1307
|
+
return outputState.records.slice(outputState.head);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
function compactControlPlaneRecords(outputState) {
|
|
1311
|
+
if (outputState.head >= 64 && outputState.head * 2 >= outputState.records.length) {
|
|
1312
|
+
outputState.records = outputState.records.slice(outputState.head);
|
|
1313
|
+
outputState.head = 0;
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
function omitOldestControlPlaneRecord(outputState) {
|
|
1318
|
+
const record = outputState.records[outputState.head++];
|
|
1319
|
+
outputState.byteLength -= record.byteLength;
|
|
1320
|
+
outputState.omittedBytes += record.byteLength;
|
|
1321
|
+
outputState.omittedRecords++;
|
|
1322
|
+
outputState.omittedDigest.update(record.text);
|
|
1323
|
+
compactControlPlaneRecords(outputState);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
function appendControlPlaneRecord(outputState, { content, timestamp, type }) {
|
|
1327
|
+
const text = `${content}\n`;
|
|
1328
|
+
const record = {
|
|
1329
|
+
sequence: outputState.nextSequence++,
|
|
1330
|
+
content,
|
|
1331
|
+
timestamp,
|
|
1332
|
+
type,
|
|
1333
|
+
text,
|
|
1334
|
+
byteLength: Buffer.byteLength(text),
|
|
1335
|
+
};
|
|
1336
|
+
outputState.records.push(record);
|
|
1337
|
+
outputState.byteLength += record.byteLength;
|
|
1338
|
+
|
|
1339
|
+
const payloadLimit = outputState.maxBytes - CONTROL_PLANE_OMISSION_MARKER_BYTES;
|
|
1340
|
+
while (
|
|
1341
|
+
outputState.byteLength > payloadLimit ||
|
|
1342
|
+
outputState.records.length - outputState.head > outputState.maxRecords
|
|
1343
|
+
) {
|
|
1344
|
+
omitOldestControlPlaneRecord(outputState);
|
|
1345
|
+
}
|
|
1346
|
+
return record;
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
function controlPlaneOmissionRecord(outputState) {
|
|
1350
|
+
if (!outputState.prefixOmitted && outputState.omittedRecords === 0) return null;
|
|
1351
|
+
const digest = outputState.omittedRecords
|
|
1352
|
+
? `, sha256=${outputState.omittedDigest.copy().digest('hex')}`
|
|
1353
|
+
: '';
|
|
1354
|
+
const known = outputState.omittedRecords
|
|
1355
|
+
? `records=${outputState.omittedRecords}, byte_length=${outputState.omittedBytes}${digest}`
|
|
1356
|
+
: 'byte_length=unknown';
|
|
1357
|
+
const content =
|
|
1358
|
+
`[ZEROSHOT] Earlier provider output omitted from the bounded control-plane tail (${known}). ` +
|
|
1359
|
+
'Complete output remains in the task log.';
|
|
1360
|
+
return {
|
|
1361
|
+
content,
|
|
1362
|
+
timestamp: Date.now(),
|
|
1363
|
+
type: 'text',
|
|
1364
|
+
text: `${content}\n`,
|
|
1365
|
+
byteLength: Buffer.byteLength(`${content}\n`),
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
function snapshotControlPlaneOutput(outputState) {
|
|
1370
|
+
const omission = controlPlaneOmissionRecord(outputState);
|
|
1371
|
+
const records = retainedControlPlaneRecords(outputState);
|
|
1372
|
+
return `${omission ? omission.text : ''}${records.map((record) => record.text).join('')}`;
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
function resetControlPlaneTail(outputState, { prefixOmitted = false } = {}) {
|
|
1376
|
+
outputState.records = [];
|
|
1377
|
+
outputState.head = 0;
|
|
1378
|
+
outputState.byteLength = 0;
|
|
1379
|
+
outputState.omittedBytes = 0;
|
|
1380
|
+
outputState.omittedRecords = 0;
|
|
1381
|
+
outputState.omittedDigest = createHash('sha256');
|
|
1382
|
+
outputState.prefixOmitted = prefixOmitted;
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
function publishLiveControlPlaneRecord(outputState, record, publish) {
|
|
1386
|
+
if (
|
|
1387
|
+
outputState.liveSuppressed ||
|
|
1388
|
+
outputState.liveBytes + record.byteLength > outputState.liveByteLimit ||
|
|
1389
|
+
outputState.liveRecords + 1 > outputState.liveRecordLimit
|
|
1390
|
+
) {
|
|
1391
|
+
outputState.liveSuppressed = true;
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
publish(record);
|
|
1395
|
+
outputState.liveBytes += record.byteLength;
|
|
1396
|
+
outputState.liveRecords++;
|
|
1397
|
+
outputState.lastLiveSequence = record.sequence;
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
function flushTerminalControlPlaneOutput(outputState, publish) {
|
|
1401
|
+
if (outputState.terminalFlushed) return;
|
|
1402
|
+
outputState.terminalFlushed = true;
|
|
1403
|
+
if (outputState.liveSuppressed) {
|
|
1404
|
+
const omission = controlPlaneOmissionRecord(outputState);
|
|
1405
|
+
if (omission) publish(omission);
|
|
1406
|
+
}
|
|
1407
|
+
for (const record of retainedControlPlaneRecords(outputState)) {
|
|
1408
|
+
if (record.sequence > outputState.lastLiveSequence) publish(record);
|
|
1409
|
+
}
|
|
1236
1410
|
}
|
|
1237
1411
|
|
|
1238
1412
|
function createLogRecordBuffer() {
|
|
@@ -1289,31 +1463,16 @@ function isValidJsonLine(content) {
|
|
|
1289
1463
|
}
|
|
1290
1464
|
}
|
|
1291
1465
|
|
|
1292
|
-
function
|
|
1293
|
-
if (!line.trim()) return;
|
|
1294
|
-
|
|
1295
|
-
const { timestamp, content } = parseTimestampedLine(line);
|
|
1296
|
-
if (shouldSkipLogLine(content)) {
|
|
1297
|
-
return;
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
const isValidJson = isValidJsonLine(content);
|
|
1301
|
-
state.output += content + '\n';
|
|
1302
|
-
|
|
1303
|
-
if (!state.nested) {
|
|
1304
|
-
agent.lastOutputTime = Date.now();
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1466
|
+
function publishAgentOutputRecord(agent, providerName, record) {
|
|
1307
1467
|
agent._publish({
|
|
1308
1468
|
topic: 'AGENT_OUTPUT',
|
|
1309
1469
|
receiver: 'broadcast',
|
|
1310
1470
|
metadata: buildRawLogOnlyMetadata(),
|
|
1311
|
-
timestamp,
|
|
1471
|
+
timestamp: record.timestamp,
|
|
1312
1472
|
content: {
|
|
1313
|
-
text: content,
|
|
1314
1473
|
data: {
|
|
1315
|
-
type:
|
|
1316
|
-
line: content,
|
|
1474
|
+
type: record.type,
|
|
1475
|
+
line: record.content,
|
|
1317
1476
|
agent: agent.id,
|
|
1318
1477
|
role: agent.role,
|
|
1319
1478
|
iteration: agent.iteration,
|
|
@@ -1323,6 +1482,38 @@ function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
|
1323
1482
|
});
|
|
1324
1483
|
}
|
|
1325
1484
|
|
|
1485
|
+
function broadcastAgentLine({ agent, providerName, state, line }) {
|
|
1486
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
1487
|
+
if (!line.trim()) return;
|
|
1488
|
+
|
|
1489
|
+
const { timestamp, content } = parseTimestampedLine(line);
|
|
1490
|
+
if (shouldSkipLogLine(content)) {
|
|
1491
|
+
return;
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
const isValidJson = isValidJsonLine(content);
|
|
1495
|
+
const record = appendControlPlaneRecord(followerState.controlPlaneOutput, {
|
|
1496
|
+
content,
|
|
1497
|
+
timestamp,
|
|
1498
|
+
type: isValidJson ? 'json' : 'text',
|
|
1499
|
+
});
|
|
1500
|
+
|
|
1501
|
+
if (!followerState.nested) {
|
|
1502
|
+
agent.lastOutputTime = Date.now();
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
|
|
1506
|
+
publishAgentOutputRecord(agent, providerName, item)
|
|
1507
|
+
);
|
|
1508
|
+
}
|
|
1509
|
+
|
|
1510
|
+
function flushAgentOutput(agent, providerName, state) {
|
|
1511
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
1512
|
+
flushTerminalControlPlaneOutput(followerState.controlPlaneOutput, (record) =>
|
|
1513
|
+
publishAgentOutputRecord(agent, providerName, record)
|
|
1514
|
+
);
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1326
1517
|
function appendLogRecordFragment(buffer, fragment) {
|
|
1327
1518
|
const fragmentBytes = Buffer.byteLength(fragment);
|
|
1328
1519
|
buffer.byteLength += fragmentBytes;
|
|
@@ -1616,7 +1807,29 @@ function finalizeLogFollow(agent, state) {
|
|
|
1616
1807
|
}
|
|
1617
1808
|
}
|
|
1618
1809
|
|
|
1619
|
-
function
|
|
1810
|
+
function settleHostStatusFailure({ agent, providerName, state, resolve, text, data, error }) {
|
|
1811
|
+
if (state.resolved) return;
|
|
1812
|
+
state.resolved = true;
|
|
1813
|
+
finalizeLogFollow(agent, state);
|
|
1814
|
+
flushAgentOutput(agent, providerName, state);
|
|
1815
|
+
agent._publish({
|
|
1816
|
+
topic: 'AGENT_ERROR',
|
|
1817
|
+
receiver: 'broadcast',
|
|
1818
|
+
content: { text, data },
|
|
1819
|
+
});
|
|
1820
|
+
resolve({ success: false, output: state.output, error });
|
|
1821
|
+
}
|
|
1822
|
+
|
|
1823
|
+
function handleStatusExecError({
|
|
1824
|
+
agent,
|
|
1825
|
+
providerName,
|
|
1826
|
+
state,
|
|
1827
|
+
ctPath,
|
|
1828
|
+
taskId,
|
|
1829
|
+
error,
|
|
1830
|
+
stderr,
|
|
1831
|
+
resolve,
|
|
1832
|
+
}) {
|
|
1620
1833
|
if (!error) {
|
|
1621
1834
|
return false;
|
|
1622
1835
|
}
|
|
@@ -1641,30 +1854,20 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
|
|
|
1641
1854
|
`[Agent ${agent.id}] ⚠️ Task ${taskId} not found - will restart to ensure completion`
|
|
1642
1855
|
);
|
|
1643
1856
|
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
},
|
|
1659
|
-
},
|
|
1660
|
-
});
|
|
1661
|
-
|
|
1662
|
-
resolve({
|
|
1663
|
-
success: false,
|
|
1664
|
-
output: state.output,
|
|
1665
|
-
error: `Task not found - restarting for safety`,
|
|
1666
|
-
});
|
|
1667
|
-
}
|
|
1857
|
+
settleHostStatusFailure({
|
|
1858
|
+
agent,
|
|
1859
|
+
providerName,
|
|
1860
|
+
state,
|
|
1861
|
+
resolve,
|
|
1862
|
+
text: `Task ${taskId} not found - restarting for safety`,
|
|
1863
|
+
data: {
|
|
1864
|
+
taskId,
|
|
1865
|
+
error: 'task_not_found',
|
|
1866
|
+
role: agent.role,
|
|
1867
|
+
iteration: agent.iteration,
|
|
1868
|
+
},
|
|
1869
|
+
error: 'Task not found - restarting for safety',
|
|
1870
|
+
});
|
|
1668
1871
|
|
|
1669
1872
|
return true;
|
|
1670
1873
|
}
|
|
@@ -1682,31 +1885,21 @@ function handleStatusExecError({ agent, state, ctPath, taskId, error, stderr, re
|
|
|
1682
1885
|
console.error(` Stderr: ${stderr || 'none'}`);
|
|
1683
1886
|
console.error(` This may indicate zeroshot is not in PATH or task storage is corrupted.`);
|
|
1684
1887
|
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
},
|
|
1701
|
-
},
|
|
1702
|
-
});
|
|
1703
|
-
|
|
1704
|
-
resolve({
|
|
1705
|
-
success: false,
|
|
1706
|
-
output: state.output,
|
|
1707
|
-
error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
|
|
1708
|
-
});
|
|
1709
|
-
}
|
|
1888
|
+
settleHostStatusFailure({
|
|
1889
|
+
agent,
|
|
1890
|
+
providerName,
|
|
1891
|
+
state,
|
|
1892
|
+
resolve,
|
|
1893
|
+
text: `Task ${taskId} polling failed after ${MAX_STATUS_FAILURES} consecutive failures`,
|
|
1894
|
+
data: {
|
|
1895
|
+
taskId,
|
|
1896
|
+
error: 'polling_timeout',
|
|
1897
|
+
attempts: state.consecutiveExecFailures,
|
|
1898
|
+
role: agent.role,
|
|
1899
|
+
iteration: agent.iteration,
|
|
1900
|
+
},
|
|
1901
|
+
error: `Status polling failed ${MAX_STATUS_FAILURES} times - task may not exist`,
|
|
1902
|
+
});
|
|
1710
1903
|
|
|
1711
1904
|
return true;
|
|
1712
1905
|
}
|
|
@@ -1761,6 +1954,7 @@ function handleStatusCompletion({
|
|
|
1761
1954
|
state.resolved = true;
|
|
1762
1955
|
|
|
1763
1956
|
finalizeLogFollow(agent, state);
|
|
1957
|
+
flushAgentOutput(agent, providerName, state);
|
|
1764
1958
|
|
|
1765
1959
|
buildCompletionResult({
|
|
1766
1960
|
agent,
|
|
@@ -1783,6 +1977,7 @@ function buildKillHandler({ agent, taskId, state, providerName, resolve }) {
|
|
|
1783
1977
|
if (state.resolved) return;
|
|
1784
1978
|
state.resolved = true;
|
|
1785
1979
|
finalizeLogFollow(agent, state);
|
|
1980
|
+
flushAgentOutput(agent, providerName, state);
|
|
1786
1981
|
if (!state.nested) {
|
|
1787
1982
|
agent._stopLivenessCheck();
|
|
1788
1983
|
}
|
|
@@ -1842,7 +2037,18 @@ function createLogFollower({
|
|
|
1842
2037
|
(error, stdout, stderr) => {
|
|
1843
2038
|
if (state.resolved) return;
|
|
1844
2039
|
|
|
1845
|
-
if (
|
|
2040
|
+
if (
|
|
2041
|
+
handleStatusExecError({
|
|
2042
|
+
agent,
|
|
2043
|
+
providerName,
|
|
2044
|
+
state,
|
|
2045
|
+
ctPath,
|
|
2046
|
+
taskId,
|
|
2047
|
+
error,
|
|
2048
|
+
stderr,
|
|
2049
|
+
resolve,
|
|
2050
|
+
})
|
|
2051
|
+
) {
|
|
1846
2052
|
return;
|
|
1847
2053
|
}
|
|
1848
2054
|
|
|
@@ -2283,7 +2489,7 @@ async function spawnClaudeTaskIsolatedExecution(agent, context, options = {}) {
|
|
|
2283
2489
|
* - Result: 10-20% overall latency reduction
|
|
2284
2490
|
*/
|
|
2285
2491
|
function createIsolatedLogState(skipStructuredResultCheck = false, nested = false) {
|
|
2286
|
-
|
|
2492
|
+
const state = {
|
|
2287
2493
|
taskExited: false,
|
|
2288
2494
|
resolved: false,
|
|
2289
2495
|
terminationPromise: null,
|
|
@@ -2291,8 +2497,10 @@ function createIsolatedLogState(skipStructuredResultCheck = false, nested = fals
|
|
|
2291
2497
|
durableTaskStatus: null,
|
|
2292
2498
|
lifecycleHandle: null,
|
|
2293
2499
|
logFilePath: null,
|
|
2294
|
-
|
|
2500
|
+
controlPlaneOutput: createControlPlaneOutputState(),
|
|
2295
2501
|
tailProcess: null,
|
|
2502
|
+
rawBytesSeen: 0,
|
|
2503
|
+
ignoreTailOutput: false,
|
|
2296
2504
|
statusCheckInterval: null,
|
|
2297
2505
|
timeoutTimer: null,
|
|
2298
2506
|
lineBuffer: createLogRecordBuffer(),
|
|
@@ -2300,6 +2508,9 @@ function createIsolatedLogState(skipStructuredResultCheck = false, nested = fals
|
|
|
2300
2508
|
skipStructuredResultCheck,
|
|
2301
2509
|
nested,
|
|
2302
2510
|
};
|
|
2511
|
+
defineControlPlaneOutputAccessor(state, 'output');
|
|
2512
|
+
defineControlPlaneOutputAccessor(state, 'fullOutput');
|
|
2513
|
+
return state;
|
|
2303
2514
|
}
|
|
2304
2515
|
|
|
2305
2516
|
function buildIsolatedCleanup(state) {
|
|
@@ -2422,6 +2633,41 @@ async function resolveIsolatedLogFilePath(manager, clusterId, taskId, state) {
|
|
|
2422
2633
|
return state.logFilePath;
|
|
2423
2634
|
}
|
|
2424
2635
|
|
|
2636
|
+
async function captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state) {
|
|
2637
|
+
stopIsolatedTailForSettlement(state);
|
|
2638
|
+
const sizeResult = await manager.execInContainer(clusterId, [
|
|
2639
|
+
'sh',
|
|
2640
|
+
'-c',
|
|
2641
|
+
`wc -c < "${logFilePath}" 2>/dev/null || echo 0`,
|
|
2642
|
+
]);
|
|
2643
|
+
const fileSize = Number.parseInt(sizeResult.stdout.trim(), 10);
|
|
2644
|
+
const missingBytes = Number.isSafeInteger(fileSize)
|
|
2645
|
+
? Math.max(0, fileSize - state.rawBytesSeen)
|
|
2646
|
+
: 0;
|
|
2647
|
+
|
|
2648
|
+
if (missingBytes > 0) {
|
|
2649
|
+
const readBytes = Math.min(missingBytes, MAX_CONTROL_PLANE_OUTPUT_BYTES);
|
|
2650
|
+
const finalReadResult = await manager.execInContainer(clusterId, [
|
|
2651
|
+
'sh',
|
|
2652
|
+
'-c',
|
|
2653
|
+
`tail -c ${readBytes} "${logFilePath}" 2>/dev/null || echo ""`,
|
|
2654
|
+
]);
|
|
2655
|
+
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
2656
|
+
if (missingBytes > readBytes) {
|
|
2657
|
+
resetControlPlaneTail(state.controlPlaneOutput, { prefixOmitted: true });
|
|
2658
|
+
state.lineBuffer = createLogRecordBuffer();
|
|
2659
|
+
}
|
|
2660
|
+
appendIsolatedContent(state, finalReadResult.stdout, (line) =>
|
|
2661
|
+
retainIsolatedLine(state, line)
|
|
2662
|
+
);
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
if (state.lineBuffer.byteLength > 0) {
|
|
2667
|
+
completeLogRecord(state.lineBuffer, (line) => retainIsolatedLine(state, line), true);
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2425
2671
|
function settleIsolatedTerminalStatus({
|
|
2426
2672
|
agent,
|
|
2427
2673
|
manager,
|
|
@@ -2434,7 +2680,6 @@ function settleIsolatedTerminalStatus({
|
|
|
2434
2680
|
cleanup,
|
|
2435
2681
|
resolve,
|
|
2436
2682
|
reject,
|
|
2437
|
-
onLine,
|
|
2438
2683
|
}) {
|
|
2439
2684
|
if (state.resolved) return Promise.resolve();
|
|
2440
2685
|
if (state.terminalSettlementPromise) return state.terminalSettlementPromise;
|
|
@@ -2448,17 +2693,9 @@ function settleIsolatedTerminalStatus({
|
|
|
2448
2693
|
const logFilePath = await resolveIsolatedLogFilePath(manager, clusterId, taskId, state);
|
|
2449
2694
|
await new Promise((settle) => setTimeout(settle, 200));
|
|
2450
2695
|
if (state.resolved) return;
|
|
2451
|
-
|
|
2452
|
-
'sh',
|
|
2453
|
-
'-c',
|
|
2454
|
-
`cat "${logFilePath}" 2>/dev/null || echo ""`,
|
|
2455
|
-
]);
|
|
2696
|
+
await captureIsolatedFinalOutputTail(manager, clusterId, logFilePath, state);
|
|
2456
2697
|
if (state.resolved) return;
|
|
2457
|
-
|
|
2458
|
-
if (finalReadResult.code === 0 && finalReadResult.stdout) {
|
|
2459
|
-
state.fullOutput = finalReadResult.stdout;
|
|
2460
|
-
replayCompleteLogContent(state.fullOutput, onLine);
|
|
2461
|
-
}
|
|
2698
|
+
flushIsolatedOutput(agent, providerName, taskId, state);
|
|
2462
2699
|
|
|
2463
2700
|
const vertexModelError =
|
|
2464
2701
|
providerName === 'claude'
|
|
@@ -2542,9 +2779,9 @@ function buildIsolatedLifecycleHandle({
|
|
|
2542
2779
|
cleanup,
|
|
2543
2780
|
resolve,
|
|
2544
2781
|
reject,
|
|
2545
|
-
onLine,
|
|
2546
2782
|
}) {
|
|
2547
|
-
const settleCancellation = (reason, details) =>
|
|
2783
|
+
const settleCancellation = (reason, details) => {
|
|
2784
|
+
flushIsolatedOutput(agent, providerName, taskId, state);
|
|
2548
2785
|
settleIsolatedFollower({
|
|
2549
2786
|
agent,
|
|
2550
2787
|
state,
|
|
@@ -2559,6 +2796,7 @@ function buildIsolatedLifecycleHandle({
|
|
|
2559
2796
|
tokenUsage: extractTokenUsage(state.fullOutput, providerName),
|
|
2560
2797
|
},
|
|
2561
2798
|
});
|
|
2799
|
+
};
|
|
2562
2800
|
const terminate = (reason = 'Task killed', details = {}) => {
|
|
2563
2801
|
if (state.durableTaskTerminal) {
|
|
2564
2802
|
if (state.nested) settleCancellation(reason, details);
|
|
@@ -2590,7 +2828,6 @@ function buildIsolatedLifecycleHandle({
|
|
|
2590
2828
|
cleanup,
|
|
2591
2829
|
resolve,
|
|
2592
2830
|
reject,
|
|
2593
|
-
onLine,
|
|
2594
2831
|
});
|
|
2595
2832
|
return termination;
|
|
2596
2833
|
}
|
|
@@ -2618,11 +2855,14 @@ function buildIsolatedLifecycleHandle({
|
|
|
2618
2855
|
};
|
|
2619
2856
|
}
|
|
2620
2857
|
|
|
2621
|
-
function
|
|
2858
|
+
function parseIsolatedLogLine(line) {
|
|
2622
2859
|
const timestampMatch = line.match(/^\[(\d{4}-\d{2}-\d{2}T[^\]]+)\]\s*(.*)$/);
|
|
2623
2860
|
const timestamp = timestampMatch ? new Date(timestampMatch[1]).getTime() : Date.now();
|
|
2624
2861
|
const content = timestampMatch ? timestampMatch[2] : line;
|
|
2862
|
+
return { timestamp, content };
|
|
2863
|
+
}
|
|
2625
2864
|
|
|
2865
|
+
function publishIsolatedOutputRecord(agent, providerName, taskId, record) {
|
|
2626
2866
|
agent.messageBus.publish({
|
|
2627
2867
|
cluster_id: agent.cluster.id,
|
|
2628
2868
|
topic: 'AGENT_OUTPUT',
|
|
@@ -2630,31 +2870,66 @@ function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
|
|
|
2630
2870
|
metadata: buildRawLogOnlyMetadata(),
|
|
2631
2871
|
content: {
|
|
2632
2872
|
data: {
|
|
2633
|
-
line: content,
|
|
2873
|
+
line: record.content,
|
|
2634
2874
|
taskId,
|
|
2635
2875
|
iteration: agent.iteration,
|
|
2636
2876
|
provider: providerName,
|
|
2637
2877
|
},
|
|
2638
2878
|
},
|
|
2879
|
+
timestamp: record.timestamp,
|
|
2880
|
+
});
|
|
2881
|
+
}
|
|
2882
|
+
|
|
2883
|
+
function retainIsolatedLine(state, line) {
|
|
2884
|
+
const { timestamp, content } = parseIsolatedLogLine(line);
|
|
2885
|
+
return appendControlPlaneRecord(state.controlPlaneOutput, {
|
|
2886
|
+
content,
|
|
2639
2887
|
timestamp,
|
|
2888
|
+
type: isValidJsonLine(content) ? 'json' : 'text',
|
|
2640
2889
|
});
|
|
2890
|
+
}
|
|
2891
|
+
|
|
2892
|
+
function broadcastIsolatedLine({ agent, providerName, taskId, state, line }) {
|
|
2893
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
2894
|
+
const record = retainIsolatedLine(followerState, line);
|
|
2895
|
+
publishLiveControlPlaneRecord(followerState.controlPlaneOutput, record, (item) =>
|
|
2896
|
+
publishIsolatedOutputRecord(agent, providerName, taskId, item)
|
|
2897
|
+
);
|
|
2641
2898
|
|
|
2642
|
-
if (!
|
|
2899
|
+
if (!followerState.nested) {
|
|
2643
2900
|
agent.lastOutputTime = Date.now();
|
|
2644
2901
|
}
|
|
2645
2902
|
}
|
|
2646
2903
|
|
|
2904
|
+
function flushIsolatedOutput(agent, providerName, taskId, state) {
|
|
2905
|
+
const followerState = ensureControlPlaneOutputState(state);
|
|
2906
|
+
flushTerminalControlPlaneOutput(followerState.controlPlaneOutput, (record) =>
|
|
2907
|
+
publishIsolatedOutputRecord(agent, providerName, taskId, record)
|
|
2908
|
+
);
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2647
2911
|
function appendIsolatedContent(state, content, onLine) {
|
|
2648
2912
|
appendContentToBuffer(state, content, onLine, true);
|
|
2649
2913
|
}
|
|
2650
2914
|
|
|
2651
2915
|
function consumeIsolatedTailChunk(state, data, onLine) {
|
|
2652
2916
|
const chunk = typeof data === 'string' ? data : state.tailDecoder.write(data);
|
|
2653
|
-
if (!chunk) return;
|
|
2654
|
-
state.
|
|
2917
|
+
if (!chunk || state.ignoreTailOutput) return;
|
|
2918
|
+
state.rawBytesSeen += Buffer.byteLength(chunk);
|
|
2655
2919
|
appendIsolatedContent(state, chunk, onLine);
|
|
2656
2920
|
}
|
|
2657
2921
|
|
|
2922
|
+
function stopIsolatedTailForSettlement(state) {
|
|
2923
|
+
state.ignoreTailOutput = true;
|
|
2924
|
+
if (!state.tailProcess) return;
|
|
2925
|
+
try {
|
|
2926
|
+
state.tailProcess.kill('SIGTERM');
|
|
2927
|
+
} catch {
|
|
2928
|
+
// Ignore - process may already be dead.
|
|
2929
|
+
}
|
|
2930
|
+
state.tailProcess = null;
|
|
2931
|
+
}
|
|
2932
|
+
|
|
2658
2933
|
function startIsolatedTail({ agent, manager, clusterId, logFilePath, state, onLine }) {
|
|
2659
2934
|
state.tailProcess = manager.spawnInContainer(clusterId, [
|
|
2660
2935
|
'sh',
|
|
@@ -2696,7 +2971,6 @@ async function checkIsolatedStatus({
|
|
|
2696
2971
|
cleanup,
|
|
2697
2972
|
resolve,
|
|
2698
2973
|
reject,
|
|
2699
|
-
onLine,
|
|
2700
2974
|
}) {
|
|
2701
2975
|
if (state.taskExited) return;
|
|
2702
2976
|
|
|
@@ -2749,7 +3023,6 @@ async function checkIsolatedStatus({
|
|
|
2749
3023
|
cleanup,
|
|
2750
3024
|
resolve,
|
|
2751
3025
|
reject,
|
|
2752
|
-
onLine,
|
|
2753
3026
|
});
|
|
2754
3027
|
}
|
|
2755
3028
|
|
|
@@ -2764,7 +3037,6 @@ function startIsolatedStatusChecks({
|
|
|
2764
3037
|
cleanup,
|
|
2765
3038
|
resolve,
|
|
2766
3039
|
reject,
|
|
2767
|
-
onLine,
|
|
2768
3040
|
}) {
|
|
2769
3041
|
state.statusCheckInterval = setInterval(() => {
|
|
2770
3042
|
checkIsolatedStatus({
|
|
@@ -2778,7 +3050,6 @@ function startIsolatedStatusChecks({
|
|
|
2778
3050
|
cleanup,
|
|
2779
3051
|
resolve,
|
|
2780
3052
|
reject,
|
|
2781
|
-
onLine,
|
|
2782
3053
|
}).catch((statusErr) => {
|
|
2783
3054
|
agent._log(`[${agent.id}] Status check error (will retry): ${statusErr.message}`);
|
|
2784
3055
|
});
|
|
@@ -2812,7 +3083,6 @@ function followClaudeTaskLogsIsolated(agent, taskId, options = {}) {
|
|
|
2812
3083
|
cleanup,
|
|
2813
3084
|
resolve,
|
|
2814
3085
|
reject,
|
|
2815
|
-
onLine,
|
|
2816
3086
|
});
|
|
2817
3087
|
// Only register the lifecycle handle on the agent for top-level tasks.
|
|
2818
3088
|
if (!options.nested) {
|
|
@@ -2883,7 +3153,6 @@ function followClaudeTaskLogsIsolated(agent, taskId, options = {}) {
|
|
|
2883
3153
|
cleanup,
|
|
2884
3154
|
resolve,
|
|
2885
3155
|
reject,
|
|
2886
|
-
onLine,
|
|
2887
3156
|
});
|
|
2888
3157
|
|
|
2889
3158
|
if (agent.timeout > 0 && !agent.enableLivenessCheck && !options.nested) {
|
|
@@ -3167,7 +3436,19 @@ module.exports = {
|
|
|
3167
3436
|
buildTaskRunArgs,
|
|
3168
3437
|
rebuildProviderSessionAfterCommit,
|
|
3169
3438
|
killTask,
|
|
3439
|
+
createLogFollowState,
|
|
3440
|
+
createIsolatedLogState,
|
|
3441
|
+
createControlPlaneOutputState,
|
|
3442
|
+
appendControlPlaneRecord,
|
|
3170
3443
|
createLogRecordBuffer,
|
|
3171
3444
|
appendContentToBuffer,
|
|
3172
3445
|
consumeIsolatedTailChunk,
|
|
3446
|
+
flushAgentOutput,
|
|
3447
|
+
flushIsolatedOutput,
|
|
3448
|
+
CONTROL_PLANE_OUTPUT_LIMITS: Object.freeze({
|
|
3449
|
+
maxBytes: MAX_CONTROL_PLANE_OUTPUT_BYTES,
|
|
3450
|
+
maxRecords: MAX_CONTROL_PLANE_OUTPUT_RECORDS,
|
|
3451
|
+
liveBytes: MAX_LIVE_OUTPUT_BYTES,
|
|
3452
|
+
liveRecords: MAX_LIVE_OUTPUT_RECORDS,
|
|
3453
|
+
}),
|
|
3173
3454
|
};
|