@hasna/loops 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +106 -3
- package/README.md +70 -17
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +750 -10
- package/dist/cli/index.js +1744 -361
- package/dist/cli/ui.d.ts +44 -0
- package/dist/daemon/index.js +948 -212
- package/dist/generated/storage-kit/health.d.ts +19 -0
- package/dist/generated/storage-kit/index.d.ts +7 -0
- package/dist/generated/storage-kit/migrations.d.ts +47 -0
- package/dist/generated/storage-kit/mode.d.ts +47 -0
- package/dist/generated/storage-kit/pool.d.ts +33 -0
- package/dist/generated/storage-kit/query.d.ts +35 -0
- package/dist/generated/storage-kit/tls.d.ts +25 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +6513 -4840
- package/dist/lib/health.d.ts +83 -3
- package/dist/lib/mode.d.ts +27 -0
- package/dist/lib/mode.js +69 -2
- package/dist/lib/scheduler.d.ts +5 -2
- package/dist/lib/storage/index.d.ts +1 -0
- package/dist/lib/storage/index.js +1329 -211
- package/dist/lib/storage/pg-executor.d.ts +27 -0
- package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
- package/dist/lib/storage/postgres-loop-storage.d.ts +114 -0
- package/dist/lib/storage/sqlite.js +336 -8
- package/dist/lib/store.d.ts +234 -0
- package/dist/lib/store.js +350 -8
- package/dist/mcp/index.js +1447 -479
- package/dist/runner/index.js +179 -25
- package/dist/sdk/http.d.ts +115 -0
- package/dist/sdk/http.js +145 -0
- package/dist/sdk/index.d.ts +5 -1
- package/dist/sdk/index.js +1106 -296
- package/dist/serve/index.d.ts +4 -0
- package/dist/serve/index.js +7619 -0
- package/dist/types.d.ts +3 -0
- package/docs/CUTOVER-RUNBOOK.md +61 -0
- package/docs/DEPLOYMENT_MODES.md +23 -1
- package/docs/USAGE.md +66 -16
- package/package.json +14 -2
package/dist/sdk/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/lib/store.ts
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as
|
|
5
|
-
import { tmpdir } from "os";
|
|
6
|
-
import { basename as basename2, dirname as dirname2, join as
|
|
4
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
|
|
5
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
6
|
+
import { basename as basename2, dirname as dirname2, join as join4 } from "path";
|
|
7
7
|
|
|
8
8
|
// src/lib/errors.ts
|
|
9
9
|
class CodedError extends Error {
|
|
@@ -1202,6 +1202,196 @@ function discardWorkflowRunManifest(staged) {
|
|
|
1202
1202
|
} catch {}
|
|
1203
1203
|
}
|
|
1204
1204
|
|
|
1205
|
+
// src/lib/route/todos-cli.ts
|
|
1206
|
+
import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
|
|
1207
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1208
|
+
import { join as join3 } from "path";
|
|
1209
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
1210
|
+
|
|
1211
|
+
// src/lib/format.ts
|
|
1212
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1213
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
1214
|
+
function redact(value, visible = 0) {
|
|
1215
|
+
if (!value)
|
|
1216
|
+
return value;
|
|
1217
|
+
if (value.length <= visible)
|
|
1218
|
+
return value;
|
|
1219
|
+
if (visible <= 0)
|
|
1220
|
+
return `[redacted ${value.length} chars]`;
|
|
1221
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1222
|
+
}
|
|
1223
|
+
function truncateTextOutput(value) {
|
|
1224
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1225
|
+
return value;
|
|
1226
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1227
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1228
|
+
}
|
|
1229
|
+
function redactSensitivePayload(value, key) {
|
|
1230
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
1231
|
+
if (typeof value === "string")
|
|
1232
|
+
return redact(value);
|
|
1233
|
+
if (value === undefined || value === null)
|
|
1234
|
+
return value;
|
|
1235
|
+
return "[redacted]";
|
|
1236
|
+
}
|
|
1237
|
+
if (Array.isArray(value))
|
|
1238
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
1239
|
+
if (value && typeof value === "object") {
|
|
1240
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
1241
|
+
}
|
|
1242
|
+
return value;
|
|
1243
|
+
}
|
|
1244
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1245
|
+
const indent = opts.indent ?? "";
|
|
1246
|
+
const nested = `${indent} `;
|
|
1247
|
+
const blocks = [];
|
|
1248
|
+
for (const [label, output] of [
|
|
1249
|
+
["stdout", value.stdout],
|
|
1250
|
+
["stderr", value.stderr]
|
|
1251
|
+
]) {
|
|
1252
|
+
if (!output)
|
|
1253
|
+
continue;
|
|
1254
|
+
blocks.push(`${indent}${label}:`);
|
|
1255
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1256
|
+
blocks.push(`${nested}${line}`);
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
return blocks;
|
|
1260
|
+
}
|
|
1261
|
+
function publicLoop(loop) {
|
|
1262
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1263
|
+
return {
|
|
1264
|
+
...loop,
|
|
1265
|
+
target
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
function publicRun(run, showOutput = false, opts = {}) {
|
|
1269
|
+
return {
|
|
1270
|
+
...run,
|
|
1271
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1272
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1273
|
+
error: opts.redactError ? redact(run.error) : run.error
|
|
1274
|
+
};
|
|
1275
|
+
}
|
|
1276
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
1277
|
+
return {
|
|
1278
|
+
...result,
|
|
1279
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
1280
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
1281
|
+
error: redact(result.error)
|
|
1282
|
+
};
|
|
1283
|
+
}
|
|
1284
|
+
function publicWorkflow(workflow) {
|
|
1285
|
+
return {
|
|
1286
|
+
...workflow,
|
|
1287
|
+
steps: workflow.steps.map((step) => ({
|
|
1288
|
+
...step,
|
|
1289
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
1290
|
+
}))
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
function publicWorkflowRun(run) {
|
|
1294
|
+
return { ...run, error: redact(run.error) };
|
|
1295
|
+
}
|
|
1296
|
+
function publicWorkflowInvocation(invocation) {
|
|
1297
|
+
return redactSensitivePayload(invocation);
|
|
1298
|
+
}
|
|
1299
|
+
function publicWorkflowWorkItem(item) {
|
|
1300
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
1301
|
+
}
|
|
1302
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
1303
|
+
return {
|
|
1304
|
+
...run,
|
|
1305
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1306
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1307
|
+
error: redact(run.error)
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
function publicWorkflowEvent(event) {
|
|
1311
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
1312
|
+
}
|
|
1313
|
+
function publicGoal(goal) {
|
|
1314
|
+
return {
|
|
1315
|
+
...goal,
|
|
1316
|
+
objective: redact(goal.objective, 120)
|
|
1317
|
+
};
|
|
1318
|
+
}
|
|
1319
|
+
function publicGoalRun(run) {
|
|
1320
|
+
return {
|
|
1321
|
+
...run,
|
|
1322
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
1323
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
1324
|
+
};
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
// src/lib/route/todos-cli.ts
|
|
1328
|
+
function defaultLoopsProject() {
|
|
1329
|
+
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
|
|
1330
|
+
}
|
|
1331
|
+
function runLocalCommand(command, args, opts = {}) {
|
|
1332
|
+
const result = spawnSync2(command, args, {
|
|
1333
|
+
input: opts.input,
|
|
1334
|
+
encoding: "utf8",
|
|
1335
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1336
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1337
|
+
env: process.env
|
|
1338
|
+
});
|
|
1339
|
+
return {
|
|
1340
|
+
ok: result.status === 0,
|
|
1341
|
+
status: result.status,
|
|
1342
|
+
stdout: result.stdout || "",
|
|
1343
|
+
stderr: result.stderr || "",
|
|
1344
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1345
|
+
};
|
|
1346
|
+
}
|
|
1347
|
+
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
1348
|
+
const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
|
|
1349
|
+
const stdoutPath = join3(tempDir, "stdout");
|
|
1350
|
+
const stdoutFd = openSync(stdoutPath, "w");
|
|
1351
|
+
let result;
|
|
1352
|
+
try {
|
|
1353
|
+
result = spawnSync2(command, args, {
|
|
1354
|
+
input: opts.input,
|
|
1355
|
+
encoding: "utf8",
|
|
1356
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1357
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1358
|
+
env: process.env,
|
|
1359
|
+
stdio: ["pipe", stdoutFd, "pipe"]
|
|
1360
|
+
});
|
|
1361
|
+
} finally {
|
|
1362
|
+
closeSync(stdoutFd);
|
|
1363
|
+
}
|
|
1364
|
+
try {
|
|
1365
|
+
return {
|
|
1366
|
+
ok: result.status === 0,
|
|
1367
|
+
status: result.status,
|
|
1368
|
+
stdout: readFileSync3(stdoutPath, "utf8"),
|
|
1369
|
+
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
1370
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1371
|
+
};
|
|
1372
|
+
} finally {
|
|
1373
|
+
rmSync2(tempDir, { recursive: true, force: true });
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
function ensureTodosTaskList(project, slug, name, description) {
|
|
1377
|
+
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
1378
|
+
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
1379
|
+
if (!list.ok)
|
|
1380
|
+
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
1381
|
+
const values = JSON.parse(list.stdout || "[]");
|
|
1382
|
+
const found = values.find((entry) => entry.slug === slug);
|
|
1383
|
+
if (!found)
|
|
1384
|
+
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
1385
|
+
return found.id;
|
|
1386
|
+
}
|
|
1387
|
+
function todosMutationSummary(result) {
|
|
1388
|
+
return {
|
|
1389
|
+
ok: result.ok,
|
|
1390
|
+
status: result.status,
|
|
1391
|
+
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
1392
|
+
};
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1205
1395
|
// src/lib/store.ts
|
|
1206
1396
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
1207
1397
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
@@ -1211,6 +1401,7 @@ var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "s
|
|
|
1211
1401
|
var PRUNE_BATCH_SIZE = 400;
|
|
1212
1402
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
1213
1403
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
1404
|
+
var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
|
|
1214
1405
|
function rowToLoop(row) {
|
|
1215
1406
|
return {
|
|
1216
1407
|
id: row.id,
|
|
@@ -1261,6 +1452,9 @@ function rowToRun(row) {
|
|
|
1261
1452
|
updatedAt: row.updated_at
|
|
1262
1453
|
};
|
|
1263
1454
|
}
|
|
1455
|
+
function latestRunTime(row) {
|
|
1456
|
+
return row.finished_at ?? row.started_at ?? row.created_at;
|
|
1457
|
+
}
|
|
1264
1458
|
function rowToWorkflow(row) {
|
|
1265
1459
|
return {
|
|
1266
1460
|
id: row.id,
|
|
@@ -1515,7 +1709,7 @@ class Store {
|
|
|
1515
1709
|
const file = path ?? dbPath();
|
|
1516
1710
|
if (file !== ":memory:")
|
|
1517
1711
|
ensurePrivateStorePath(file);
|
|
1518
|
-
this.rootDir = file === ":memory:" ?
|
|
1712
|
+
this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
|
|
1519
1713
|
if (file === ":memory:")
|
|
1520
1714
|
this.memoryRootDir = this.rootDir;
|
|
1521
1715
|
this.db = new Database(file);
|
|
@@ -2013,7 +2207,32 @@ class Store {
|
|
|
2013
2207
|
} else {
|
|
2014
2208
|
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2015
2209
|
}
|
|
2016
|
-
return rows.map(rowToLoop);
|
|
2210
|
+
return this.withLatestRunSummaries(rows.map(rowToLoop));
|
|
2211
|
+
}
|
|
2212
|
+
withLatestRunSummaries(loops) {
|
|
2213
|
+
if (loops.length === 0)
|
|
2214
|
+
return loops;
|
|
2215
|
+
const placeholders = loops.map(() => "?").join(",");
|
|
2216
|
+
const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
|
|
2217
|
+
FROM (
|
|
2218
|
+
SELECT loop_id, id, status, started_at, finished_at, created_at,
|
|
2219
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
|
|
2220
|
+
FROM loop_runs
|
|
2221
|
+
WHERE loop_id IN (${placeholders})
|
|
2222
|
+
)
|
|
2223
|
+
WHERE rn = 1`).all(...loops.map((loop) => loop.id));
|
|
2224
|
+
const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
|
|
2225
|
+
return loops.map((loop) => {
|
|
2226
|
+
const latest = latestByLoopId.get(loop.id);
|
|
2227
|
+
if (!latest)
|
|
2228
|
+
return loop;
|
|
2229
|
+
return {
|
|
2230
|
+
...loop,
|
|
2231
|
+
latestRunId: latest.id,
|
|
2232
|
+
latestRunStatus: latest.status,
|
|
2233
|
+
lastRunAt: latestRunTime(latest)
|
|
2234
|
+
};
|
|
2235
|
+
});
|
|
2017
2236
|
}
|
|
2018
2237
|
dueLoops(now, limit = 500) {
|
|
2019
2238
|
const rows = this.db.query(`SELECT * FROM loops
|
|
@@ -2132,6 +2351,45 @@ class Store {
|
|
|
2132
2351
|
throw error;
|
|
2133
2352
|
}
|
|
2134
2353
|
}
|
|
2354
|
+
updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
|
|
2355
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
2356
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2357
|
+
try {
|
|
2358
|
+
const current = this.requireUniqueLoop(idOrName);
|
|
2359
|
+
if (current.archivedAt)
|
|
2360
|
+
throw new LoopArchivedError(current.name || current.id);
|
|
2361
|
+
if (current.target.type !== "agent")
|
|
2362
|
+
throw new Error(`loop is not an agent loop: ${idOrName}`);
|
|
2363
|
+
if (this.hasRunningRun(current.id))
|
|
2364
|
+
throw new Error(`refusing to update running loop: ${current.id}`);
|
|
2365
|
+
const target = { ...current.target, timeoutMs };
|
|
2366
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
2367
|
+
delete target.idleTimeoutMs;
|
|
2368
|
+
const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
|
|
2369
|
+
WHERE id=$id
|
|
2370
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2371
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2372
|
+
))`).run({
|
|
2373
|
+
$id: current.id,
|
|
2374
|
+
$target: JSON.stringify(target),
|
|
2375
|
+
$updated: updated,
|
|
2376
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2377
|
+
$now: updated
|
|
2378
|
+
});
|
|
2379
|
+
if (res.changes !== 1)
|
|
2380
|
+
throw new Error("daemon lease lost");
|
|
2381
|
+
this.db.exec("COMMIT");
|
|
2382
|
+
const after = this.getLoop(current.id);
|
|
2383
|
+
if (!after)
|
|
2384
|
+
throw new Error(`loop not found after timeout update: ${current.id}`);
|
|
2385
|
+
return after;
|
|
2386
|
+
} catch (error) {
|
|
2387
|
+
try {
|
|
2388
|
+
this.db.exec("ROLLBACK");
|
|
2389
|
+
} catch {}
|
|
2390
|
+
throw error;
|
|
2391
|
+
}
|
|
2392
|
+
}
|
|
2135
2393
|
createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
|
|
2136
2394
|
const normalized = normalizeCreateWorkflowInput(workflowInput);
|
|
2137
2395
|
const updated = (opts.now ?? new Date).toISOString();
|
|
@@ -2462,6 +2720,59 @@ class Store {
|
|
|
2462
2720
|
updated
|
|
2463
2721
|
});
|
|
2464
2722
|
}
|
|
2723
|
+
taskLifecycleTodosPointerContext(workflowRunId) {
|
|
2724
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
2725
|
+
if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
|
|
2726
|
+
return;
|
|
2727
|
+
const workItem = this.getWorkflowWorkItem(run.workItemId);
|
|
2728
|
+
if (!workItem || workItem.routeKey !== "todos-task")
|
|
2729
|
+
return;
|
|
2730
|
+
const invocation = this.getWorkflowInvocation(run.invocationId);
|
|
2731
|
+
if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
|
|
2732
|
+
return;
|
|
2733
|
+
const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
|
|
2734
|
+
const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
|
|
2735
|
+
if (!projectPath || !taskId)
|
|
2736
|
+
return;
|
|
2737
|
+
return {
|
|
2738
|
+
projectPath,
|
|
2739
|
+
taskId,
|
|
2740
|
+
invocationId: invocation.id,
|
|
2741
|
+
workflowRunId: run.id,
|
|
2742
|
+
manifestPath: run.manifestPath
|
|
2743
|
+
};
|
|
2744
|
+
}
|
|
2745
|
+
syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
|
|
2746
|
+
const context = this.taskLifecycleTodosPointerContext(workflowRunId);
|
|
2747
|
+
if (!context)
|
|
2748
|
+
return;
|
|
2749
|
+
const result = runLocalCommand("todos", [
|
|
2750
|
+
"--project",
|
|
2751
|
+
context.projectPath,
|
|
2752
|
+
"task",
|
|
2753
|
+
"workflow-pointers",
|
|
2754
|
+
context.taskId,
|
|
2755
|
+
"--clear",
|
|
2756
|
+
"--invocation",
|
|
2757
|
+
context.invocationId,
|
|
2758
|
+
"--run",
|
|
2759
|
+
context.workflowRunId,
|
|
2760
|
+
"--manifest",
|
|
2761
|
+
context.manifestPath,
|
|
2762
|
+
"--state",
|
|
2763
|
+
"succeeded",
|
|
2764
|
+
"--actor",
|
|
2765
|
+
"openloops:task-lifecycle"
|
|
2766
|
+
]);
|
|
2767
|
+
this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
|
|
2768
|
+
projectPath: context.projectPath,
|
|
2769
|
+
taskId: context.taskId,
|
|
2770
|
+
invocationId: context.invocationId,
|
|
2771
|
+
workflowRunId: context.workflowRunId,
|
|
2772
|
+
manifestPath: context.manifestPath,
|
|
2773
|
+
mutation: todosMutationSummary(result)
|
|
2774
|
+
});
|
|
2775
|
+
}
|
|
2465
2776
|
createWorkflowInvocation(input) {
|
|
2466
2777
|
const now = nowIso();
|
|
2467
2778
|
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
@@ -3424,6 +3735,8 @@ class Store {
|
|
|
3424
3735
|
const run = this.getWorkflowRun(workflowRunId);
|
|
3425
3736
|
if (!run)
|
|
3426
3737
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
3738
|
+
if (changed && status === "succeeded")
|
|
3739
|
+
this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
|
|
3427
3740
|
return run;
|
|
3428
3741
|
}
|
|
3429
3742
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -3485,6 +3798,17 @@ class Store {
|
|
|
3485
3798
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
3486
3799
|
return (row?.count ?? 0) > 0;
|
|
3487
3800
|
}
|
|
3801
|
+
hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
|
|
3802
|
+
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
3803
|
+
WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
|
|
3804
|
+
return rows.some((row) => {
|
|
3805
|
+
if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
|
|
3806
|
+
return true;
|
|
3807
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at))
|
|
3808
|
+
return true;
|
|
3809
|
+
return this.hasLiveWorkflowStepProcesses(row.id);
|
|
3810
|
+
});
|
|
3811
|
+
}
|
|
3488
3812
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
3489
3813
|
const now = (opts.now ?? new Date).toISOString();
|
|
3490
3814
|
const startedMs = processStartTimeMs(pid);
|
|
@@ -3617,6 +3941,10 @@ class Store {
|
|
|
3617
3941
|
loop = currentLoop;
|
|
3618
3942
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
3619
3943
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
3944
|
+
if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
|
|
3945
|
+
this.db.exec("COMMIT");
|
|
3946
|
+
return;
|
|
3947
|
+
}
|
|
3620
3948
|
if (existing) {
|
|
3621
3949
|
if (existing.status === "running") {
|
|
3622
3950
|
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
@@ -4188,9 +4516,9 @@ class Store {
|
|
|
4188
4516
|
const runDir = dirname2(manifestPath);
|
|
4189
4517
|
try {
|
|
4190
4518
|
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
4191
|
-
|
|
4519
|
+
rmSync3(runDir, { recursive: true, force: true });
|
|
4192
4520
|
} else {
|
|
4193
|
-
|
|
4521
|
+
rmSync3(manifestPath, { force: true });
|
|
4194
4522
|
}
|
|
4195
4523
|
} catch {}
|
|
4196
4524
|
}
|
|
@@ -4257,7 +4585,7 @@ class Store {
|
|
|
4257
4585
|
this.db.close();
|
|
4258
4586
|
if (this.memoryRootDir) {
|
|
4259
4587
|
try {
|
|
4260
|
-
|
|
4588
|
+
rmSync3(this.memoryRootDir, { recursive: true, force: true });
|
|
4261
4589
|
} catch {}
|
|
4262
4590
|
this.memoryRootDir = undefined;
|
|
4263
4591
|
}
|
|
@@ -4266,7 +4594,7 @@ class Store {
|
|
|
4266
4594
|
// package.json
|
|
4267
4595
|
var package_default = {
|
|
4268
4596
|
name: "@hasna/loops",
|
|
4269
|
-
version: "0.4.
|
|
4597
|
+
version: "0.4.14",
|
|
4270
4598
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4271
4599
|
type: "module",
|
|
4272
4600
|
main: "dist/index.js",
|
|
@@ -4275,6 +4603,7 @@ var package_default = {
|
|
|
4275
4603
|
loops: "dist/cli/index.js",
|
|
4276
4604
|
"loops-daemon": "dist/daemon/index.js",
|
|
4277
4605
|
"loops-api": "dist/api/index.js",
|
|
4606
|
+
"loops-serve": "dist/serve/index.js",
|
|
4278
4607
|
"loops-runner": "dist/runner/index.js",
|
|
4279
4608
|
"loops-mcp": "dist/mcp/index.js"
|
|
4280
4609
|
},
|
|
@@ -4287,6 +4616,14 @@ var package_default = {
|
|
|
4287
4616
|
types: "./dist/sdk/index.d.ts",
|
|
4288
4617
|
import: "./dist/sdk/index.js"
|
|
4289
4618
|
},
|
|
4619
|
+
"./sdk/http": {
|
|
4620
|
+
types: "./dist/sdk/http.d.ts",
|
|
4621
|
+
import: "./dist/sdk/http.js"
|
|
4622
|
+
},
|
|
4623
|
+
"./serve": {
|
|
4624
|
+
types: "./dist/serve/index.d.ts",
|
|
4625
|
+
import: "./dist/serve/index.js"
|
|
4626
|
+
},
|
|
4290
4627
|
"./mcp": {
|
|
4291
4628
|
types: "./dist/mcp/index.d.ts",
|
|
4292
4629
|
import: "./dist/mcp/index.js"
|
|
@@ -4332,7 +4669,7 @@ var package_default = {
|
|
|
4332
4669
|
"LICENSE"
|
|
4333
4670
|
],
|
|
4334
4671
|
scripts: {
|
|
4335
|
-
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4672
|
+
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4336
4673
|
"build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
|
|
4337
4674
|
typecheck: "tsc --noEmit",
|
|
4338
4675
|
test: "bun test",
|
|
@@ -4367,11 +4704,13 @@ var package_default = {
|
|
|
4367
4704
|
bun: ">=1.0.0"
|
|
4368
4705
|
},
|
|
4369
4706
|
dependencies: {
|
|
4707
|
+
"@hasna/contracts": "^0.4.1",
|
|
4370
4708
|
"@hasna/events": "^0.1.9",
|
|
4371
4709
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
4372
4710
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
4373
4711
|
ai: "6.0.204",
|
|
4374
4712
|
commander: "^13.1.0",
|
|
4713
|
+
pg: "^8.13.1",
|
|
4375
4714
|
zod: "4.4.3"
|
|
4376
4715
|
},
|
|
4377
4716
|
optionalDependencies: {
|
|
@@ -4379,6 +4718,7 @@ var package_default = {
|
|
|
4379
4718
|
},
|
|
4380
4719
|
devDependencies: {
|
|
4381
4720
|
"@types/bun": "latest",
|
|
4721
|
+
"@types/pg": "^8.11.10",
|
|
4382
4722
|
typescript: "^5.7.3"
|
|
4383
4723
|
},
|
|
4384
4724
|
publishConfig: {
|
|
@@ -4457,6 +4797,50 @@ function sourceOfTruthForMode(mode) {
|
|
|
4457
4797
|
return "self_hosted_control_plane";
|
|
4458
4798
|
return "cloud_control_plane";
|
|
4459
4799
|
}
|
|
4800
|
+
var ROUTE_ADMISSION_GATES = [
|
|
4801
|
+
"max_dispatch",
|
|
4802
|
+
"max_active",
|
|
4803
|
+
"max_active_per_project",
|
|
4804
|
+
"max_active_per_project_group",
|
|
4805
|
+
"max_active_scope",
|
|
4806
|
+
"max_per_profile"
|
|
4807
|
+
];
|
|
4808
|
+
function remoteSchedulerBackendForMode(mode, config) {
|
|
4809
|
+
if (mode === "local")
|
|
4810
|
+
return "none";
|
|
4811
|
+
if (mode === "cloud")
|
|
4812
|
+
return config.cloudApiUrl ? "hosted_control_plane_contract" : "unconfigured";
|
|
4813
|
+
if (config.databaseUrlPresent)
|
|
4814
|
+
return "postgres_contract";
|
|
4815
|
+
if (config.apiUrl)
|
|
4816
|
+
return "api_control_plane_contract";
|
|
4817
|
+
return "unconfigured";
|
|
4818
|
+
}
|
|
4819
|
+
function schedulerStateForMode(args) {
|
|
4820
|
+
const nonLocal = args.deploymentMode !== "local";
|
|
4821
|
+
return {
|
|
4822
|
+
authority: args.sourceOfTruth,
|
|
4823
|
+
localStore: {
|
|
4824
|
+
backend: "sqlite",
|
|
4825
|
+
role: args.localRole,
|
|
4826
|
+
runArtifacts: "local_files",
|
|
4827
|
+
routeAdmissionState: "workflow_work_items"
|
|
4828
|
+
},
|
|
4829
|
+
remoteStore: {
|
|
4830
|
+
backend: remoteSchedulerBackendForMode(args.deploymentMode, args.config),
|
|
4831
|
+
configured: nonLocal && args.controlPlaneConfigured,
|
|
4832
|
+
applySupported: false,
|
|
4833
|
+
objectArtifacts: nonLocal ? "object_store_contract" : "none",
|
|
4834
|
+
mutatesAws: false
|
|
4835
|
+
},
|
|
4836
|
+
routeAdmission: {
|
|
4837
|
+
stateStore: nonLocal ? "control_plane_contract" : "local_sqlite",
|
|
4838
|
+
activeStatuses: ["admitted", "running"],
|
|
4839
|
+
gates: ROUTE_ADMISSION_GATES,
|
|
4840
|
+
dryRunEvaluatesLiveCounts: false
|
|
4841
|
+
}
|
|
4842
|
+
};
|
|
4843
|
+
}
|
|
4460
4844
|
function displayControlPlaneUrl(value) {
|
|
4461
4845
|
if (!value)
|
|
4462
4846
|
return;
|
|
@@ -4482,6 +4866,9 @@ function buildDeploymentStatus(opts = {}) {
|
|
|
4482
4866
|
if (deploymentMode === "self_hosted" && !controlPlaneConfigured) {
|
|
4483
4867
|
warnings.push("self_hosted mode needs LOOPS_API_URL or LOOPS_DATABASE_URL before it can become authoritative");
|
|
4484
4868
|
}
|
|
4869
|
+
if (deploymentMode === "self_hosted" && config.databaseUrlPresent && !config.apiUrl) {
|
|
4870
|
+
warnings.push("LOOPS_DATABASE_URL selects the self_hosted storage contract; loops-runner still needs LOOPS_API_URL to claim remote work");
|
|
4871
|
+
}
|
|
4485
4872
|
if (deploymentMode === "cloud" && config.apiUrl && !config.cloudApiUrl) {
|
|
4486
4873
|
warnings.push("LOOPS_API_URL selects self_hosted; cloud mode uses LOOPS_CLOUD_API_URL");
|
|
4487
4874
|
}
|
|
@@ -4515,6 +4902,13 @@ function buildDeploymentStatus(opts = {}) {
|
|
|
4515
4902
|
required: deploymentMode !== "local",
|
|
4516
4903
|
role: deploymentMode === "local" ? "daemon" : "control_plane_worker"
|
|
4517
4904
|
},
|
|
4905
|
+
schedulerState: schedulerStateForMode({
|
|
4906
|
+
deploymentMode,
|
|
4907
|
+
sourceOfTruth: sourceOfTruthForMode(deploymentMode),
|
|
4908
|
+
localRole: deploymentMode === "local" ? "authoritative" : "cache_and_spool",
|
|
4909
|
+
controlPlaneConfigured,
|
|
4910
|
+
config
|
|
4911
|
+
}),
|
|
4518
4912
|
warnings
|
|
4519
4913
|
};
|
|
4520
4914
|
}
|
|
@@ -4527,19 +4921,16 @@ function deploymentStatusLine(status) {
|
|
|
4527
4921
|
`source=${status.deploymentModeSource}`,
|
|
4528
4922
|
`truth=${status.sourceOfTruth}`,
|
|
4529
4923
|
`local=${status.localStore.role}`,
|
|
4924
|
+
`scheduler=${status.schedulerState.routeAdmission.stateStore}`,
|
|
4530
4925
|
`control_plane=${configured}`
|
|
4531
4926
|
].join(" ");
|
|
4532
4927
|
}
|
|
4533
4928
|
|
|
4534
|
-
// src/lib/doctor.ts
|
|
4535
|
-
import { spawnSync as spawnSync5 } from "child_process";
|
|
4536
|
-
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
4537
|
-
|
|
4538
4929
|
// src/daemon/control.ts
|
|
4539
|
-
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as
|
|
4540
|
-
import { spawnSync as
|
|
4930
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
4931
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
4541
4932
|
import { hostname } from "os";
|
|
4542
|
-
import { dirname as dirname3, join as
|
|
4933
|
+
import { dirname as dirname3, join as join5 } from "path";
|
|
4543
4934
|
|
|
4544
4935
|
// src/daemon/loop.ts
|
|
4545
4936
|
function realSleep(ms) {
|
|
@@ -4569,7 +4960,7 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
4569
4960
|
return;
|
|
4570
4961
|
let raw;
|
|
4571
4962
|
try {
|
|
4572
|
-
raw =
|
|
4963
|
+
raw = readFileSync4(path, "utf8").trim();
|
|
4573
4964
|
} catch {
|
|
4574
4965
|
return;
|
|
4575
4966
|
}
|
|
@@ -4595,7 +4986,7 @@ function writePid(pid = process.pid, path = pidFilePath()) {
|
|
|
4595
4986
|
writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
4596
4987
|
}
|
|
4597
4988
|
function removePid(path = pidFilePath()) {
|
|
4598
|
-
|
|
4989
|
+
rmSync4(path, { force: true });
|
|
4599
4990
|
}
|
|
4600
4991
|
function isAlive(pid, startedAt) {
|
|
4601
4992
|
try {
|
|
@@ -4625,7 +5016,7 @@ function ownProcessGroupId() {
|
|
|
4625
5016
|
return pgrp;
|
|
4626
5017
|
}
|
|
4627
5018
|
try {
|
|
4628
|
-
const run =
|
|
5019
|
+
const run = spawnSync3("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
4629
5020
|
const pgid = Number(run.stdout.trim());
|
|
4630
5021
|
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
4631
5022
|
return pgid;
|
|
@@ -4721,7 +5112,7 @@ async function stopDaemon(opts = {}) {
|
|
|
4721
5112
|
if (!state.running || !state.pid)
|
|
4722
5113
|
return { wasRunning: false, stopped: false, forced: false };
|
|
4723
5114
|
const sleep = opts.sleep ?? realSleep;
|
|
4724
|
-
const store = new Store(
|
|
5115
|
+
const store = new Store(join5(dirname3(path), "loops.db"));
|
|
4725
5116
|
try {
|
|
4726
5117
|
const lease = store.getDaemonLease();
|
|
4727
5118
|
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
@@ -4758,15 +5149,19 @@ async function stopDaemon(opts = {}) {
|
|
|
4758
5149
|
}
|
|
4759
5150
|
}
|
|
4760
5151
|
|
|
5152
|
+
// src/lib/doctor.ts
|
|
5153
|
+
import { spawnSync as spawnSync6 } from "child_process";
|
|
5154
|
+
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
5155
|
+
|
|
4761
5156
|
// src/lib/executor.ts
|
|
4762
|
-
import { spawn as spawn2, spawnSync as
|
|
5157
|
+
import { spawn as spawn2, spawnSync as spawnSync5 } from "child_process";
|
|
4763
5158
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
4764
5159
|
import { once } from "events";
|
|
4765
5160
|
import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
|
|
4766
5161
|
import { dirname as dirname4, resolve as resolve2 } from "path";
|
|
4767
5162
|
|
|
4768
5163
|
// src/lib/accounts.ts
|
|
4769
|
-
import { spawnSync as
|
|
5164
|
+
import { spawnSync as spawnSync4 } from "child_process";
|
|
4770
5165
|
import { existsSync as existsSync3 } from "fs";
|
|
4771
5166
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4772
5167
|
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
@@ -4871,7 +5266,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4871
5266
|
if (!account)
|
|
4872
5267
|
return {};
|
|
4873
5268
|
const tool = requiredAccountTool(account, toolHint);
|
|
4874
|
-
const result =
|
|
5269
|
+
const result = spawnSync4("accounts", ["env", account.profile, "--tool", tool], {
|
|
4875
5270
|
encoding: "utf8",
|
|
4876
5271
|
env,
|
|
4877
5272
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -4887,8 +5282,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4887
5282
|
|
|
4888
5283
|
// src/lib/env.ts
|
|
4889
5284
|
import { accessSync, constants } from "fs";
|
|
4890
|
-
import { homedir as
|
|
4891
|
-
import { delimiter, join as
|
|
5285
|
+
import { homedir as homedir3 } from "os";
|
|
5286
|
+
import { delimiter, join as join6 } from "path";
|
|
4892
5287
|
function compactPathParts(parts) {
|
|
4893
5288
|
const seen = new Set;
|
|
4894
5289
|
const result = [];
|
|
@@ -4902,16 +5297,16 @@ function compactPathParts(parts) {
|
|
|
4902
5297
|
return result;
|
|
4903
5298
|
}
|
|
4904
5299
|
function commonExecutableDirs(env = process.env) {
|
|
4905
|
-
const home = env.HOME ||
|
|
5300
|
+
const home = env.HOME || homedir3();
|
|
4906
5301
|
return compactPathParts([
|
|
4907
|
-
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
env.BUN_INSTALL ?
|
|
5302
|
+
join6(home, ".local", "bin"),
|
|
5303
|
+
join6(home, ".bun", "bin"),
|
|
5304
|
+
join6(home, ".cargo", "bin"),
|
|
5305
|
+
join6(home, ".npm-global", "bin"),
|
|
5306
|
+
join6(home, "bin"),
|
|
5307
|
+
env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
|
|
4913
5308
|
env.PNPM_HOME,
|
|
4914
|
-
env.NPM_CONFIG_PREFIX ?
|
|
5309
|
+
env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
4915
5310
|
"/opt/homebrew/bin",
|
|
4916
5311
|
"/usr/local/bin",
|
|
4917
5312
|
"/usr/bin",
|
|
@@ -4935,7 +5330,7 @@ function executableExists(command, env = process.env) {
|
|
|
4935
5330
|
if (command.includes("/"))
|
|
4936
5331
|
return isExecutable(command);
|
|
4937
5332
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
4938
|
-
if (dir && isExecutable(
|
|
5333
|
+
if (dir && isExecutable(join6(dir, command)))
|
|
4939
5334
|
return true;
|
|
4940
5335
|
}
|
|
4941
5336
|
return false;
|
|
@@ -5076,6 +5471,32 @@ function timeoutResult(startedAt, error, fields = {}) {
|
|
|
5076
5471
|
function successResult(startedAt, fields = {}) {
|
|
5077
5472
|
return buildResult("succeeded", startedAt, fields);
|
|
5078
5473
|
}
|
|
5474
|
+
function codewithJsonlHasTerminalSuccess(stdout) {
|
|
5475
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
5476
|
+
const trimmed = line.trim();
|
|
5477
|
+
if (!trimmed || !trimmed.startsWith("{"))
|
|
5478
|
+
continue;
|
|
5479
|
+
let event;
|
|
5480
|
+
try {
|
|
5481
|
+
event = JSON.parse(trimmed);
|
|
5482
|
+
} catch {
|
|
5483
|
+
continue;
|
|
5484
|
+
}
|
|
5485
|
+
if (!event || typeof event !== "object")
|
|
5486
|
+
continue;
|
|
5487
|
+
const record = event;
|
|
5488
|
+
if (record.type === "task_complete")
|
|
5489
|
+
return true;
|
|
5490
|
+
const payload = record.payload;
|
|
5491
|
+
if (payload && typeof payload === "object" && payload.type === "task_complete") {
|
|
5492
|
+
return true;
|
|
5493
|
+
}
|
|
5494
|
+
}
|
|
5495
|
+
return false;
|
|
5496
|
+
}
|
|
5497
|
+
function codewithJsonlReconciledSuccess(spec, fields) {
|
|
5498
|
+
return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
|
|
5499
|
+
}
|
|
5079
5500
|
function notifySpawn(pid, opts) {
|
|
5080
5501
|
if (!pid)
|
|
5081
5502
|
return;
|
|
@@ -5091,10 +5512,48 @@ function shellQuote(value) {
|
|
|
5091
5512
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
5092
5513
|
}
|
|
5093
5514
|
function codewithProfileCandidateFromLine(line) {
|
|
5094
|
-
const
|
|
5095
|
-
if (
|
|
5096
|
-
return
|
|
5097
|
-
|
|
5515
|
+
const trimmed = line.trim();
|
|
5516
|
+
if (!trimmed || trimmed === "No auth profiles saved.")
|
|
5517
|
+
return;
|
|
5518
|
+
const cols = trimmed.split(/\s+/);
|
|
5519
|
+
const candidate = cols[0] === "*" ? cols[1] : cols[0];
|
|
5520
|
+
if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
|
|
5521
|
+
return;
|
|
5522
|
+
if (candidate === '"name":') {
|
|
5523
|
+
const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
|
|
5524
|
+
if (jsonName)
|
|
5525
|
+
return jsonName;
|
|
5526
|
+
}
|
|
5527
|
+
return candidate;
|
|
5528
|
+
}
|
|
5529
|
+
function codewithProfileCandidatesFromJson(value) {
|
|
5530
|
+
const candidates = [];
|
|
5531
|
+
const root = value && typeof value === "object" ? value : undefined;
|
|
5532
|
+
const addProfile = (entry) => {
|
|
5533
|
+
if (!entry || typeof entry !== "object")
|
|
5534
|
+
return;
|
|
5535
|
+
const name = entry.name;
|
|
5536
|
+
if (typeof name === "string" && name)
|
|
5537
|
+
candidates.push(name);
|
|
5538
|
+
};
|
|
5539
|
+
const data = root?.data;
|
|
5540
|
+
if (Array.isArray(data))
|
|
5541
|
+
data.forEach(addProfile);
|
|
5542
|
+
const profiles = root?.profiles;
|
|
5543
|
+
if (Array.isArray(profiles))
|
|
5544
|
+
profiles.forEach(addProfile);
|
|
5545
|
+
return candidates;
|
|
5546
|
+
}
|
|
5547
|
+
function codewithProfileCandidatesFromOutput(output) {
|
|
5548
|
+
const trimmed = output.trim();
|
|
5549
|
+
const jsonStart = trimmed.indexOf("{");
|
|
5550
|
+
const jsonEnd = trimmed.lastIndexOf("}");
|
|
5551
|
+
if (jsonStart >= 0 && jsonEnd >= jsonStart) {
|
|
5552
|
+
try {
|
|
5553
|
+
return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
|
|
5554
|
+
} catch {}
|
|
5555
|
+
}
|
|
5556
|
+
return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
|
|
5098
5557
|
}
|
|
5099
5558
|
function codewithProfileForError(profile) {
|
|
5100
5559
|
return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
|
|
@@ -5187,6 +5646,7 @@ function commandSpec(target, opts) {
|
|
|
5187
5646
|
preflightAnyOf: invocation.preflightAnyOf,
|
|
5188
5647
|
stdin: invocation.stdin,
|
|
5189
5648
|
allowlist: agentTarget.allowlist,
|
|
5649
|
+
agentProvider: agentTarget.provider,
|
|
5190
5650
|
worktree: agentTarget.worktree
|
|
5191
5651
|
};
|
|
5192
5652
|
}
|
|
@@ -5346,7 +5806,7 @@ function remotePreflightScript(spec, metadata) {
|
|
|
5346
5806
|
}
|
|
5347
5807
|
if (spec.nativeAuthProfile?.provider === "codewith") {
|
|
5348
5808
|
const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
|
|
5349
|
-
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE",
|
|
5809
|
+
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
|
|
5350
5810
|
}
|
|
5351
5811
|
return lines.join(`
|
|
5352
5812
|
`);
|
|
@@ -5364,6 +5824,12 @@ function transportEnv(opts) {
|
|
|
5364
5824
|
return env;
|
|
5365
5825
|
}
|
|
5366
5826
|
function assertCodewithProfileListed(profile, result) {
|
|
5827
|
+
const profiles = codewithProfileSetFromResult(result);
|
|
5828
|
+
if (!profiles.has(profile)) {
|
|
5829
|
+
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5830
|
+
}
|
|
5831
|
+
}
|
|
5832
|
+
function codewithProfileSetFromResult(result) {
|
|
5367
5833
|
if (result.error) {
|
|
5368
5834
|
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
5369
5835
|
}
|
|
@@ -5371,32 +5837,42 @@ function assertCodewithProfileListed(profile, result) {
|
|
|
5371
5837
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5372
5838
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
5373
5839
|
}
|
|
5374
|
-
|
|
5375
|
-
if (!profiles.has(profile)) {
|
|
5376
|
-
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5377
|
-
}
|
|
5840
|
+
return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
|
|
5378
5841
|
}
|
|
5379
5842
|
function preflightNativeAuthProfileSync(spec, env) {
|
|
5380
5843
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5381
5844
|
return;
|
|
5382
|
-
const
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
|
|
5386
|
-
|
|
5387
|
-
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5845
|
+
const runProfileList = (args) => {
|
|
5846
|
+
const result = spawnSync5(spec.command, args, {
|
|
5847
|
+
encoding: "utf8",
|
|
5848
|
+
env,
|
|
5849
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
5850
|
+
timeout: 15000
|
|
5851
|
+
});
|
|
5852
|
+
return {
|
|
5853
|
+
status: result.status,
|
|
5854
|
+
stdout: result.stdout ?? "",
|
|
5855
|
+
stderr: result.stderr ?? "",
|
|
5856
|
+
error: result.error?.message
|
|
5857
|
+
};
|
|
5858
|
+
};
|
|
5859
|
+
const jsonResult = runProfileList(["profile", "list", "--json"]);
|
|
5860
|
+
try {
|
|
5861
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
5862
|
+
return;
|
|
5863
|
+
} catch {}
|
|
5864
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
|
|
5394
5865
|
}
|
|
5395
5866
|
async function preflightNativeAuthProfile(spec, env) {
|
|
5396
5867
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5397
5868
|
return;
|
|
5398
|
-
const
|
|
5399
|
-
|
|
5869
|
+
const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
|
|
5870
|
+
try {
|
|
5871
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
5872
|
+
return;
|
|
5873
|
+
} catch {}
|
|
5874
|
+
const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
5875
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
|
|
5400
5876
|
}
|
|
5401
5877
|
function resolvedDirEquals(left, right) {
|
|
5402
5878
|
try {
|
|
@@ -5499,7 +5975,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
|
5499
5975
|
}
|
|
5500
5976
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
5501
5977
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
5502
|
-
const result =
|
|
5978
|
+
const result = spawnSync5(plan.command, plan.args, {
|
|
5503
5979
|
encoding: "utf8",
|
|
5504
5980
|
env: transportEnv(opts),
|
|
5505
5981
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -5598,6 +6074,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
|
5598
6074
|
if (timedOut || idleTimedOut) {
|
|
5599
6075
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5600
6076
|
}
|
|
6077
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6078
|
+
return successResult(startedAt, fields);
|
|
6079
|
+
}
|
|
5601
6080
|
if (error || exitCode !== 0) {
|
|
5602
6081
|
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5603
6082
|
}
|
|
@@ -5733,6 +6212,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
5733
6212
|
if (timedOut || idleTimedOut) {
|
|
5734
6213
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5735
6214
|
}
|
|
6215
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6216
|
+
return successResult(startedAt, fields);
|
|
6217
|
+
}
|
|
5736
6218
|
if (error || exitCode !== 0) {
|
|
5737
6219
|
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5738
6220
|
}
|
|
@@ -5763,226 +6245,20 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
5763
6245
|
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5764
6246
|
}
|
|
5765
6247
|
|
|
5766
|
-
// src/lib/doctor.ts
|
|
5767
|
-
var PROVIDER_COMMANDS = [
|
|
5768
|
-
"claude",
|
|
5769
|
-
"agent",
|
|
5770
|
-
"codewith",
|
|
5771
|
-
"aicopilot",
|
|
5772
|
-
"opencode",
|
|
5773
|
-
"codex"
|
|
5774
|
-
];
|
|
5775
|
-
function hasCommand(command) {
|
|
5776
|
-
const result = spawnSync5("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
5777
|
-
return (result.status ?? 1) === 0;
|
|
5778
|
-
}
|
|
5779
|
-
function commandVersion(command) {
|
|
5780
|
-
const result = spawnSync5(command, ["--version"], {
|
|
5781
|
-
encoding: "utf8",
|
|
5782
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
5783
|
-
});
|
|
5784
|
-
if ((result.status ?? 1) !== 0)
|
|
5785
|
-
return;
|
|
5786
|
-
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
5787
|
-
}
|
|
5788
|
-
function runDoctor(store) {
|
|
5789
|
-
const checks = [];
|
|
5790
|
-
try {
|
|
5791
|
-
const dir = ensureDataDir();
|
|
5792
|
-
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
5793
|
-
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
5794
|
-
} catch (error) {
|
|
5795
|
-
checks.push({
|
|
5796
|
-
id: "data-dir",
|
|
5797
|
-
status: "fail",
|
|
5798
|
-
message: "data directory is not writable",
|
|
5799
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5800
|
-
});
|
|
5801
|
-
}
|
|
5802
|
-
const bunVersion = commandVersion("bun");
|
|
5803
|
-
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
5804
|
-
const accountsVersion = commandVersion("accounts");
|
|
5805
|
-
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
5806
|
-
try {
|
|
5807
|
-
const machines = listOpenMachines();
|
|
5808
|
-
const local = machines.find((machine) => machine.local);
|
|
5809
|
-
checks.push({
|
|
5810
|
-
id: "machines",
|
|
5811
|
-
status: "ok",
|
|
5812
|
-
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
5813
|
-
detail: local ? `local=${local.id}` : undefined
|
|
5814
|
-
});
|
|
5815
|
-
} catch (error) {
|
|
5816
|
-
checks.push({
|
|
5817
|
-
id: "machines",
|
|
5818
|
-
status: "warn",
|
|
5819
|
-
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
5820
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5821
|
-
});
|
|
5822
|
-
}
|
|
5823
|
-
for (const command of PROVIDER_COMMANDS) {
|
|
5824
|
-
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
5825
|
-
}
|
|
5826
|
-
const status = daemonStatus(store);
|
|
5827
|
-
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
5828
|
-
const failedRuns = store.countRuns("failed");
|
|
5829
|
-
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
5830
|
-
for (const loop of store.listLoops({ status: "active" })) {
|
|
5831
|
-
try {
|
|
5832
|
-
if (loop.target.type === "workflow") {
|
|
5833
|
-
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
5834
|
-
for (const step of workflowExecutionOrder(workflow)) {
|
|
5835
|
-
preflightTarget({
|
|
5836
|
-
...step.target,
|
|
5837
|
-
account: step.account ?? step.target.account,
|
|
5838
|
-
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
5839
|
-
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
5840
|
-
}
|
|
5841
|
-
} else {
|
|
5842
|
-
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
5843
|
-
}
|
|
5844
|
-
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
5845
|
-
} catch (error) {
|
|
5846
|
-
checks.push({
|
|
5847
|
-
id: `loop:${loop.id}:preflight`,
|
|
5848
|
-
status: "fail",
|
|
5849
|
-
message: `active loop target preflight failed: ${loop.name}`,
|
|
5850
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5851
|
-
});
|
|
5852
|
-
}
|
|
5853
|
-
}
|
|
5854
|
-
return {
|
|
5855
|
-
ok: checks.every((check) => check.status !== "fail"),
|
|
5856
|
-
checks
|
|
5857
|
-
};
|
|
5858
|
-
}
|
|
5859
|
-
|
|
5860
6248
|
// src/lib/health.ts
|
|
5861
6249
|
import { createHash as createHash2 } from "crypto";
|
|
5862
|
-
import { existsSync as existsSync4, readFileSync as
|
|
5863
|
-
|
|
5864
|
-
// src/lib/format.ts
|
|
5865
|
-
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
5866
|
-
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
5867
|
-
function redact(value, visible = 0) {
|
|
5868
|
-
if (!value)
|
|
5869
|
-
return value;
|
|
5870
|
-
if (value.length <= visible)
|
|
5871
|
-
return value;
|
|
5872
|
-
if (visible <= 0)
|
|
5873
|
-
return `[redacted ${value.length} chars]`;
|
|
5874
|
-
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
5875
|
-
}
|
|
5876
|
-
function truncateTextOutput(value) {
|
|
5877
|
-
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
5878
|
-
return value;
|
|
5879
|
-
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
5880
|
-
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
5881
|
-
}
|
|
5882
|
-
function redactSensitivePayload(value, key) {
|
|
5883
|
-
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
5884
|
-
if (typeof value === "string")
|
|
5885
|
-
return redact(value);
|
|
5886
|
-
if (value === undefined || value === null)
|
|
5887
|
-
return value;
|
|
5888
|
-
return "[redacted]";
|
|
5889
|
-
}
|
|
5890
|
-
if (Array.isArray(value))
|
|
5891
|
-
return value.map((item) => redactSensitivePayload(item));
|
|
5892
|
-
if (value && typeof value === "object") {
|
|
5893
|
-
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
5894
|
-
}
|
|
5895
|
-
return value;
|
|
5896
|
-
}
|
|
5897
|
-
function textOutputBlocks(value, opts = {}) {
|
|
5898
|
-
const indent = opts.indent ?? "";
|
|
5899
|
-
const nested = `${indent} `;
|
|
5900
|
-
const blocks = [];
|
|
5901
|
-
for (const [label, output] of [
|
|
5902
|
-
["stdout", value.stdout],
|
|
5903
|
-
["stderr", value.stderr]
|
|
5904
|
-
]) {
|
|
5905
|
-
if (!output)
|
|
5906
|
-
continue;
|
|
5907
|
-
blocks.push(`${indent}${label}:`);
|
|
5908
|
-
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
5909
|
-
blocks.push(`${nested}${line}`);
|
|
5910
|
-
}
|
|
5911
|
-
}
|
|
5912
|
-
return blocks;
|
|
5913
|
-
}
|
|
5914
|
-
function publicLoop(loop) {
|
|
5915
|
-
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
5916
|
-
return {
|
|
5917
|
-
...loop,
|
|
5918
|
-
target
|
|
5919
|
-
};
|
|
5920
|
-
}
|
|
5921
|
-
function publicRun(run, showOutput = false, opts = {}) {
|
|
5922
|
-
return {
|
|
5923
|
-
...run,
|
|
5924
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
5925
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
5926
|
-
error: opts.redactError ? redact(run.error) : run.error
|
|
5927
|
-
};
|
|
5928
|
-
}
|
|
5929
|
-
function publicExecutorResult(result, showOutput = false) {
|
|
5930
|
-
return {
|
|
5931
|
-
...result,
|
|
5932
|
-
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
5933
|
-
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
5934
|
-
error: redact(result.error)
|
|
5935
|
-
};
|
|
5936
|
-
}
|
|
5937
|
-
function publicWorkflow(workflow) {
|
|
5938
|
-
return {
|
|
5939
|
-
...workflow,
|
|
5940
|
-
steps: workflow.steps.map((step) => ({
|
|
5941
|
-
...step,
|
|
5942
|
-
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
5943
|
-
}))
|
|
5944
|
-
};
|
|
5945
|
-
}
|
|
5946
|
-
function publicWorkflowRun(run) {
|
|
5947
|
-
return { ...run, error: redact(run.error) };
|
|
5948
|
-
}
|
|
5949
|
-
function publicWorkflowInvocation(invocation) {
|
|
5950
|
-
return redactSensitivePayload(invocation);
|
|
5951
|
-
}
|
|
5952
|
-
function publicWorkflowWorkItem(item) {
|
|
5953
|
-
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
5954
|
-
}
|
|
5955
|
-
function publicWorkflowStepRun(run, showOutput = false) {
|
|
5956
|
-
return {
|
|
5957
|
-
...run,
|
|
5958
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
5959
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
5960
|
-
error: redact(run.error)
|
|
5961
|
-
};
|
|
5962
|
-
}
|
|
5963
|
-
function publicWorkflowEvent(event) {
|
|
5964
|
-
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
5965
|
-
}
|
|
5966
|
-
function publicGoal(goal) {
|
|
5967
|
-
return {
|
|
5968
|
-
...goal,
|
|
5969
|
-
objective: redact(goal.objective, 120)
|
|
5970
|
-
};
|
|
5971
|
-
}
|
|
5972
|
-
function publicGoalRun(run) {
|
|
5973
|
-
return {
|
|
5974
|
-
...run,
|
|
5975
|
-
evidence: redactSensitivePayload(run.evidence),
|
|
5976
|
-
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
5977
|
-
};
|
|
5978
|
-
}
|
|
5979
|
-
|
|
5980
|
-
// src/lib/health.ts
|
|
6250
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
6251
|
+
import { join as join7 } from "path";
|
|
5981
6252
|
var EVIDENCE_CHARS = 2000;
|
|
5982
6253
|
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6254
|
+
var DEFAULT_SCAN_LIMIT = 200;
|
|
6255
|
+
var DEFAULT_SCAN_STATUSES = ["active", "paused"];
|
|
6256
|
+
var DEFAULT_MAX_FINDINGS = 100;
|
|
6257
|
+
var MIN_STALE_RUNNING_MS = 10 * 60000;
|
|
5983
6258
|
var CLASSIFICATIONS = [
|
|
5984
6259
|
"rate_limit",
|
|
5985
6260
|
"auth",
|
|
6261
|
+
"provider_unavailable",
|
|
5986
6262
|
"model_not_found",
|
|
5987
6263
|
"context_length",
|
|
5988
6264
|
"schema_response_format",
|
|
@@ -5991,10 +6267,12 @@ var CLASSIFICATIONS = [
|
|
|
5991
6267
|
"route_functional",
|
|
5992
6268
|
"timeout",
|
|
5993
6269
|
"sigsegv",
|
|
6270
|
+
"restart_interrupted",
|
|
5994
6271
|
"skipped_previous_active",
|
|
5995
6272
|
"circuit_breaker",
|
|
5996
6273
|
"unknown"
|
|
5997
6274
|
];
|
|
6275
|
+
var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
|
|
5998
6276
|
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
5999
6277
|
if (!value)
|
|
6000
6278
|
return;
|
|
@@ -6008,7 +6286,7 @@ function redactedEvidence(value) {
|
|
|
6008
6286
|
}
|
|
6009
6287
|
function searchableText(run) {
|
|
6010
6288
|
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
6011
|
-
`)
|
|
6289
|
+
`);
|
|
6012
6290
|
}
|
|
6013
6291
|
function isRecord(value) {
|
|
6014
6292
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
@@ -6033,13 +6311,36 @@ function stableFingerprint(parts) {
|
|
|
6033
6311
|
return createHash2("sha256").update(parts.join(`
|
|
6034
6312
|
`)).digest("hex").slice(0, 16);
|
|
6035
6313
|
}
|
|
6036
|
-
function
|
|
6314
|
+
function stableScanFingerprint(parts) {
|
|
6315
|
+
return `openloops:health-scan:${stableFingerprint(parts)}`;
|
|
6316
|
+
}
|
|
6317
|
+
function safeHost(value) {
|
|
6318
|
+
if (!value)
|
|
6319
|
+
return;
|
|
6320
|
+
let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
|
|
6321
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
6322
|
+
return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
|
|
6323
|
+
}
|
|
6324
|
+
function isCursorHost(host) {
|
|
6325
|
+
return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
|
|
6326
|
+
}
|
|
6327
|
+
function providerUnavailableSummary(rawText) {
|
|
6328
|
+
const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
|
|
6329
|
+
if (dns) {
|
|
6330
|
+
const host = safeHost(dns[2]);
|
|
6331
|
+
if (isCursorHost(host))
|
|
6332
|
+
return `provider DNS lookup failed: ${dns[1]} ${host}`;
|
|
6333
|
+
}
|
|
6334
|
+
return;
|
|
6335
|
+
}
|
|
6336
|
+
function stableFailureFingerprint(run, classification, summary) {
|
|
6337
|
+
const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
|
|
6037
6338
|
return stableFingerprint([
|
|
6038
6339
|
run.loopId,
|
|
6039
6340
|
classification,
|
|
6040
6341
|
String(run.status),
|
|
6041
6342
|
String(run.exitCode ?? ""),
|
|
6042
|
-
|
|
6343
|
+
evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6043
6344
|
]);
|
|
6044
6345
|
}
|
|
6045
6346
|
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
@@ -6057,13 +6358,279 @@ function healthRun(run) {
|
|
|
6057
6358
|
stderr: redactedEvidence(run.stderr)
|
|
6058
6359
|
};
|
|
6059
6360
|
}
|
|
6361
|
+
function compactText(value, limit = 500) {
|
|
6362
|
+
const text = redact(bounded(value, limit));
|
|
6363
|
+
return text?.replace(/\s+/g, " ").trim();
|
|
6364
|
+
}
|
|
6365
|
+
function publicDoctorCheck(check) {
|
|
6366
|
+
return {
|
|
6367
|
+
id: check.id,
|
|
6368
|
+
status: check.status,
|
|
6369
|
+
message: redact(bounded(check.message, 500)) ?? "",
|
|
6370
|
+
detail: redact(bounded(check.detail, 800))
|
|
6371
|
+
};
|
|
6372
|
+
}
|
|
6373
|
+
function publicDoctorReport(report) {
|
|
6374
|
+
return {
|
|
6375
|
+
ok: report.ok,
|
|
6376
|
+
checks: report.checks.map(publicDoctorCheck)
|
|
6377
|
+
};
|
|
6378
|
+
}
|
|
6379
|
+
function includedStatusSet(statuses) {
|
|
6380
|
+
const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
|
|
6381
|
+
return new Set(values);
|
|
6382
|
+
}
|
|
6383
|
+
function statusCounts(loops) {
|
|
6384
|
+
return {
|
|
6385
|
+
loops: loops.length,
|
|
6386
|
+
active: loops.filter((loop) => loop.status === "active").length,
|
|
6387
|
+
paused: loops.filter((loop) => loop.status === "paused").length,
|
|
6388
|
+
stopped: loops.filter((loop) => loop.status === "stopped").length,
|
|
6389
|
+
expired: loops.filter((loop) => loop.status === "expired").length
|
|
6390
|
+
};
|
|
6391
|
+
}
|
|
6392
|
+
function compareLoopsForScan(left, right) {
|
|
6393
|
+
const statusOrder = left.status.localeCompare(right.status);
|
|
6394
|
+
if (statusOrder !== 0)
|
|
6395
|
+
return statusOrder;
|
|
6396
|
+
return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
|
|
6397
|
+
}
|
|
6398
|
+
function scanLoops(store, statuses, opts) {
|
|
6399
|
+
const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
|
|
6400
|
+
return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
|
|
6401
|
+
}
|
|
6402
|
+
function healthReportForLoops(store, loops, generatedAt) {
|
|
6403
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
6404
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
6405
|
+
for (const expectation of expectations) {
|
|
6406
|
+
if (expectation.failure)
|
|
6407
|
+
classifications[expectation.failure.classification] += 1;
|
|
6408
|
+
}
|
|
6409
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
6410
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
6411
|
+
return {
|
|
6412
|
+
ok: unhealthy === 0,
|
|
6413
|
+
generatedAt,
|
|
6414
|
+
summary: {
|
|
6415
|
+
loops: expectations.length,
|
|
6416
|
+
healthy: expectations.length - unhealthy,
|
|
6417
|
+
unhealthy,
|
|
6418
|
+
warnings
|
|
6419
|
+
},
|
|
6420
|
+
classifications,
|
|
6421
|
+
expectations
|
|
6422
|
+
};
|
|
6423
|
+
}
|
|
6424
|
+
function ageMs(run, now) {
|
|
6425
|
+
const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
|
|
6426
|
+
const time = Date.parse(stamp);
|
|
6427
|
+
return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
|
|
6428
|
+
}
|
|
6429
|
+
function shortLoop(loop) {
|
|
6430
|
+
return {
|
|
6431
|
+
id: loop.id,
|
|
6432
|
+
name: loop.name,
|
|
6433
|
+
status: loop.status,
|
|
6434
|
+
nextRunAt: loop.nextRunAt,
|
|
6435
|
+
leaseMs: loop.leaseMs
|
|
6436
|
+
};
|
|
6437
|
+
}
|
|
6438
|
+
function priorityForSeverity(severity) {
|
|
6439
|
+
if (severity === "critical")
|
|
6440
|
+
return "critical";
|
|
6441
|
+
if (severity === "high")
|
|
6442
|
+
return "high";
|
|
6443
|
+
if (severity === "low")
|
|
6444
|
+
return "low";
|
|
6445
|
+
return "medium";
|
|
6446
|
+
}
|
|
6447
|
+
function recommendedFindingTask(finding, route) {
|
|
6448
|
+
const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
|
|
6449
|
+
if (finding.classification)
|
|
6450
|
+
tags.push(finding.classification);
|
|
6451
|
+
const description = [
|
|
6452
|
+
`OpenLoops health scan found a ${finding.kind} issue.`,
|
|
6453
|
+
finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
|
|
6454
|
+
finding.run ? `Run: ${finding.run.id}` : undefined,
|
|
6455
|
+
finding.classification ? `Classification: ${finding.classification}` : undefined,
|
|
6456
|
+
finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
|
|
6457
|
+
finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
|
|
6458
|
+
`Fingerprint: ${finding.fingerprint}`,
|
|
6459
|
+
`Severity: ${finding.severity}`,
|
|
6460
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6461
|
+
route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6462
|
+
route?.provider ? `Provider: ${route.provider}` : undefined,
|
|
6463
|
+
"",
|
|
6464
|
+
finding.message,
|
|
6465
|
+
finding.run?.error ? `Error:
|
|
6466
|
+
${finding.run.error}` : undefined,
|
|
6467
|
+
finding.run?.stderr ? `Stderr:
|
|
6468
|
+
${finding.run.stderr}` : undefined
|
|
6469
|
+
].filter(Boolean).join(`
|
|
6470
|
+
|
|
6471
|
+
`);
|
|
6472
|
+
return {
|
|
6473
|
+
title: finding.title,
|
|
6474
|
+
description,
|
|
6475
|
+
priority: priorityForSeverity(finding.severity),
|
|
6476
|
+
tags,
|
|
6477
|
+
dedupeKey: finding.fingerprint,
|
|
6478
|
+
search: { query: finding.fingerprint },
|
|
6479
|
+
compatibilityFallback: {
|
|
6480
|
+
search: ["todos", "search", finding.fingerprint, "--json"],
|
|
6481
|
+
add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
|
|
6482
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
6483
|
+
},
|
|
6484
|
+
futureNativeUpsert: {
|
|
6485
|
+
command: "todos task upsert",
|
|
6486
|
+
fields: {
|
|
6487
|
+
title: finding.title,
|
|
6488
|
+
description,
|
|
6489
|
+
priority: priorityForSeverity(finding.severity),
|
|
6490
|
+
tags,
|
|
6491
|
+
dedupeKey: finding.fingerprint,
|
|
6492
|
+
routeSource: route?.source ?? "openloops",
|
|
6493
|
+
routeKind: route?.kind ?? "health_scan",
|
|
6494
|
+
routeLoopId: finding.loop?.id ?? "",
|
|
6495
|
+
routeLoopName: finding.loop?.name ?? ""
|
|
6496
|
+
}
|
|
6497
|
+
}
|
|
6498
|
+
};
|
|
6499
|
+
}
|
|
6500
|
+
function daemonFinding(daemon) {
|
|
6501
|
+
if (daemon.running && !daemon.stale)
|
|
6502
|
+
return;
|
|
6503
|
+
const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
|
|
6504
|
+
const severity = "critical";
|
|
6505
|
+
const finding = {
|
|
6506
|
+
kind: "daemon",
|
|
6507
|
+
severity,
|
|
6508
|
+
fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
|
|
6509
|
+
title: "OpenLoops daemon health issue",
|
|
6510
|
+
message: reason
|
|
6511
|
+
};
|
|
6512
|
+
return {
|
|
6513
|
+
...finding,
|
|
6514
|
+
recommendedTask: recommendedFindingTask(finding, undefined)
|
|
6515
|
+
};
|
|
6516
|
+
}
|
|
6517
|
+
function doctorSeverity(check) {
|
|
6518
|
+
if (check.status === "fail" && check.id === "data-dir")
|
|
6519
|
+
return "critical";
|
|
6520
|
+
if (check.status === "fail")
|
|
6521
|
+
return "high";
|
|
6522
|
+
return "medium";
|
|
6523
|
+
}
|
|
6524
|
+
function doctorFinding(check, loop, route) {
|
|
6525
|
+
if (check.status === "ok" || check.id === "loop-runs")
|
|
6526
|
+
return;
|
|
6527
|
+
const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
|
|
6528
|
+
const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
|
|
6529
|
+
const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
|
|
6530
|
+
const finding = {
|
|
6531
|
+
kind,
|
|
6532
|
+
severity,
|
|
6533
|
+
fingerprint,
|
|
6534
|
+
title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
|
|
6535
|
+
message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
|
|
6536
|
+
loop: loop ? shortLoop(loop) : undefined,
|
|
6537
|
+
route,
|
|
6538
|
+
doctorCheck: publicDoctorCheck(check)
|
|
6539
|
+
};
|
|
6540
|
+
return {
|
|
6541
|
+
...finding,
|
|
6542
|
+
recommendedTask: recommendedFindingTask(finding, route)
|
|
6543
|
+
};
|
|
6544
|
+
}
|
|
6545
|
+
function latestRunFinding(expectation) {
|
|
6546
|
+
if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
|
|
6547
|
+
return;
|
|
6548
|
+
const failure = expectation.failure;
|
|
6549
|
+
if (!failure)
|
|
6550
|
+
return;
|
|
6551
|
+
const severity = expectation.loop.status === "active" ? "high" : "medium";
|
|
6552
|
+
return {
|
|
6553
|
+
kind: "latest-run",
|
|
6554
|
+
severity,
|
|
6555
|
+
fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
|
|
6556
|
+
title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
|
|
6557
|
+
message: expectation.check.message,
|
|
6558
|
+
loop: expectation.loop,
|
|
6559
|
+
run: expectation.latestRun,
|
|
6560
|
+
route: expectation.route,
|
|
6561
|
+
classification: failure.classification,
|
|
6562
|
+
doctorCheck: undefined,
|
|
6563
|
+
recommendedTask: expectation.recommendedTask
|
|
6564
|
+
};
|
|
6565
|
+
}
|
|
6566
|
+
function staleRunningFinding(loop, expectation, now, staleRunningMs) {
|
|
6567
|
+
const run = expectation.latestRun;
|
|
6568
|
+
if (loop.status !== "active" || run?.status !== "running")
|
|
6569
|
+
return;
|
|
6570
|
+
const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
|
|
6571
|
+
const age = ageMs(run, now);
|
|
6572
|
+
if (age <= threshold)
|
|
6573
|
+
return;
|
|
6574
|
+
const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
|
|
6575
|
+
const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
|
|
6576
|
+
const finding = {
|
|
6577
|
+
kind: "stale-running",
|
|
6578
|
+
severity: "critical",
|
|
6579
|
+
fingerprint,
|
|
6580
|
+
title: `OpenLoops stale running run - ${loop.name}`,
|
|
6581
|
+
message,
|
|
6582
|
+
loop: shortLoop(loop),
|
|
6583
|
+
run,
|
|
6584
|
+
route: expectation.route,
|
|
6585
|
+
ageMs: age,
|
|
6586
|
+
staleThresholdMs: threshold
|
|
6587
|
+
};
|
|
6588
|
+
return {
|
|
6589
|
+
...finding,
|
|
6590
|
+
recommendedTask: recommendedFindingTask(finding, expectation.route)
|
|
6591
|
+
};
|
|
6592
|
+
}
|
|
6593
|
+
function scanStatus(findings) {
|
|
6594
|
+
if (findings.some((finding) => finding.severity === "critical"))
|
|
6595
|
+
return "critical";
|
|
6596
|
+
if (findings.length > 0)
|
|
6597
|
+
return "degraded";
|
|
6598
|
+
return "ok";
|
|
6599
|
+
}
|
|
6600
|
+
function timestampDir(root, generatedAt) {
|
|
6601
|
+
const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
|
|
6602
|
+
return join7(root, stamp);
|
|
6603
|
+
}
|
|
6604
|
+
function healthScanMarkdown(scan) {
|
|
6605
|
+
return [
|
|
6606
|
+
"# OpenLoops Health Scan",
|
|
6607
|
+
"",
|
|
6608
|
+
`- status: ${scan.status}`,
|
|
6609
|
+
`- generated_at: ${scan.generatedAt}`,
|
|
6610
|
+
`- included_statuses: ${scan.includedStatuses.join(",")}`,
|
|
6611
|
+
`- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
|
|
6612
|
+
`- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
|
|
6613
|
+
scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
|
|
6614
|
+
scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
|
|
6615
|
+
scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
|
|
6616
|
+
"",
|
|
6617
|
+
"## Findings",
|
|
6618
|
+
scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
|
|
6619
|
+
`) : "None."
|
|
6620
|
+
].join(`
|
|
6621
|
+
`);
|
|
6622
|
+
}
|
|
6060
6623
|
function classifyRunFailure(run) {
|
|
6061
6624
|
if (run.status === "succeeded" || run.status === "running")
|
|
6062
6625
|
return;
|
|
6063
|
-
const
|
|
6626
|
+
const rawText = searchableText(run);
|
|
6627
|
+
const text = rawText.toLowerCase();
|
|
6064
6628
|
let classification = "unknown";
|
|
6629
|
+
let summary;
|
|
6065
6630
|
if (run.status === "timed_out")
|
|
6066
6631
|
classification = "timeout";
|
|
6632
|
+
else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
|
|
6633
|
+
classification = "restart_interrupted";
|
|
6067
6634
|
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
6068
6635
|
classification = "circuit_breaker";
|
|
6069
6636
|
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
@@ -6072,8 +6639,10 @@ function classifyRunFailure(run) {
|
|
|
6072
6639
|
classification = "preflight";
|
|
6073
6640
|
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
6074
6641
|
classification = "rate_limit";
|
|
6075
|
-
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
6642
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
|
|
6076
6643
|
classification = "auth";
|
|
6644
|
+
else if (summary = providerUnavailableSummary(rawText))
|
|
6645
|
+
classification = "provider_unavailable";
|
|
6077
6646
|
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
6078
6647
|
classification = "model_not_found";
|
|
6079
6648
|
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
@@ -6086,8 +6655,9 @@ function classifyRunFailure(run) {
|
|
|
6086
6655
|
classification = "sigsegv";
|
|
6087
6656
|
return {
|
|
6088
6657
|
classification,
|
|
6089
|
-
fingerprint: stableFailureFingerprint(run, classification),
|
|
6658
|
+
fingerprint: stableFailureFingerprint(run, classification, summary),
|
|
6090
6659
|
evidence: {
|
|
6660
|
+
summary,
|
|
6091
6661
|
error: redactedEvidence(run.error),
|
|
6092
6662
|
stdout: redactedEvidence(run.stdout),
|
|
6093
6663
|
stderr: redactedEvidence(run.stderr),
|
|
@@ -6132,7 +6702,7 @@ function routeEvidenceReport(run) {
|
|
|
6132
6702
|
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
6133
6703
|
if (evidencePath && existsSync4(evidencePath)) {
|
|
6134
6704
|
try {
|
|
6135
|
-
return parseJsonObject(
|
|
6705
|
+
return parseJsonObject(readFileSync5(evidencePath, "utf8")) ?? stdoutReport;
|
|
6136
6706
|
} catch {
|
|
6137
6707
|
return stdoutReport;
|
|
6138
6708
|
}
|
|
@@ -6264,6 +6834,12 @@ function targetRoute(loop) {
|
|
|
6264
6834
|
loopName: loop.name
|
|
6265
6835
|
};
|
|
6266
6836
|
}
|
|
6837
|
+
function expectationLoop(loop) {
|
|
6838
|
+
return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
|
|
6839
|
+
}
|
|
6840
|
+
function hasPendingRetry(loop, run) {
|
|
6841
|
+
return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
|
|
6842
|
+
}
|
|
6267
6843
|
function recommendedTask(loop, run, failure, route) {
|
|
6268
6844
|
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
6269
6845
|
const description = [
|
|
@@ -6275,6 +6851,7 @@ function recommendedTask(loop, run, failure, route) {
|
|
|
6275
6851
|
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6276
6852
|
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6277
6853
|
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
6854
|
+
failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
|
|
6278
6855
|
failure.evidence.error ? `Error:
|
|
6279
6856
|
${failure.evidence.error}` : undefined,
|
|
6280
6857
|
failure.evidence.stderr ? `Stderr:
|
|
@@ -6284,7 +6861,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6284
6861
|
`);
|
|
6285
6862
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6286
6863
|
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6287
|
-
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
6864
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
|
|
6288
6865
|
return {
|
|
6289
6866
|
title,
|
|
6290
6867
|
description,
|
|
@@ -6318,7 +6895,7 @@ function expectationForLoop(store, loop) {
|
|
|
6318
6895
|
const route = targetRoute(loop);
|
|
6319
6896
|
if (!latestRun) {
|
|
6320
6897
|
return {
|
|
6321
|
-
loop:
|
|
6898
|
+
loop: expectationLoop(loop),
|
|
6322
6899
|
ok: true,
|
|
6323
6900
|
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
6324
6901
|
route
|
|
@@ -6327,7 +6904,7 @@ function expectationForLoop(store, loop) {
|
|
|
6327
6904
|
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
6328
6905
|
if (routeFailure) {
|
|
6329
6906
|
return {
|
|
6330
|
-
loop:
|
|
6907
|
+
loop: expectationLoop(loop),
|
|
6331
6908
|
ok: false,
|
|
6332
6909
|
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
6333
6910
|
latestRun: healthRun(latestRun),
|
|
@@ -6338,7 +6915,7 @@ function expectationForLoop(store, loop) {
|
|
|
6338
6915
|
}
|
|
6339
6916
|
if (latestRun.status === "succeeded") {
|
|
6340
6917
|
return {
|
|
6341
|
-
loop:
|
|
6918
|
+
loop: expectationLoop(loop),
|
|
6342
6919
|
ok: true,
|
|
6343
6920
|
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
6344
6921
|
latestRun: healthRun(latestRun),
|
|
@@ -6349,7 +6926,7 @@ function expectationForLoop(store, loop) {
|
|
|
6349
6926
|
if (failure?.classification === "circuit_breaker") {
|
|
6350
6927
|
if (loop.status !== "paused") {
|
|
6351
6928
|
return {
|
|
6352
|
-
loop:
|
|
6929
|
+
loop: expectationLoop(loop),
|
|
6353
6930
|
ok: true,
|
|
6354
6931
|
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
6355
6932
|
latestRun: healthRun(latestRun),
|
|
@@ -6357,7 +6934,7 @@ function expectationForLoop(store, loop) {
|
|
|
6357
6934
|
};
|
|
6358
6935
|
}
|
|
6359
6936
|
return {
|
|
6360
|
-
loop:
|
|
6937
|
+
loop: expectationLoop(loop),
|
|
6361
6938
|
ok: false,
|
|
6362
6939
|
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
6363
6940
|
latestRun: healthRun(latestRun),
|
|
@@ -6366,8 +6943,33 @@ function expectationForLoop(store, loop) {
|
|
|
6366
6943
|
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
6367
6944
|
};
|
|
6368
6945
|
}
|
|
6946
|
+
if (failure?.classification === "restart_interrupted") {
|
|
6947
|
+
return {
|
|
6948
|
+
loop: expectationLoop(loop),
|
|
6949
|
+
ok: true,
|
|
6950
|
+
check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
|
|
6951
|
+
latestRun: healthRun(latestRun),
|
|
6952
|
+
failure,
|
|
6953
|
+
route
|
|
6954
|
+
};
|
|
6955
|
+
}
|
|
6956
|
+
if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
|
|
6957
|
+
const message = [
|
|
6958
|
+
"provider unavailable/network failure; retry is scheduled",
|
|
6959
|
+
loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
|
|
6960
|
+
failure.evidence.summary
|
|
6961
|
+
].filter(Boolean).join("; ");
|
|
6962
|
+
return {
|
|
6963
|
+
loop: expectationLoop(loop),
|
|
6964
|
+
ok: true,
|
|
6965
|
+
check: { id: "latest-run-succeeded", status: "warn", message },
|
|
6966
|
+
latestRun: healthRun(latestRun),
|
|
6967
|
+
failure,
|
|
6968
|
+
route
|
|
6969
|
+
};
|
|
6970
|
+
}
|
|
6369
6971
|
return {
|
|
6370
|
-
loop:
|
|
6972
|
+
loop: expectationLoop(loop),
|
|
6371
6973
|
ok: false,
|
|
6372
6974
|
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
6373
6975
|
latestRun: healthRun(latestRun),
|
|
@@ -6399,6 +7001,206 @@ function buildHealthReport(store, opts = {}) {
|
|
|
6399
7001
|
expectations
|
|
6400
7002
|
};
|
|
6401
7003
|
}
|
|
7004
|
+
function buildHealthScan(store, opts = {}) {
|
|
7005
|
+
const generatedAt = (opts.now ?? new Date).toISOString();
|
|
7006
|
+
const now = opts.now ?? new Date(generatedAt);
|
|
7007
|
+
const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
|
|
7008
|
+
const loops = scanLoops(store, includeStatuses, opts);
|
|
7009
|
+
const health = healthReportForLoops(store, loops, generatedAt);
|
|
7010
|
+
const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
|
|
7011
|
+
const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
|
|
7012
|
+
const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
|
|
7013
|
+
const allFindings = [];
|
|
7014
|
+
const pushFinding = (finding) => {
|
|
7015
|
+
if (!finding)
|
|
7016
|
+
return;
|
|
7017
|
+
allFindings.push(finding);
|
|
7018
|
+
};
|
|
7019
|
+
if (opts.daemon)
|
|
7020
|
+
pushFinding(daemonFinding(opts.daemon));
|
|
7021
|
+
if (opts.latestRun !== false) {
|
|
7022
|
+
for (const loop of loops) {
|
|
7023
|
+
const expectation = expectationsByLoopId.get(loop.id);
|
|
7024
|
+
if (!expectation)
|
|
7025
|
+
continue;
|
|
7026
|
+
pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
|
|
7027
|
+
pushFinding(latestRunFinding(expectation));
|
|
7028
|
+
}
|
|
7029
|
+
}
|
|
7030
|
+
const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
|
|
7031
|
+
if (doctor) {
|
|
7032
|
+
for (const check of doctor.checks) {
|
|
7033
|
+
const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
|
|
7034
|
+
const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
|
|
7035
|
+
const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
|
|
7036
|
+
pushFinding(doctorFinding(check, loop, route));
|
|
7037
|
+
}
|
|
7038
|
+
}
|
|
7039
|
+
const findings = allFindings.slice(0, maxFindings);
|
|
7040
|
+
const status = scanStatus(allFindings);
|
|
7041
|
+
const baseCounts = statusCounts(loops);
|
|
7042
|
+
return {
|
|
7043
|
+
ok: status === "ok",
|
|
7044
|
+
status,
|
|
7045
|
+
generatedAt,
|
|
7046
|
+
includedStatuses: includeStatuses,
|
|
7047
|
+
counts: {
|
|
7048
|
+
...baseCounts,
|
|
7049
|
+
latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
|
|
7050
|
+
staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
|
|
7051
|
+
daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
|
|
7052
|
+
doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
|
|
7053
|
+
preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
|
|
7054
|
+
findings: allFindings.length,
|
|
7055
|
+
reportedFindings: findings.length,
|
|
7056
|
+
truncatedFindings: Math.max(0, allFindings.length - findings.length)
|
|
7057
|
+
},
|
|
7058
|
+
daemon: opts.daemon ? {
|
|
7059
|
+
running: opts.daemon.running,
|
|
7060
|
+
stale: opts.daemon.stale,
|
|
7061
|
+
pid: opts.daemon.pid,
|
|
7062
|
+
host: opts.daemon.host,
|
|
7063
|
+
loops: opts.daemon.loops,
|
|
7064
|
+
runs: opts.daemon.runs,
|
|
7065
|
+
logPath: opts.daemon.logPath
|
|
7066
|
+
} : undefined,
|
|
7067
|
+
doctor,
|
|
7068
|
+
health,
|
|
7069
|
+
selfHeals: opts.selfHeals ?? [],
|
|
7070
|
+
findings
|
|
7071
|
+
};
|
|
7072
|
+
}
|
|
7073
|
+
function writeHealthScanReports(scan, opts = {}) {
|
|
7074
|
+
const root = opts.reportDir ?? join7(dataDir(), "reports", "health-scan");
|
|
7075
|
+
const dir = timestampDir(root, scan.generatedAt);
|
|
7076
|
+
mkdirSync6(dir, { recursive: true, mode: 448 });
|
|
7077
|
+
const json = join7(dir, "summary.json");
|
|
7078
|
+
const markdown = join7(dir, "report.md");
|
|
7079
|
+
const withReports = {
|
|
7080
|
+
...scan,
|
|
7081
|
+
reports: { dir, json, markdown }
|
|
7082
|
+
};
|
|
7083
|
+
writeFileSync3(json, JSON.stringify(withReports, null, 2), { mode: 384 });
|
|
7084
|
+
writeFileSync3(markdown, healthScanMarkdown(withReports), { mode: 384 });
|
|
7085
|
+
return withReports;
|
|
7086
|
+
}
|
|
7087
|
+
|
|
7088
|
+
// src/lib/doctor.ts
|
|
7089
|
+
var PROVIDER_COMMANDS = [
|
|
7090
|
+
"claude",
|
|
7091
|
+
"agent",
|
|
7092
|
+
"codewith",
|
|
7093
|
+
"aicopilot",
|
|
7094
|
+
"opencode",
|
|
7095
|
+
"codex"
|
|
7096
|
+
];
|
|
7097
|
+
function hasCommand(command) {
|
|
7098
|
+
const result = spawnSync6("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
7099
|
+
return (result.status ?? 1) === 0;
|
|
7100
|
+
}
|
|
7101
|
+
function commandVersion(command) {
|
|
7102
|
+
const result = spawnSync6(command, ["--version"], {
|
|
7103
|
+
encoding: "utf8",
|
|
7104
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
7105
|
+
});
|
|
7106
|
+
if ((result.status ?? 1) !== 0)
|
|
7107
|
+
return;
|
|
7108
|
+
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
7109
|
+
}
|
|
7110
|
+
function runDoctor(store) {
|
|
7111
|
+
const checks = [];
|
|
7112
|
+
try {
|
|
7113
|
+
const dir = ensureDataDir();
|
|
7114
|
+
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
7115
|
+
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
7116
|
+
} catch (error) {
|
|
7117
|
+
checks.push({
|
|
7118
|
+
id: "data-dir",
|
|
7119
|
+
status: "fail",
|
|
7120
|
+
message: "data directory is not writable",
|
|
7121
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7122
|
+
});
|
|
7123
|
+
}
|
|
7124
|
+
const bunVersion = commandVersion("bun");
|
|
7125
|
+
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
7126
|
+
const accountsVersion = commandVersion("accounts");
|
|
7127
|
+
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
7128
|
+
try {
|
|
7129
|
+
const machines = listOpenMachines();
|
|
7130
|
+
const local = machines.find((machine) => machine.local);
|
|
7131
|
+
checks.push({
|
|
7132
|
+
id: "machines",
|
|
7133
|
+
status: "ok",
|
|
7134
|
+
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
7135
|
+
detail: local ? `local=${local.id}` : undefined
|
|
7136
|
+
});
|
|
7137
|
+
} catch (error) {
|
|
7138
|
+
checks.push({
|
|
7139
|
+
id: "machines",
|
|
7140
|
+
status: "warn",
|
|
7141
|
+
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
7142
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7143
|
+
});
|
|
7144
|
+
}
|
|
7145
|
+
for (const command of PROVIDER_COMMANDS) {
|
|
7146
|
+
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
7147
|
+
}
|
|
7148
|
+
const status = daemonStatus(store);
|
|
7149
|
+
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
7150
|
+
const failedRuns = store.countRuns("failed");
|
|
7151
|
+
const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
|
|
7152
|
+
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
7153
|
+
if (restartInterruptedRuns > 0) {
|
|
7154
|
+
checks.push({
|
|
7155
|
+
id: "loop-runs:restart-interrupted",
|
|
7156
|
+
status: "warn",
|
|
7157
|
+
message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
|
|
7158
|
+
});
|
|
7159
|
+
}
|
|
7160
|
+
const deployment = buildDeploymentStatus();
|
|
7161
|
+
const schedulerState = deployment.schedulerState;
|
|
7162
|
+
checks.push({
|
|
7163
|
+
id: "scheduler-state",
|
|
7164
|
+
status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
|
|
7165
|
+
message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
|
|
7166
|
+
detail: [
|
|
7167
|
+
`route_state=${schedulerState.routeAdmission.stateStore}`,
|
|
7168
|
+
`active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
|
|
7169
|
+
`gates=${schedulerState.routeAdmission.gates.join(",")}`,
|
|
7170
|
+
`artifacts=${schedulerState.localStore.runArtifacts}`,
|
|
7171
|
+
`remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
|
|
7172
|
+
`remote_apply=${String(schedulerState.remoteStore.applySupported)}`
|
|
7173
|
+
].join(" ")
|
|
7174
|
+
});
|
|
7175
|
+
for (const loop of store.listLoops({ status: "active" })) {
|
|
7176
|
+
try {
|
|
7177
|
+
if (loop.target.type === "workflow") {
|
|
7178
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
7179
|
+
for (const step of workflowExecutionOrder(workflow)) {
|
|
7180
|
+
preflightTarget({
|
|
7181
|
+
...step.target,
|
|
7182
|
+
account: step.account ?? step.target.account,
|
|
7183
|
+
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
7184
|
+
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
7185
|
+
}
|
|
7186
|
+
} else {
|
|
7187
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
7188
|
+
}
|
|
7189
|
+
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
7190
|
+
} catch (error) {
|
|
7191
|
+
checks.push({
|
|
7192
|
+
id: `loop:${loop.id}:preflight`,
|
|
7193
|
+
status: "fail",
|
|
7194
|
+
message: `active loop target preflight failed: ${loop.name}`,
|
|
7195
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7196
|
+
});
|
|
7197
|
+
}
|
|
7198
|
+
}
|
|
7199
|
+
return {
|
|
7200
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
7201
|
+
checks
|
|
7202
|
+
};
|
|
7203
|
+
}
|
|
6402
7204
|
|
|
6403
7205
|
// src/lib/migration.ts
|
|
6404
7206
|
import { createHash as createHash3 } from "crypto";
|
|
@@ -7907,7 +8709,7 @@ var MAX_RETRY_EXPONENT = 20;
|
|
|
7907
8709
|
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
7908
8710
|
const attempt = Math.max(1, run.attempt);
|
|
7909
8711
|
const failure = classifyRunFailure(run);
|
|
7910
|
-
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
8712
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
|
|
7911
8713
|
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
7912
8714
|
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
7913
8715
|
const jitter = 0.5 + random();
|
|
@@ -8034,20 +8836,21 @@ async function executeClaimedRun(deps) {
|
|
|
8034
8836
|
daemonLeaseId: deps.daemonLeaseId,
|
|
8035
8837
|
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
8036
8838
|
})))(deps.loop, deps.run);
|
|
8839
|
+
const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
|
|
8037
8840
|
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
8038
8841
|
return deps.store.finalizeRun(deps.run.id, {
|
|
8039
|
-
status:
|
|
8040
|
-
finishedAt:
|
|
8041
|
-
durationMs:
|
|
8042
|
-
stdout:
|
|
8043
|
-
stderr:
|
|
8044
|
-
exitCode:
|
|
8045
|
-
error:
|
|
8046
|
-
pid:
|
|
8842
|
+
status: finalResult.status,
|
|
8843
|
+
finishedAt: finalResult.finishedAt,
|
|
8844
|
+
durationMs: finalResult.durationMs,
|
|
8845
|
+
stdout: finalResult.stdout,
|
|
8846
|
+
stderr: finalResult.stderr,
|
|
8847
|
+
exitCode: finalResult.exitCode,
|
|
8848
|
+
error: finalResult.error,
|
|
8849
|
+
pid: finalResult.pid
|
|
8047
8850
|
}, {
|
|
8048
8851
|
claimedBy: deps.runnerId,
|
|
8049
8852
|
daemonLeaseId: deps.daemonLeaseId,
|
|
8050
|
-
now: deps.now?.() ?? new Date(
|
|
8853
|
+
now: deps.now?.() ?? new Date(finalResult.finishedAt)
|
|
8051
8854
|
});
|
|
8052
8855
|
} catch (err) {
|
|
8053
8856
|
deps.onError?.(deps.loop, err);
|
|
@@ -8348,6 +9151,13 @@ class LoopsClient {
|
|
|
8348
9151
|
health(opts = {}) {
|
|
8349
9152
|
return buildHealthReport(this.store, opts);
|
|
8350
9153
|
}
|
|
9154
|
+
healthScan(opts = {}) {
|
|
9155
|
+
return buildHealthScan(this.store, {
|
|
9156
|
+
...opts,
|
|
9157
|
+
doctor: opts.doctor ? runDoctor(this.store) : undefined,
|
|
9158
|
+
daemon: opts.daemon ? daemonStatus(this.store) : undefined
|
|
9159
|
+
});
|
|
9160
|
+
}
|
|
8351
9161
|
goal(idOrName) {
|
|
8352
9162
|
const goal = this.store.getGoal(idOrName) ?? this.store.findGoalByLoop(idOrName) ?? this.store.findGoalByRunId(idOrName);
|
|
8353
9163
|
return {
|