@hasna/loops 0.4.13 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -3
- package/README.md +54 -12
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +695 -10
- package/dist/cli/index.js +1633 -366
- package/dist/cli/ui.d.ts +44 -0
- package/dist/daemon/index.js +948 -212
- package/dist/generated/storage-kit/health.d.ts +19 -0
- package/dist/generated/storage-kit/index.d.ts +7 -0
- package/dist/generated/storage-kit/migrations.d.ts +47 -0
- package/dist/generated/storage-kit/mode.d.ts +47 -0
- package/dist/generated/storage-kit/pool.d.ts +33 -0
- package/dist/generated/storage-kit/query.d.ts +35 -0
- package/dist/generated/storage-kit/tls.d.ts +25 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3567 -2010
- package/dist/lib/health.d.ts +83 -3
- package/dist/lib/mode.js +14 -2
- package/dist/lib/scheduler.d.ts +5 -2
- package/dist/lib/storage/index.d.ts +1 -0
- package/dist/lib/storage/index.js +1329 -211
- package/dist/lib/storage/pg-executor.d.ts +27 -0
- package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
- package/dist/lib/storage/postgres-loop-storage.d.ts +114 -0
- package/dist/lib/storage/sqlite.js +336 -8
- package/dist/lib/store.d.ts +234 -0
- package/dist/lib/store.js +350 -8
- package/dist/mcp/index.js +1067 -306
- package/dist/runner/index.js +124 -25
- package/dist/sdk/http.d.ts +115 -0
- package/dist/sdk/http.js +145 -0
- package/dist/sdk/index.d.ts +5 -1
- package/dist/sdk/index.js +1052 -312
- package/dist/serve/index.d.ts +4 -0
- package/dist/serve/index.js +7619 -0
- package/dist/types.d.ts +3 -0
- package/docs/CUTOVER-RUNBOOK.md +61 -0
- package/docs/USAGE.md +50 -11
- package/package.json +14 -2
package/dist/daemon/index.js
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
|
|
4
4
|
// src/lib/store.ts
|
|
5
5
|
import { Database } from "bun:sqlite";
|
|
6
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as
|
|
7
|
-
import { tmpdir } from "os";
|
|
8
|
-
import { basename as basename2, dirname as dirname2, join as
|
|
6
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
|
|
7
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
8
|
+
import { basename as basename2, dirname as dirname2, join as join4 } from "path";
|
|
9
9
|
|
|
10
10
|
// src/lib/errors.ts
|
|
11
11
|
class CodedError extends Error {
|
|
@@ -1204,6 +1204,196 @@ function discardWorkflowRunManifest(staged) {
|
|
|
1204
1204
|
} catch {}
|
|
1205
1205
|
}
|
|
1206
1206
|
|
|
1207
|
+
// src/lib/route/todos-cli.ts
|
|
1208
|
+
import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
|
|
1209
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1210
|
+
import { join as join3 } from "path";
|
|
1211
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
1212
|
+
|
|
1213
|
+
// src/lib/format.ts
|
|
1214
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1215
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
1216
|
+
function redact(value, visible = 0) {
|
|
1217
|
+
if (!value)
|
|
1218
|
+
return value;
|
|
1219
|
+
if (value.length <= visible)
|
|
1220
|
+
return value;
|
|
1221
|
+
if (visible <= 0)
|
|
1222
|
+
return `[redacted ${value.length} chars]`;
|
|
1223
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1224
|
+
}
|
|
1225
|
+
function truncateTextOutput(value) {
|
|
1226
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1227
|
+
return value;
|
|
1228
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1229
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1230
|
+
}
|
|
1231
|
+
function redactSensitivePayload(value, key) {
|
|
1232
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
1233
|
+
if (typeof value === "string")
|
|
1234
|
+
return redact(value);
|
|
1235
|
+
if (value === undefined || value === null)
|
|
1236
|
+
return value;
|
|
1237
|
+
return "[redacted]";
|
|
1238
|
+
}
|
|
1239
|
+
if (Array.isArray(value))
|
|
1240
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
1241
|
+
if (value && typeof value === "object") {
|
|
1242
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
1243
|
+
}
|
|
1244
|
+
return value;
|
|
1245
|
+
}
|
|
1246
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1247
|
+
const indent = opts.indent ?? "";
|
|
1248
|
+
const nested = `${indent} `;
|
|
1249
|
+
const blocks = [];
|
|
1250
|
+
for (const [label, output] of [
|
|
1251
|
+
["stdout", value.stdout],
|
|
1252
|
+
["stderr", value.stderr]
|
|
1253
|
+
]) {
|
|
1254
|
+
if (!output)
|
|
1255
|
+
continue;
|
|
1256
|
+
blocks.push(`${indent}${label}:`);
|
|
1257
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1258
|
+
blocks.push(`${nested}${line}`);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
return blocks;
|
|
1262
|
+
}
|
|
1263
|
+
function publicLoop(loop) {
|
|
1264
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1265
|
+
return {
|
|
1266
|
+
...loop,
|
|
1267
|
+
target
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
function publicRun(run, showOutput = false, opts = {}) {
|
|
1271
|
+
return {
|
|
1272
|
+
...run,
|
|
1273
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1274
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1275
|
+
error: opts.redactError ? redact(run.error) : run.error
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
1279
|
+
return {
|
|
1280
|
+
...result,
|
|
1281
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
1282
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
1283
|
+
error: redact(result.error)
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
function publicWorkflow(workflow) {
|
|
1287
|
+
return {
|
|
1288
|
+
...workflow,
|
|
1289
|
+
steps: workflow.steps.map((step) => ({
|
|
1290
|
+
...step,
|
|
1291
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
1292
|
+
}))
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
function publicWorkflowRun(run) {
|
|
1296
|
+
return { ...run, error: redact(run.error) };
|
|
1297
|
+
}
|
|
1298
|
+
function publicWorkflowInvocation(invocation) {
|
|
1299
|
+
return redactSensitivePayload(invocation);
|
|
1300
|
+
}
|
|
1301
|
+
function publicWorkflowWorkItem(item) {
|
|
1302
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
1303
|
+
}
|
|
1304
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
1305
|
+
return {
|
|
1306
|
+
...run,
|
|
1307
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1308
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1309
|
+
error: redact(run.error)
|
|
1310
|
+
};
|
|
1311
|
+
}
|
|
1312
|
+
function publicWorkflowEvent(event) {
|
|
1313
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
1314
|
+
}
|
|
1315
|
+
function publicGoal(goal) {
|
|
1316
|
+
return {
|
|
1317
|
+
...goal,
|
|
1318
|
+
objective: redact(goal.objective, 120)
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
1321
|
+
function publicGoalRun(run) {
|
|
1322
|
+
return {
|
|
1323
|
+
...run,
|
|
1324
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
1325
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// src/lib/route/todos-cli.ts
|
|
1330
|
+
function defaultLoopsProject() {
|
|
1331
|
+
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
|
|
1332
|
+
}
|
|
1333
|
+
function runLocalCommand(command, args, opts = {}) {
|
|
1334
|
+
const result = spawnSync2(command, args, {
|
|
1335
|
+
input: opts.input,
|
|
1336
|
+
encoding: "utf8",
|
|
1337
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1338
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1339
|
+
env: process.env
|
|
1340
|
+
});
|
|
1341
|
+
return {
|
|
1342
|
+
ok: result.status === 0,
|
|
1343
|
+
status: result.status,
|
|
1344
|
+
stdout: result.stdout || "",
|
|
1345
|
+
stderr: result.stderr || "",
|
|
1346
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1347
|
+
};
|
|
1348
|
+
}
|
|
1349
|
+
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
1350
|
+
const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
|
|
1351
|
+
const stdoutPath = join3(tempDir, "stdout");
|
|
1352
|
+
const stdoutFd = openSync(stdoutPath, "w");
|
|
1353
|
+
let result;
|
|
1354
|
+
try {
|
|
1355
|
+
result = spawnSync2(command, args, {
|
|
1356
|
+
input: opts.input,
|
|
1357
|
+
encoding: "utf8",
|
|
1358
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1359
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1360
|
+
env: process.env,
|
|
1361
|
+
stdio: ["pipe", stdoutFd, "pipe"]
|
|
1362
|
+
});
|
|
1363
|
+
} finally {
|
|
1364
|
+
closeSync(stdoutFd);
|
|
1365
|
+
}
|
|
1366
|
+
try {
|
|
1367
|
+
return {
|
|
1368
|
+
ok: result.status === 0,
|
|
1369
|
+
status: result.status,
|
|
1370
|
+
stdout: readFileSync3(stdoutPath, "utf8"),
|
|
1371
|
+
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
1372
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1373
|
+
};
|
|
1374
|
+
} finally {
|
|
1375
|
+
rmSync2(tempDir, { recursive: true, force: true });
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
function ensureTodosTaskList(project, slug, name, description) {
|
|
1379
|
+
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
1380
|
+
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
1381
|
+
if (!list.ok)
|
|
1382
|
+
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
1383
|
+
const values = JSON.parse(list.stdout || "[]");
|
|
1384
|
+
const found = values.find((entry) => entry.slug === slug);
|
|
1385
|
+
if (!found)
|
|
1386
|
+
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
1387
|
+
return found.id;
|
|
1388
|
+
}
|
|
1389
|
+
function todosMutationSummary(result) {
|
|
1390
|
+
return {
|
|
1391
|
+
ok: result.ok,
|
|
1392
|
+
status: result.status,
|
|
1393
|
+
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1207
1397
|
// src/lib/store.ts
|
|
1208
1398
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
1209
1399
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
@@ -1213,6 +1403,7 @@ var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "s
|
|
|
1213
1403
|
var PRUNE_BATCH_SIZE = 400;
|
|
1214
1404
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
1215
1405
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
1406
|
+
var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
|
|
1216
1407
|
function rowToLoop(row) {
|
|
1217
1408
|
return {
|
|
1218
1409
|
id: row.id,
|
|
@@ -1263,6 +1454,9 @@ function rowToRun(row) {
|
|
|
1263
1454
|
updatedAt: row.updated_at
|
|
1264
1455
|
};
|
|
1265
1456
|
}
|
|
1457
|
+
function latestRunTime(row) {
|
|
1458
|
+
return row.finished_at ?? row.started_at ?? row.created_at;
|
|
1459
|
+
}
|
|
1266
1460
|
function rowToWorkflow(row) {
|
|
1267
1461
|
return {
|
|
1268
1462
|
id: row.id,
|
|
@@ -1517,7 +1711,7 @@ class Store {
|
|
|
1517
1711
|
const file = path ?? dbPath();
|
|
1518
1712
|
if (file !== ":memory:")
|
|
1519
1713
|
ensurePrivateStorePath(file);
|
|
1520
|
-
this.rootDir = file === ":memory:" ?
|
|
1714
|
+
this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
|
|
1521
1715
|
if (file === ":memory:")
|
|
1522
1716
|
this.memoryRootDir = this.rootDir;
|
|
1523
1717
|
this.db = new Database(file);
|
|
@@ -2015,7 +2209,32 @@ class Store {
|
|
|
2015
2209
|
} else {
|
|
2016
2210
|
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2017
2211
|
}
|
|
2018
|
-
return rows.map(rowToLoop);
|
|
2212
|
+
return this.withLatestRunSummaries(rows.map(rowToLoop));
|
|
2213
|
+
}
|
|
2214
|
+
withLatestRunSummaries(loops) {
|
|
2215
|
+
if (loops.length === 0)
|
|
2216
|
+
return loops;
|
|
2217
|
+
const placeholders = loops.map(() => "?").join(",");
|
|
2218
|
+
const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
|
|
2219
|
+
FROM (
|
|
2220
|
+
SELECT loop_id, id, status, started_at, finished_at, created_at,
|
|
2221
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
|
|
2222
|
+
FROM loop_runs
|
|
2223
|
+
WHERE loop_id IN (${placeholders})
|
|
2224
|
+
)
|
|
2225
|
+
WHERE rn = 1`).all(...loops.map((loop) => loop.id));
|
|
2226
|
+
const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
|
|
2227
|
+
return loops.map((loop) => {
|
|
2228
|
+
const latest = latestByLoopId.get(loop.id);
|
|
2229
|
+
if (!latest)
|
|
2230
|
+
return loop;
|
|
2231
|
+
return {
|
|
2232
|
+
...loop,
|
|
2233
|
+
latestRunId: latest.id,
|
|
2234
|
+
latestRunStatus: latest.status,
|
|
2235
|
+
lastRunAt: latestRunTime(latest)
|
|
2236
|
+
};
|
|
2237
|
+
});
|
|
2019
2238
|
}
|
|
2020
2239
|
dueLoops(now, limit = 500) {
|
|
2021
2240
|
const rows = this.db.query(`SELECT * FROM loops
|
|
@@ -2134,6 +2353,45 @@ class Store {
|
|
|
2134
2353
|
throw error;
|
|
2135
2354
|
}
|
|
2136
2355
|
}
|
|
2356
|
+
updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
|
|
2357
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
2358
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2359
|
+
try {
|
|
2360
|
+
const current = this.requireUniqueLoop(idOrName);
|
|
2361
|
+
if (current.archivedAt)
|
|
2362
|
+
throw new LoopArchivedError(current.name || current.id);
|
|
2363
|
+
if (current.target.type !== "agent")
|
|
2364
|
+
throw new Error(`loop is not an agent loop: ${idOrName}`);
|
|
2365
|
+
if (this.hasRunningRun(current.id))
|
|
2366
|
+
throw new Error(`refusing to update running loop: ${current.id}`);
|
|
2367
|
+
const target = { ...current.target, timeoutMs };
|
|
2368
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
2369
|
+
delete target.idleTimeoutMs;
|
|
2370
|
+
const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
|
|
2371
|
+
WHERE id=$id
|
|
2372
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2373
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2374
|
+
))`).run({
|
|
2375
|
+
$id: current.id,
|
|
2376
|
+
$target: JSON.stringify(target),
|
|
2377
|
+
$updated: updated,
|
|
2378
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2379
|
+
$now: updated
|
|
2380
|
+
});
|
|
2381
|
+
if (res.changes !== 1)
|
|
2382
|
+
throw new Error("daemon lease lost");
|
|
2383
|
+
this.db.exec("COMMIT");
|
|
2384
|
+
const after = this.getLoop(current.id);
|
|
2385
|
+
if (!after)
|
|
2386
|
+
throw new Error(`loop not found after timeout update: ${current.id}`);
|
|
2387
|
+
return after;
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
try {
|
|
2390
|
+
this.db.exec("ROLLBACK");
|
|
2391
|
+
} catch {}
|
|
2392
|
+
throw error;
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2137
2395
|
createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
|
|
2138
2396
|
const normalized = normalizeCreateWorkflowInput(workflowInput);
|
|
2139
2397
|
const updated = (opts.now ?? new Date).toISOString();
|
|
@@ -2464,6 +2722,59 @@ class Store {
|
|
|
2464
2722
|
updated
|
|
2465
2723
|
});
|
|
2466
2724
|
}
|
|
2725
|
+
taskLifecycleTodosPointerContext(workflowRunId) {
|
|
2726
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
2727
|
+
if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
|
|
2728
|
+
return;
|
|
2729
|
+
const workItem = this.getWorkflowWorkItem(run.workItemId);
|
|
2730
|
+
if (!workItem || workItem.routeKey !== "todos-task")
|
|
2731
|
+
return;
|
|
2732
|
+
const invocation = this.getWorkflowInvocation(run.invocationId);
|
|
2733
|
+
if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
|
|
2734
|
+
return;
|
|
2735
|
+
const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
|
|
2736
|
+
const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
|
|
2737
|
+
if (!projectPath || !taskId)
|
|
2738
|
+
return;
|
|
2739
|
+
return {
|
|
2740
|
+
projectPath,
|
|
2741
|
+
taskId,
|
|
2742
|
+
invocationId: invocation.id,
|
|
2743
|
+
workflowRunId: run.id,
|
|
2744
|
+
manifestPath: run.manifestPath
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
|
|
2748
|
+
const context = this.taskLifecycleTodosPointerContext(workflowRunId);
|
|
2749
|
+
if (!context)
|
|
2750
|
+
return;
|
|
2751
|
+
const result = runLocalCommand("todos", [
|
|
2752
|
+
"--project",
|
|
2753
|
+
context.projectPath,
|
|
2754
|
+
"task",
|
|
2755
|
+
"workflow-pointers",
|
|
2756
|
+
context.taskId,
|
|
2757
|
+
"--clear",
|
|
2758
|
+
"--invocation",
|
|
2759
|
+
context.invocationId,
|
|
2760
|
+
"--run",
|
|
2761
|
+
context.workflowRunId,
|
|
2762
|
+
"--manifest",
|
|
2763
|
+
context.manifestPath,
|
|
2764
|
+
"--state",
|
|
2765
|
+
"succeeded",
|
|
2766
|
+
"--actor",
|
|
2767
|
+
"openloops:task-lifecycle"
|
|
2768
|
+
]);
|
|
2769
|
+
this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
|
|
2770
|
+
projectPath: context.projectPath,
|
|
2771
|
+
taskId: context.taskId,
|
|
2772
|
+
invocationId: context.invocationId,
|
|
2773
|
+
workflowRunId: context.workflowRunId,
|
|
2774
|
+
manifestPath: context.manifestPath,
|
|
2775
|
+
mutation: todosMutationSummary(result)
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2467
2778
|
createWorkflowInvocation(input) {
|
|
2468
2779
|
const now = nowIso();
|
|
2469
2780
|
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
@@ -3426,6 +3737,8 @@ class Store {
|
|
|
3426
3737
|
const run = this.getWorkflowRun(workflowRunId);
|
|
3427
3738
|
if (!run)
|
|
3428
3739
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
3740
|
+
if (changed && status === "succeeded")
|
|
3741
|
+
this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
|
|
3429
3742
|
return run;
|
|
3430
3743
|
}
|
|
3431
3744
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -3487,6 +3800,17 @@ class Store {
|
|
|
3487
3800
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
3488
3801
|
return (row?.count ?? 0) > 0;
|
|
3489
3802
|
}
|
|
3803
|
+
hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
|
|
3804
|
+
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
3805
|
+
WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
|
|
3806
|
+
return rows.some((row) => {
|
|
3807
|
+
if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
|
|
3808
|
+
return true;
|
|
3809
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at))
|
|
3810
|
+
return true;
|
|
3811
|
+
return this.hasLiveWorkflowStepProcesses(row.id);
|
|
3812
|
+
});
|
|
3813
|
+
}
|
|
3490
3814
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
3491
3815
|
const now = (opts.now ?? new Date).toISOString();
|
|
3492
3816
|
const startedMs = processStartTimeMs(pid);
|
|
@@ -3619,6 +3943,10 @@ class Store {
|
|
|
3619
3943
|
loop = currentLoop;
|
|
3620
3944
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
3621
3945
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
3946
|
+
if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
|
|
3947
|
+
this.db.exec("COMMIT");
|
|
3948
|
+
return;
|
|
3949
|
+
}
|
|
3622
3950
|
if (existing) {
|
|
3623
3951
|
if (existing.status === "running") {
|
|
3624
3952
|
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
@@ -4190,9 +4518,9 @@ class Store {
|
|
|
4190
4518
|
const runDir = dirname2(manifestPath);
|
|
4191
4519
|
try {
|
|
4192
4520
|
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
4193
|
-
|
|
4521
|
+
rmSync3(runDir, { recursive: true, force: true });
|
|
4194
4522
|
} else {
|
|
4195
|
-
|
|
4523
|
+
rmSync3(manifestPath, { force: true });
|
|
4196
4524
|
}
|
|
4197
4525
|
} catch {}
|
|
4198
4526
|
}
|
|
@@ -4259,7 +4587,7 @@ class Store {
|
|
|
4259
4587
|
this.db.close();
|
|
4260
4588
|
if (this.memoryRootDir) {
|
|
4261
4589
|
try {
|
|
4262
|
-
|
|
4590
|
+
rmSync3(this.memoryRootDir, { recursive: true, force: true });
|
|
4263
4591
|
} catch {}
|
|
4264
4592
|
this.memoryRootDir = undefined;
|
|
4265
4593
|
}
|
|
@@ -4270,136 +4598,24 @@ class Store {
|
|
|
4270
4598
|
import { Command } from "commander";
|
|
4271
4599
|
|
|
4272
4600
|
// src/daemon/daemon.ts
|
|
4273
|
-
import { copyFileSync, existsSync as existsSync5, openSync, renameSync as renameSync2, rmSync as
|
|
4601
|
+
import { copyFileSync, existsSync as existsSync5, openSync as openSync2, renameSync as renameSync2, rmSync as rmSync5, statSync, truncateSync } from "fs";
|
|
4274
4602
|
import { hostname as hostname2 } from "os";
|
|
4275
4603
|
import { spawn as spawn3 } from "child_process";
|
|
4276
4604
|
|
|
4277
4605
|
// src/lib/health.ts
|
|
4278
4606
|
import { createHash as createHash2 } from "crypto";
|
|
4279
|
-
import { existsSync as existsSync2, readFileSync as
|
|
4280
|
-
|
|
4281
|
-
// src/lib/format.ts
|
|
4282
|
-
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
4283
|
-
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
4284
|
-
function redact(value, visible = 0) {
|
|
4285
|
-
if (!value)
|
|
4286
|
-
return value;
|
|
4287
|
-
if (value.length <= visible)
|
|
4288
|
-
return value;
|
|
4289
|
-
if (visible <= 0)
|
|
4290
|
-
return `[redacted ${value.length} chars]`;
|
|
4291
|
-
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
4292
|
-
}
|
|
4293
|
-
function truncateTextOutput(value) {
|
|
4294
|
-
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
4295
|
-
return value;
|
|
4296
|
-
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
4297
|
-
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
4298
|
-
}
|
|
4299
|
-
function redactSensitivePayload(value, key) {
|
|
4300
|
-
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
4301
|
-
if (typeof value === "string")
|
|
4302
|
-
return redact(value);
|
|
4303
|
-
if (value === undefined || value === null)
|
|
4304
|
-
return value;
|
|
4305
|
-
return "[redacted]";
|
|
4306
|
-
}
|
|
4307
|
-
if (Array.isArray(value))
|
|
4308
|
-
return value.map((item) => redactSensitivePayload(item));
|
|
4309
|
-
if (value && typeof value === "object") {
|
|
4310
|
-
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
4311
|
-
}
|
|
4312
|
-
return value;
|
|
4313
|
-
}
|
|
4314
|
-
function textOutputBlocks(value, opts = {}) {
|
|
4315
|
-
const indent = opts.indent ?? "";
|
|
4316
|
-
const nested = `${indent} `;
|
|
4317
|
-
const blocks = [];
|
|
4318
|
-
for (const [label, output] of [
|
|
4319
|
-
["stdout", value.stdout],
|
|
4320
|
-
["stderr", value.stderr]
|
|
4321
|
-
]) {
|
|
4322
|
-
if (!output)
|
|
4323
|
-
continue;
|
|
4324
|
-
blocks.push(`${indent}${label}:`);
|
|
4325
|
-
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
4326
|
-
blocks.push(`${nested}${line}`);
|
|
4327
|
-
}
|
|
4328
|
-
}
|
|
4329
|
-
return blocks;
|
|
4330
|
-
}
|
|
4331
|
-
function publicLoop(loop) {
|
|
4332
|
-
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
4333
|
-
return {
|
|
4334
|
-
...loop,
|
|
4335
|
-
target
|
|
4336
|
-
};
|
|
4337
|
-
}
|
|
4338
|
-
function publicRun(run, showOutput = false, opts = {}) {
|
|
4339
|
-
return {
|
|
4340
|
-
...run,
|
|
4341
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
4342
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
4343
|
-
error: opts.redactError ? redact(run.error) : run.error
|
|
4344
|
-
};
|
|
4345
|
-
}
|
|
4346
|
-
function publicExecutorResult(result, showOutput = false) {
|
|
4347
|
-
return {
|
|
4348
|
-
...result,
|
|
4349
|
-
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
4350
|
-
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
4351
|
-
error: redact(result.error)
|
|
4352
|
-
};
|
|
4353
|
-
}
|
|
4354
|
-
function publicWorkflow(workflow) {
|
|
4355
|
-
return {
|
|
4356
|
-
...workflow,
|
|
4357
|
-
steps: workflow.steps.map((step) => ({
|
|
4358
|
-
...step,
|
|
4359
|
-
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
4360
|
-
}))
|
|
4361
|
-
};
|
|
4362
|
-
}
|
|
4363
|
-
function publicWorkflowRun(run) {
|
|
4364
|
-
return { ...run, error: redact(run.error) };
|
|
4365
|
-
}
|
|
4366
|
-
function publicWorkflowInvocation(invocation) {
|
|
4367
|
-
return redactSensitivePayload(invocation);
|
|
4368
|
-
}
|
|
4369
|
-
function publicWorkflowWorkItem(item) {
|
|
4370
|
-
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
4371
|
-
}
|
|
4372
|
-
function publicWorkflowStepRun(run, showOutput = false) {
|
|
4373
|
-
return {
|
|
4374
|
-
...run,
|
|
4375
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
4376
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
4377
|
-
error: redact(run.error)
|
|
4378
|
-
};
|
|
4379
|
-
}
|
|
4380
|
-
function publicWorkflowEvent(event) {
|
|
4381
|
-
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
4382
|
-
}
|
|
4383
|
-
function publicGoal(goal) {
|
|
4384
|
-
return {
|
|
4385
|
-
...goal,
|
|
4386
|
-
objective: redact(goal.objective, 120)
|
|
4387
|
-
};
|
|
4388
|
-
}
|
|
4389
|
-
function publicGoalRun(run) {
|
|
4390
|
-
return {
|
|
4391
|
-
...run,
|
|
4392
|
-
evidence: redactSensitivePayload(run.evidence),
|
|
4393
|
-
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
4394
|
-
};
|
|
4395
|
-
}
|
|
4396
|
-
|
|
4397
|
-
// src/lib/health.ts
|
|
4607
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
4608
|
+
import { join as join5 } from "path";
|
|
4398
4609
|
var EVIDENCE_CHARS = 2000;
|
|
4399
4610
|
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
4611
|
+
var DEFAULT_SCAN_LIMIT = 200;
|
|
4612
|
+
var DEFAULT_SCAN_STATUSES = ["active", "paused"];
|
|
4613
|
+
var DEFAULT_MAX_FINDINGS = 100;
|
|
4614
|
+
var MIN_STALE_RUNNING_MS = 10 * 60000;
|
|
4400
4615
|
var CLASSIFICATIONS = [
|
|
4401
4616
|
"rate_limit",
|
|
4402
4617
|
"auth",
|
|
4618
|
+
"provider_unavailable",
|
|
4403
4619
|
"model_not_found",
|
|
4404
4620
|
"context_length",
|
|
4405
4621
|
"schema_response_format",
|
|
@@ -4408,10 +4624,12 @@ var CLASSIFICATIONS = [
|
|
|
4408
4624
|
"route_functional",
|
|
4409
4625
|
"timeout",
|
|
4410
4626
|
"sigsegv",
|
|
4627
|
+
"restart_interrupted",
|
|
4411
4628
|
"skipped_previous_active",
|
|
4412
4629
|
"circuit_breaker",
|
|
4413
4630
|
"unknown"
|
|
4414
4631
|
];
|
|
4632
|
+
var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
|
|
4415
4633
|
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
4416
4634
|
if (!value)
|
|
4417
4635
|
return;
|
|
@@ -4425,7 +4643,7 @@ function redactedEvidence(value) {
|
|
|
4425
4643
|
}
|
|
4426
4644
|
function searchableText(run) {
|
|
4427
4645
|
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
4428
|
-
`)
|
|
4646
|
+
`);
|
|
4429
4647
|
}
|
|
4430
4648
|
function isRecord(value) {
|
|
4431
4649
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
@@ -4450,13 +4668,36 @@ function stableFingerprint(parts) {
|
|
|
4450
4668
|
return createHash2("sha256").update(parts.join(`
|
|
4451
4669
|
`)).digest("hex").slice(0, 16);
|
|
4452
4670
|
}
|
|
4453
|
-
function
|
|
4671
|
+
function stableScanFingerprint(parts) {
|
|
4672
|
+
return `openloops:health-scan:${stableFingerprint(parts)}`;
|
|
4673
|
+
}
|
|
4674
|
+
function safeHost(value) {
|
|
4675
|
+
if (!value)
|
|
4676
|
+
return;
|
|
4677
|
+
let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
|
|
4678
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
4679
|
+
return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
|
|
4680
|
+
}
|
|
4681
|
+
function isCursorHost(host) {
|
|
4682
|
+
return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
|
|
4683
|
+
}
|
|
4684
|
+
function providerUnavailableSummary(rawText) {
|
|
4685
|
+
const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
|
|
4686
|
+
if (dns) {
|
|
4687
|
+
const host = safeHost(dns[2]);
|
|
4688
|
+
if (isCursorHost(host))
|
|
4689
|
+
return `provider DNS lookup failed: ${dns[1]} ${host}`;
|
|
4690
|
+
}
|
|
4691
|
+
return;
|
|
4692
|
+
}
|
|
4693
|
+
function stableFailureFingerprint(run, classification, summary) {
|
|
4694
|
+
const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
|
|
4454
4695
|
return stableFingerprint([
|
|
4455
4696
|
run.loopId,
|
|
4456
4697
|
classification,
|
|
4457
4698
|
String(run.status),
|
|
4458
4699
|
String(run.exitCode ?? ""),
|
|
4459
|
-
|
|
4700
|
+
evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
4460
4701
|
]);
|
|
4461
4702
|
}
|
|
4462
4703
|
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
@@ -4474,13 +4715,279 @@ function healthRun(run) {
|
|
|
4474
4715
|
stderr: redactedEvidence(run.stderr)
|
|
4475
4716
|
};
|
|
4476
4717
|
}
|
|
4718
|
+
function compactText(value, limit = 500) {
|
|
4719
|
+
const text = redact(bounded(value, limit));
|
|
4720
|
+
return text?.replace(/\s+/g, " ").trim();
|
|
4721
|
+
}
|
|
4722
|
+
function publicDoctorCheck(check) {
|
|
4723
|
+
return {
|
|
4724
|
+
id: check.id,
|
|
4725
|
+
status: check.status,
|
|
4726
|
+
message: redact(bounded(check.message, 500)) ?? "",
|
|
4727
|
+
detail: redact(bounded(check.detail, 800))
|
|
4728
|
+
};
|
|
4729
|
+
}
|
|
4730
|
+
function publicDoctorReport(report) {
|
|
4731
|
+
return {
|
|
4732
|
+
ok: report.ok,
|
|
4733
|
+
checks: report.checks.map(publicDoctorCheck)
|
|
4734
|
+
};
|
|
4735
|
+
}
|
|
4736
|
+
function includedStatusSet(statuses) {
|
|
4737
|
+
const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
|
|
4738
|
+
return new Set(values);
|
|
4739
|
+
}
|
|
4740
|
+
function statusCounts(loops) {
|
|
4741
|
+
return {
|
|
4742
|
+
loops: loops.length,
|
|
4743
|
+
active: loops.filter((loop) => loop.status === "active").length,
|
|
4744
|
+
paused: loops.filter((loop) => loop.status === "paused").length,
|
|
4745
|
+
stopped: loops.filter((loop) => loop.status === "stopped").length,
|
|
4746
|
+
expired: loops.filter((loop) => loop.status === "expired").length
|
|
4747
|
+
};
|
|
4748
|
+
}
|
|
4749
|
+
function compareLoopsForScan(left, right) {
|
|
4750
|
+
const statusOrder = left.status.localeCompare(right.status);
|
|
4751
|
+
if (statusOrder !== 0)
|
|
4752
|
+
return statusOrder;
|
|
4753
|
+
return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
|
|
4754
|
+
}
|
|
4755
|
+
function scanLoops(store, statuses, opts) {
|
|
4756
|
+
const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
|
|
4757
|
+
return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
|
|
4758
|
+
}
|
|
4759
|
+
function healthReportForLoops(store, loops, generatedAt) {
|
|
4760
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
4761
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
4762
|
+
for (const expectation of expectations) {
|
|
4763
|
+
if (expectation.failure)
|
|
4764
|
+
classifications[expectation.failure.classification] += 1;
|
|
4765
|
+
}
|
|
4766
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
4767
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
4768
|
+
return {
|
|
4769
|
+
ok: unhealthy === 0,
|
|
4770
|
+
generatedAt,
|
|
4771
|
+
summary: {
|
|
4772
|
+
loops: expectations.length,
|
|
4773
|
+
healthy: expectations.length - unhealthy,
|
|
4774
|
+
unhealthy,
|
|
4775
|
+
warnings
|
|
4776
|
+
},
|
|
4777
|
+
classifications,
|
|
4778
|
+
expectations
|
|
4779
|
+
};
|
|
4780
|
+
}
|
|
4781
|
+
function ageMs(run, now) {
|
|
4782
|
+
const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
|
|
4783
|
+
const time = Date.parse(stamp);
|
|
4784
|
+
return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
|
|
4785
|
+
}
|
|
4786
|
+
function shortLoop(loop) {
|
|
4787
|
+
return {
|
|
4788
|
+
id: loop.id,
|
|
4789
|
+
name: loop.name,
|
|
4790
|
+
status: loop.status,
|
|
4791
|
+
nextRunAt: loop.nextRunAt,
|
|
4792
|
+
leaseMs: loop.leaseMs
|
|
4793
|
+
};
|
|
4794
|
+
}
|
|
4795
|
+
function priorityForSeverity(severity) {
|
|
4796
|
+
if (severity === "critical")
|
|
4797
|
+
return "critical";
|
|
4798
|
+
if (severity === "high")
|
|
4799
|
+
return "high";
|
|
4800
|
+
if (severity === "low")
|
|
4801
|
+
return "low";
|
|
4802
|
+
return "medium";
|
|
4803
|
+
}
|
|
4804
|
+
function recommendedFindingTask(finding, route) {
|
|
4805
|
+
const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
|
|
4806
|
+
if (finding.classification)
|
|
4807
|
+
tags.push(finding.classification);
|
|
4808
|
+
const description = [
|
|
4809
|
+
`OpenLoops health scan found a ${finding.kind} issue.`,
|
|
4810
|
+
finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
|
|
4811
|
+
finding.run ? `Run: ${finding.run.id}` : undefined,
|
|
4812
|
+
finding.classification ? `Classification: ${finding.classification}` : undefined,
|
|
4813
|
+
finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
|
|
4814
|
+
finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
|
|
4815
|
+
`Fingerprint: ${finding.fingerprint}`,
|
|
4816
|
+
`Severity: ${finding.severity}`,
|
|
4817
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
4818
|
+
route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
4819
|
+
route?.provider ? `Provider: ${route.provider}` : undefined,
|
|
4820
|
+
"",
|
|
4821
|
+
finding.message,
|
|
4822
|
+
finding.run?.error ? `Error:
|
|
4823
|
+
${finding.run.error}` : undefined,
|
|
4824
|
+
finding.run?.stderr ? `Stderr:
|
|
4825
|
+
${finding.run.stderr}` : undefined
|
|
4826
|
+
].filter(Boolean).join(`
|
|
4827
|
+
|
|
4828
|
+
`);
|
|
4829
|
+
return {
|
|
4830
|
+
title: finding.title,
|
|
4831
|
+
description,
|
|
4832
|
+
priority: priorityForSeverity(finding.severity),
|
|
4833
|
+
tags,
|
|
4834
|
+
dedupeKey: finding.fingerprint,
|
|
4835
|
+
search: { query: finding.fingerprint },
|
|
4836
|
+
compatibilityFallback: {
|
|
4837
|
+
search: ["todos", "search", finding.fingerprint, "--json"],
|
|
4838
|
+
add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
|
|
4839
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
4840
|
+
},
|
|
4841
|
+
futureNativeUpsert: {
|
|
4842
|
+
command: "todos task upsert",
|
|
4843
|
+
fields: {
|
|
4844
|
+
title: finding.title,
|
|
4845
|
+
description,
|
|
4846
|
+
priority: priorityForSeverity(finding.severity),
|
|
4847
|
+
tags,
|
|
4848
|
+
dedupeKey: finding.fingerprint,
|
|
4849
|
+
routeSource: route?.source ?? "openloops",
|
|
4850
|
+
routeKind: route?.kind ?? "health_scan",
|
|
4851
|
+
routeLoopId: finding.loop?.id ?? "",
|
|
4852
|
+
routeLoopName: finding.loop?.name ?? ""
|
|
4853
|
+
}
|
|
4854
|
+
}
|
|
4855
|
+
};
|
|
4856
|
+
}
|
|
4857
|
+
function daemonFinding(daemon) {
|
|
4858
|
+
if (daemon.running && !daemon.stale)
|
|
4859
|
+
return;
|
|
4860
|
+
const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
|
|
4861
|
+
const severity = "critical";
|
|
4862
|
+
const finding = {
|
|
4863
|
+
kind: "daemon",
|
|
4864
|
+
severity,
|
|
4865
|
+
fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
|
|
4866
|
+
title: "OpenLoops daemon health issue",
|
|
4867
|
+
message: reason
|
|
4868
|
+
};
|
|
4869
|
+
return {
|
|
4870
|
+
...finding,
|
|
4871
|
+
recommendedTask: recommendedFindingTask(finding, undefined)
|
|
4872
|
+
};
|
|
4873
|
+
}
|
|
4874
|
+
function doctorSeverity(check) {
|
|
4875
|
+
if (check.status === "fail" && check.id === "data-dir")
|
|
4876
|
+
return "critical";
|
|
4877
|
+
if (check.status === "fail")
|
|
4878
|
+
return "high";
|
|
4879
|
+
return "medium";
|
|
4880
|
+
}
|
|
4881
|
+
function doctorFinding(check, loop, route) {
|
|
4882
|
+
if (check.status === "ok" || check.id === "loop-runs")
|
|
4883
|
+
return;
|
|
4884
|
+
const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
|
|
4885
|
+
const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
|
|
4886
|
+
const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
|
|
4887
|
+
const finding = {
|
|
4888
|
+
kind,
|
|
4889
|
+
severity,
|
|
4890
|
+
fingerprint,
|
|
4891
|
+
title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
|
|
4892
|
+
message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
|
|
4893
|
+
loop: loop ? shortLoop(loop) : undefined,
|
|
4894
|
+
route,
|
|
4895
|
+
doctorCheck: publicDoctorCheck(check)
|
|
4896
|
+
};
|
|
4897
|
+
return {
|
|
4898
|
+
...finding,
|
|
4899
|
+
recommendedTask: recommendedFindingTask(finding, route)
|
|
4900
|
+
};
|
|
4901
|
+
}
|
|
4902
|
+
function latestRunFinding(expectation) {
|
|
4903
|
+
if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
|
|
4904
|
+
return;
|
|
4905
|
+
const failure = expectation.failure;
|
|
4906
|
+
if (!failure)
|
|
4907
|
+
return;
|
|
4908
|
+
const severity = expectation.loop.status === "active" ? "high" : "medium";
|
|
4909
|
+
return {
|
|
4910
|
+
kind: "latest-run",
|
|
4911
|
+
severity,
|
|
4912
|
+
fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
|
|
4913
|
+
title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
|
|
4914
|
+
message: expectation.check.message,
|
|
4915
|
+
loop: expectation.loop,
|
|
4916
|
+
run: expectation.latestRun,
|
|
4917
|
+
route: expectation.route,
|
|
4918
|
+
classification: failure.classification,
|
|
4919
|
+
doctorCheck: undefined,
|
|
4920
|
+
recommendedTask: expectation.recommendedTask
|
|
4921
|
+
};
|
|
4922
|
+
}
|
|
4923
|
+
function staleRunningFinding(loop, expectation, now, staleRunningMs) {
|
|
4924
|
+
const run = expectation.latestRun;
|
|
4925
|
+
if (loop.status !== "active" || run?.status !== "running")
|
|
4926
|
+
return;
|
|
4927
|
+
const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
|
|
4928
|
+
const age = ageMs(run, now);
|
|
4929
|
+
if (age <= threshold)
|
|
4930
|
+
return;
|
|
4931
|
+
const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
|
|
4932
|
+
const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
|
|
4933
|
+
const finding = {
|
|
4934
|
+
kind: "stale-running",
|
|
4935
|
+
severity: "critical",
|
|
4936
|
+
fingerprint,
|
|
4937
|
+
title: `OpenLoops stale running run - ${loop.name}`,
|
|
4938
|
+
message,
|
|
4939
|
+
loop: shortLoop(loop),
|
|
4940
|
+
run,
|
|
4941
|
+
route: expectation.route,
|
|
4942
|
+
ageMs: age,
|
|
4943
|
+
staleThresholdMs: threshold
|
|
4944
|
+
};
|
|
4945
|
+
return {
|
|
4946
|
+
...finding,
|
|
4947
|
+
recommendedTask: recommendedFindingTask(finding, expectation.route)
|
|
4948
|
+
};
|
|
4949
|
+
}
|
|
4950
|
+
function scanStatus(findings) {
|
|
4951
|
+
if (findings.some((finding) => finding.severity === "critical"))
|
|
4952
|
+
return "critical";
|
|
4953
|
+
if (findings.length > 0)
|
|
4954
|
+
return "degraded";
|
|
4955
|
+
return "ok";
|
|
4956
|
+
}
|
|
4957
|
+
function timestampDir(root, generatedAt) {
|
|
4958
|
+
const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
|
|
4959
|
+
return join5(root, stamp);
|
|
4960
|
+
}
|
|
4961
|
+
function healthScanMarkdown(scan) {
|
|
4962
|
+
return [
|
|
4963
|
+
"# OpenLoops Health Scan",
|
|
4964
|
+
"",
|
|
4965
|
+
`- status: ${scan.status}`,
|
|
4966
|
+
`- generated_at: ${scan.generatedAt}`,
|
|
4967
|
+
`- included_statuses: ${scan.includedStatuses.join(",")}`,
|
|
4968
|
+
`- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
|
|
4969
|
+
`- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
|
|
4970
|
+
scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
|
|
4971
|
+
scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
|
|
4972
|
+
scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
|
|
4973
|
+
"",
|
|
4974
|
+
"## Findings",
|
|
4975
|
+
scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
|
|
4976
|
+
`) : "None."
|
|
4977
|
+
].join(`
|
|
4978
|
+
`);
|
|
4979
|
+
}
|
|
4477
4980
|
function classifyRunFailure(run) {
|
|
4478
4981
|
if (run.status === "succeeded" || run.status === "running")
|
|
4479
4982
|
return;
|
|
4480
|
-
const
|
|
4983
|
+
const rawText = searchableText(run);
|
|
4984
|
+
const text = rawText.toLowerCase();
|
|
4481
4985
|
let classification = "unknown";
|
|
4986
|
+
let summary;
|
|
4482
4987
|
if (run.status === "timed_out")
|
|
4483
4988
|
classification = "timeout";
|
|
4989
|
+
else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
|
|
4990
|
+
classification = "restart_interrupted";
|
|
4484
4991
|
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
4485
4992
|
classification = "circuit_breaker";
|
|
4486
4993
|
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
@@ -4489,8 +4996,10 @@ function classifyRunFailure(run) {
|
|
|
4489
4996
|
classification = "preflight";
|
|
4490
4997
|
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
4491
4998
|
classification = "rate_limit";
|
|
4492
|
-
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
4999
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
|
|
4493
5000
|
classification = "auth";
|
|
5001
|
+
else if (summary = providerUnavailableSummary(rawText))
|
|
5002
|
+
classification = "provider_unavailable";
|
|
4494
5003
|
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
4495
5004
|
classification = "model_not_found";
|
|
4496
5005
|
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
@@ -4503,8 +5012,9 @@ function classifyRunFailure(run) {
|
|
|
4503
5012
|
classification = "sigsegv";
|
|
4504
5013
|
return {
|
|
4505
5014
|
classification,
|
|
4506
|
-
fingerprint: stableFailureFingerprint(run, classification),
|
|
5015
|
+
fingerprint: stableFailureFingerprint(run, classification, summary),
|
|
4507
5016
|
evidence: {
|
|
5017
|
+
summary,
|
|
4508
5018
|
error: redactedEvidence(run.error),
|
|
4509
5019
|
stdout: redactedEvidence(run.stdout),
|
|
4510
5020
|
stderr: redactedEvidence(run.stderr),
|
|
@@ -4549,7 +5059,7 @@ function routeEvidenceReport(run) {
|
|
|
4549
5059
|
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
4550
5060
|
if (evidencePath && existsSync2(evidencePath)) {
|
|
4551
5061
|
try {
|
|
4552
|
-
return parseJsonObject(
|
|
5062
|
+
return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
|
|
4553
5063
|
} catch {
|
|
4554
5064
|
return stdoutReport;
|
|
4555
5065
|
}
|
|
@@ -4681,6 +5191,12 @@ function targetRoute(loop) {
|
|
|
4681
5191
|
loopName: loop.name
|
|
4682
5192
|
};
|
|
4683
5193
|
}
|
|
5194
|
+
function expectationLoop(loop) {
|
|
5195
|
+
return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
|
|
5196
|
+
}
|
|
5197
|
+
function hasPendingRetry(loop, run) {
|
|
5198
|
+
return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
|
|
5199
|
+
}
|
|
4684
5200
|
function recommendedTask(loop, run, failure, route) {
|
|
4685
5201
|
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
4686
5202
|
const description = [
|
|
@@ -4692,6 +5208,7 @@ function recommendedTask(loop, run, failure, route) {
|
|
|
4692
5208
|
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
4693
5209
|
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
4694
5210
|
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
5211
|
+
failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
|
|
4695
5212
|
failure.evidence.error ? `Error:
|
|
4696
5213
|
${failure.evidence.error}` : undefined,
|
|
4697
5214
|
failure.evidence.stderr ? `Stderr:
|
|
@@ -4701,7 +5218,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
4701
5218
|
`);
|
|
4702
5219
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
4703
5220
|
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
4704
|
-
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
5221
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
|
|
4705
5222
|
return {
|
|
4706
5223
|
title,
|
|
4707
5224
|
description,
|
|
@@ -4735,7 +5252,7 @@ function expectationForLoop(store, loop) {
|
|
|
4735
5252
|
const route = targetRoute(loop);
|
|
4736
5253
|
if (!latestRun) {
|
|
4737
5254
|
return {
|
|
4738
|
-
loop:
|
|
5255
|
+
loop: expectationLoop(loop),
|
|
4739
5256
|
ok: true,
|
|
4740
5257
|
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
4741
5258
|
route
|
|
@@ -4744,7 +5261,7 @@ function expectationForLoop(store, loop) {
|
|
|
4744
5261
|
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
4745
5262
|
if (routeFailure) {
|
|
4746
5263
|
return {
|
|
4747
|
-
loop:
|
|
5264
|
+
loop: expectationLoop(loop),
|
|
4748
5265
|
ok: false,
|
|
4749
5266
|
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
4750
5267
|
latestRun: healthRun(latestRun),
|
|
@@ -4755,7 +5272,7 @@ function expectationForLoop(store, loop) {
|
|
|
4755
5272
|
}
|
|
4756
5273
|
if (latestRun.status === "succeeded") {
|
|
4757
5274
|
return {
|
|
4758
|
-
loop:
|
|
5275
|
+
loop: expectationLoop(loop),
|
|
4759
5276
|
ok: true,
|
|
4760
5277
|
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
4761
5278
|
latestRun: healthRun(latestRun),
|
|
@@ -4766,7 +5283,7 @@ function expectationForLoop(store, loop) {
|
|
|
4766
5283
|
if (failure?.classification === "circuit_breaker") {
|
|
4767
5284
|
if (loop.status !== "paused") {
|
|
4768
5285
|
return {
|
|
4769
|
-
loop:
|
|
5286
|
+
loop: expectationLoop(loop),
|
|
4770
5287
|
ok: true,
|
|
4771
5288
|
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
4772
5289
|
latestRun: healthRun(latestRun),
|
|
@@ -4774,7 +5291,7 @@ function expectationForLoop(store, loop) {
|
|
|
4774
5291
|
};
|
|
4775
5292
|
}
|
|
4776
5293
|
return {
|
|
4777
|
-
loop:
|
|
5294
|
+
loop: expectationLoop(loop),
|
|
4778
5295
|
ok: false,
|
|
4779
5296
|
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
4780
5297
|
latestRun: healthRun(latestRun),
|
|
@@ -4783,8 +5300,33 @@ function expectationForLoop(store, loop) {
|
|
|
4783
5300
|
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
4784
5301
|
};
|
|
4785
5302
|
}
|
|
5303
|
+
if (failure?.classification === "restart_interrupted") {
|
|
5304
|
+
return {
|
|
5305
|
+
loop: expectationLoop(loop),
|
|
5306
|
+
ok: true,
|
|
5307
|
+
check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
|
|
5308
|
+
latestRun: healthRun(latestRun),
|
|
5309
|
+
failure,
|
|
5310
|
+
route
|
|
5311
|
+
};
|
|
5312
|
+
}
|
|
5313
|
+
if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
|
|
5314
|
+
const message = [
|
|
5315
|
+
"provider unavailable/network failure; retry is scheduled",
|
|
5316
|
+
loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
|
|
5317
|
+
failure.evidence.summary
|
|
5318
|
+
].filter(Boolean).join("; ");
|
|
5319
|
+
return {
|
|
5320
|
+
loop: expectationLoop(loop),
|
|
5321
|
+
ok: true,
|
|
5322
|
+
check: { id: "latest-run-succeeded", status: "warn", message },
|
|
5323
|
+
latestRun: healthRun(latestRun),
|
|
5324
|
+
failure,
|
|
5325
|
+
route
|
|
5326
|
+
};
|
|
5327
|
+
}
|
|
4786
5328
|
return {
|
|
4787
|
-
loop:
|
|
5329
|
+
loop: expectationLoop(loop),
|
|
4788
5330
|
ok: false,
|
|
4789
5331
|
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
4790
5332
|
latestRun: healthRun(latestRun),
|
|
@@ -4816,16 +5358,99 @@ function buildHealthReport(store, opts = {}) {
|
|
|
4816
5358
|
expectations
|
|
4817
5359
|
};
|
|
4818
5360
|
}
|
|
5361
|
+
function buildHealthScan(store, opts = {}) {
|
|
5362
|
+
const generatedAt = (opts.now ?? new Date).toISOString();
|
|
5363
|
+
const now = opts.now ?? new Date(generatedAt);
|
|
5364
|
+
const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
|
|
5365
|
+
const loops = scanLoops(store, includeStatuses, opts);
|
|
5366
|
+
const health = healthReportForLoops(store, loops, generatedAt);
|
|
5367
|
+
const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
|
|
5368
|
+
const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
|
|
5369
|
+
const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
|
|
5370
|
+
const allFindings = [];
|
|
5371
|
+
const pushFinding = (finding) => {
|
|
5372
|
+
if (!finding)
|
|
5373
|
+
return;
|
|
5374
|
+
allFindings.push(finding);
|
|
5375
|
+
};
|
|
5376
|
+
if (opts.daemon)
|
|
5377
|
+
pushFinding(daemonFinding(opts.daemon));
|
|
5378
|
+
if (opts.latestRun !== false) {
|
|
5379
|
+
for (const loop of loops) {
|
|
5380
|
+
const expectation = expectationsByLoopId.get(loop.id);
|
|
5381
|
+
if (!expectation)
|
|
5382
|
+
continue;
|
|
5383
|
+
pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
|
|
5384
|
+
pushFinding(latestRunFinding(expectation));
|
|
5385
|
+
}
|
|
5386
|
+
}
|
|
5387
|
+
const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
|
|
5388
|
+
if (doctor) {
|
|
5389
|
+
for (const check of doctor.checks) {
|
|
5390
|
+
const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
|
|
5391
|
+
const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
|
|
5392
|
+
const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
|
|
5393
|
+
pushFinding(doctorFinding(check, loop, route));
|
|
5394
|
+
}
|
|
5395
|
+
}
|
|
5396
|
+
const findings = allFindings.slice(0, maxFindings);
|
|
5397
|
+
const status = scanStatus(allFindings);
|
|
5398
|
+
const baseCounts = statusCounts(loops);
|
|
5399
|
+
return {
|
|
5400
|
+
ok: status === "ok",
|
|
5401
|
+
status,
|
|
5402
|
+
generatedAt,
|
|
5403
|
+
includedStatuses: includeStatuses,
|
|
5404
|
+
counts: {
|
|
5405
|
+
...baseCounts,
|
|
5406
|
+
latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
|
|
5407
|
+
staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
|
|
5408
|
+
daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
|
|
5409
|
+
doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
|
|
5410
|
+
preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
|
|
5411
|
+
findings: allFindings.length,
|
|
5412
|
+
reportedFindings: findings.length,
|
|
5413
|
+
truncatedFindings: Math.max(0, allFindings.length - findings.length)
|
|
5414
|
+
},
|
|
5415
|
+
daemon: opts.daemon ? {
|
|
5416
|
+
running: opts.daemon.running,
|
|
5417
|
+
stale: opts.daemon.stale,
|
|
5418
|
+
pid: opts.daemon.pid,
|
|
5419
|
+
host: opts.daemon.host,
|
|
5420
|
+
loops: opts.daemon.loops,
|
|
5421
|
+
runs: opts.daemon.runs,
|
|
5422
|
+
logPath: opts.daemon.logPath
|
|
5423
|
+
} : undefined,
|
|
5424
|
+
doctor,
|
|
5425
|
+
health,
|
|
5426
|
+
selfHeals: opts.selfHeals ?? [],
|
|
5427
|
+
findings
|
|
5428
|
+
};
|
|
5429
|
+
}
|
|
5430
|
+
function writeHealthScanReports(scan, opts = {}) {
|
|
5431
|
+
const root = opts.reportDir ?? join5(dataDir(), "reports", "health-scan");
|
|
5432
|
+
const dir = timestampDir(root, scan.generatedAt);
|
|
5433
|
+
mkdirSync4(dir, { recursive: true, mode: 448 });
|
|
5434
|
+
const json = join5(dir, "summary.json");
|
|
5435
|
+
const markdown = join5(dir, "report.md");
|
|
5436
|
+
const withReports = {
|
|
5437
|
+
...scan,
|
|
5438
|
+
reports: { dir, json, markdown }
|
|
5439
|
+
};
|
|
5440
|
+
writeFileSync2(json, JSON.stringify(withReports, null, 2), { mode: 384 });
|
|
5441
|
+
writeFileSync2(markdown, healthScanMarkdown(withReports), { mode: 384 });
|
|
5442
|
+
return withReports;
|
|
5443
|
+
}
|
|
4819
5444
|
|
|
4820
5445
|
// src/lib/executor.ts
|
|
4821
|
-
import { spawn as spawn2, spawnSync as
|
|
5446
|
+
import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
|
|
4822
5447
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
4823
5448
|
import { once } from "events";
|
|
4824
|
-
import { lstatSync, mkdirSync as
|
|
5449
|
+
import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
|
|
4825
5450
|
import { dirname as dirname3, resolve as resolve2 } from "path";
|
|
4826
5451
|
|
|
4827
5452
|
// src/lib/accounts.ts
|
|
4828
|
-
import { spawnSync as
|
|
5453
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
4829
5454
|
import { existsSync as existsSync3 } from "fs";
|
|
4830
5455
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4831
5456
|
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
@@ -4930,7 +5555,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4930
5555
|
if (!account)
|
|
4931
5556
|
return {};
|
|
4932
5557
|
const tool = requiredAccountTool(account, toolHint);
|
|
4933
|
-
const result =
|
|
5558
|
+
const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
|
|
4934
5559
|
encoding: "utf8",
|
|
4935
5560
|
env,
|
|
4936
5561
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -4946,8 +5571,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4946
5571
|
|
|
4947
5572
|
// src/lib/env.ts
|
|
4948
5573
|
import { accessSync, constants } from "fs";
|
|
4949
|
-
import { homedir as
|
|
4950
|
-
import { delimiter, join as
|
|
5574
|
+
import { homedir as homedir3 } from "os";
|
|
5575
|
+
import { delimiter, join as join6 } from "path";
|
|
4951
5576
|
function compactPathParts(parts) {
|
|
4952
5577
|
const seen = new Set;
|
|
4953
5578
|
const result = [];
|
|
@@ -4961,16 +5586,16 @@ function compactPathParts(parts) {
|
|
|
4961
5586
|
return result;
|
|
4962
5587
|
}
|
|
4963
5588
|
function commonExecutableDirs(env = process.env) {
|
|
4964
|
-
const home = env.HOME ||
|
|
5589
|
+
const home = env.HOME || homedir3();
|
|
4965
5590
|
return compactPathParts([
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
env.BUN_INSTALL ?
|
|
5591
|
+
join6(home, ".local", "bin"),
|
|
5592
|
+
join6(home, ".bun", "bin"),
|
|
5593
|
+
join6(home, ".cargo", "bin"),
|
|
5594
|
+
join6(home, ".npm-global", "bin"),
|
|
5595
|
+
join6(home, "bin"),
|
|
5596
|
+
env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
|
|
4972
5597
|
env.PNPM_HOME,
|
|
4973
|
-
env.NPM_CONFIG_PREFIX ?
|
|
5598
|
+
env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
4974
5599
|
"/opt/homebrew/bin",
|
|
4975
5600
|
"/usr/local/bin",
|
|
4976
5601
|
"/usr/bin",
|
|
@@ -4994,7 +5619,7 @@ function executableExists(command, env = process.env) {
|
|
|
4994
5619
|
if (command.includes("/"))
|
|
4995
5620
|
return isExecutable(command);
|
|
4996
5621
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
4997
|
-
if (dir && isExecutable(
|
|
5622
|
+
if (dir && isExecutable(join6(dir, command)))
|
|
4998
5623
|
return true;
|
|
4999
5624
|
}
|
|
5000
5625
|
return false;
|
|
@@ -5135,6 +5760,32 @@ function timeoutResult(startedAt, error, fields = {}) {
|
|
|
5135
5760
|
function successResult(startedAt, fields = {}) {
|
|
5136
5761
|
return buildResult("succeeded", startedAt, fields);
|
|
5137
5762
|
}
|
|
5763
|
+
function codewithJsonlHasTerminalSuccess(stdout) {
|
|
5764
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
5765
|
+
const trimmed = line.trim();
|
|
5766
|
+
if (!trimmed || !trimmed.startsWith("{"))
|
|
5767
|
+
continue;
|
|
5768
|
+
let event;
|
|
5769
|
+
try {
|
|
5770
|
+
event = JSON.parse(trimmed);
|
|
5771
|
+
} catch {
|
|
5772
|
+
continue;
|
|
5773
|
+
}
|
|
5774
|
+
if (!event || typeof event !== "object")
|
|
5775
|
+
continue;
|
|
5776
|
+
const record = event;
|
|
5777
|
+
if (record.type === "task_complete")
|
|
5778
|
+
return true;
|
|
5779
|
+
const payload = record.payload;
|
|
5780
|
+
if (payload && typeof payload === "object" && payload.type === "task_complete") {
|
|
5781
|
+
return true;
|
|
5782
|
+
}
|
|
5783
|
+
}
|
|
5784
|
+
return false;
|
|
5785
|
+
}
|
|
5786
|
+
function codewithJsonlReconciledSuccess(spec, fields) {
|
|
5787
|
+
return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
|
|
5788
|
+
}
|
|
5138
5789
|
function notifySpawn(pid, opts) {
|
|
5139
5790
|
if (!pid)
|
|
5140
5791
|
return;
|
|
@@ -5150,10 +5801,48 @@ function shellQuote(value) {
|
|
|
5150
5801
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
5151
5802
|
}
|
|
5152
5803
|
function codewithProfileCandidateFromLine(line) {
|
|
5153
|
-
const
|
|
5154
|
-
if (
|
|
5155
|
-
return
|
|
5156
|
-
|
|
5804
|
+
const trimmed = line.trim();
|
|
5805
|
+
if (!trimmed || trimmed === "No auth profiles saved.")
|
|
5806
|
+
return;
|
|
5807
|
+
const cols = trimmed.split(/\s+/);
|
|
5808
|
+
const candidate = cols[0] === "*" ? cols[1] : cols[0];
|
|
5809
|
+
if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
|
|
5810
|
+
return;
|
|
5811
|
+
if (candidate === '"name":') {
|
|
5812
|
+
const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
|
|
5813
|
+
if (jsonName)
|
|
5814
|
+
return jsonName;
|
|
5815
|
+
}
|
|
5816
|
+
return candidate;
|
|
5817
|
+
}
|
|
5818
|
+
function codewithProfileCandidatesFromJson(value) {
|
|
5819
|
+
const candidates = [];
|
|
5820
|
+
const root = value && typeof value === "object" ? value : undefined;
|
|
5821
|
+
const addProfile = (entry) => {
|
|
5822
|
+
if (!entry || typeof entry !== "object")
|
|
5823
|
+
return;
|
|
5824
|
+
const name = entry.name;
|
|
5825
|
+
if (typeof name === "string" && name)
|
|
5826
|
+
candidates.push(name);
|
|
5827
|
+
};
|
|
5828
|
+
const data = root?.data;
|
|
5829
|
+
if (Array.isArray(data))
|
|
5830
|
+
data.forEach(addProfile);
|
|
5831
|
+
const profiles = root?.profiles;
|
|
5832
|
+
if (Array.isArray(profiles))
|
|
5833
|
+
profiles.forEach(addProfile);
|
|
5834
|
+
return candidates;
|
|
5835
|
+
}
|
|
5836
|
+
function codewithProfileCandidatesFromOutput(output) {
|
|
5837
|
+
const trimmed = output.trim();
|
|
5838
|
+
const jsonStart = trimmed.indexOf("{");
|
|
5839
|
+
const jsonEnd = trimmed.lastIndexOf("}");
|
|
5840
|
+
if (jsonStart >= 0 && jsonEnd >= jsonStart) {
|
|
5841
|
+
try {
|
|
5842
|
+
return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
|
|
5843
|
+
} catch {}
|
|
5844
|
+
}
|
|
5845
|
+
return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
|
|
5157
5846
|
}
|
|
5158
5847
|
function codewithProfileForError(profile) {
|
|
5159
5848
|
return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
|
|
@@ -5246,6 +5935,7 @@ function commandSpec(target, opts) {
|
|
|
5246
5935
|
preflightAnyOf: invocation.preflightAnyOf,
|
|
5247
5936
|
stdin: invocation.stdin,
|
|
5248
5937
|
allowlist: agentTarget.allowlist,
|
|
5938
|
+
agentProvider: agentTarget.provider,
|
|
5249
5939
|
worktree: agentTarget.worktree
|
|
5250
5940
|
};
|
|
5251
5941
|
}
|
|
@@ -5405,7 +6095,7 @@ function remotePreflightScript(spec, metadata) {
|
|
|
5405
6095
|
}
|
|
5406
6096
|
if (spec.nativeAuthProfile?.provider === "codewith") {
|
|
5407
6097
|
const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
|
|
5408
|
-
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE",
|
|
6098
|
+
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
|
|
5409
6099
|
}
|
|
5410
6100
|
return lines.join(`
|
|
5411
6101
|
`);
|
|
@@ -5423,6 +6113,12 @@ function transportEnv(opts) {
|
|
|
5423
6113
|
return env;
|
|
5424
6114
|
}
|
|
5425
6115
|
function assertCodewithProfileListed(profile, result) {
|
|
6116
|
+
const profiles = codewithProfileSetFromResult(result);
|
|
6117
|
+
if (!profiles.has(profile)) {
|
|
6118
|
+
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
6119
|
+
}
|
|
6120
|
+
}
|
|
6121
|
+
function codewithProfileSetFromResult(result) {
|
|
5426
6122
|
if (result.error) {
|
|
5427
6123
|
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
5428
6124
|
}
|
|
@@ -5430,32 +6126,42 @@ function assertCodewithProfileListed(profile, result) {
|
|
|
5430
6126
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5431
6127
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
5432
6128
|
}
|
|
5433
|
-
|
|
5434
|
-
if (!profiles.has(profile)) {
|
|
5435
|
-
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5436
|
-
}
|
|
6129
|
+
return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
|
|
5437
6130
|
}
|
|
5438
6131
|
function preflightNativeAuthProfileSync(spec, env) {
|
|
5439
6132
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5440
6133
|
return;
|
|
5441
|
-
const
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
6134
|
+
const runProfileList = (args) => {
|
|
6135
|
+
const result = spawnSync4(spec.command, args, {
|
|
6136
|
+
encoding: "utf8",
|
|
6137
|
+
env,
|
|
6138
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
6139
|
+
timeout: 15000
|
|
6140
|
+
});
|
|
6141
|
+
return {
|
|
6142
|
+
status: result.status,
|
|
6143
|
+
stdout: result.stdout ?? "",
|
|
6144
|
+
stderr: result.stderr ?? "",
|
|
6145
|
+
error: result.error?.message
|
|
6146
|
+
};
|
|
6147
|
+
};
|
|
6148
|
+
const jsonResult = runProfileList(["profile", "list", "--json"]);
|
|
6149
|
+
try {
|
|
6150
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
6151
|
+
return;
|
|
6152
|
+
} catch {}
|
|
6153
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
|
|
5453
6154
|
}
|
|
5454
6155
|
async function preflightNativeAuthProfile(spec, env) {
|
|
5455
6156
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5456
6157
|
return;
|
|
5457
|
-
const
|
|
5458
|
-
|
|
6158
|
+
const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
|
|
6159
|
+
try {
|
|
6160
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
6161
|
+
return;
|
|
6162
|
+
} catch {}
|
|
6163
|
+
const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
6164
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
|
|
5459
6165
|
}
|
|
5460
6166
|
function resolvedDirEquals(left, right) {
|
|
5461
6167
|
try {
|
|
@@ -5516,7 +6222,7 @@ async function ensureLocalWorktree(worktree, env) {
|
|
|
5516
6222
|
return { error: `worktree repoRoot is not a git repository: ${repoRoot}` };
|
|
5517
6223
|
}
|
|
5518
6224
|
try {
|
|
5519
|
-
|
|
6225
|
+
mkdirSync5(dirname3(path), { recursive: true });
|
|
5520
6226
|
} catch (error) {
|
|
5521
6227
|
return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
|
|
5522
6228
|
}
|
|
@@ -5558,7 +6264,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
|
5558
6264
|
}
|
|
5559
6265
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
5560
6266
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
5561
|
-
const result =
|
|
6267
|
+
const result = spawnSync4(plan.command, plan.args, {
|
|
5562
6268
|
encoding: "utf8",
|
|
5563
6269
|
env: transportEnv(opts),
|
|
5564
6270
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -5657,6 +6363,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
|
5657
6363
|
if (timedOut || idleTimedOut) {
|
|
5658
6364
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5659
6365
|
}
|
|
6366
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6367
|
+
return successResult(startedAt, fields);
|
|
6368
|
+
}
|
|
5660
6369
|
if (error || exitCode !== 0) {
|
|
5661
6370
|
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5662
6371
|
}
|
|
@@ -5792,6 +6501,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
5792
6501
|
if (timedOut || idleTimedOut) {
|
|
5793
6502
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5794
6503
|
}
|
|
6504
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6505
|
+
return successResult(startedAt, fields);
|
|
6506
|
+
}
|
|
5795
6507
|
if (error || exitCode !== 0) {
|
|
5796
6508
|
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5797
6509
|
}
|
|
@@ -6820,7 +7532,7 @@ var MAX_RETRY_EXPONENT = 20;
|
|
|
6820
7532
|
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
6821
7533
|
const attempt = Math.max(1, run.attempt);
|
|
6822
7534
|
const failure = classifyRunFailure(run);
|
|
6823
|
-
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
7535
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
|
|
6824
7536
|
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
6825
7537
|
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
6826
7538
|
const jitter = 0.5 + random();
|
|
@@ -6947,20 +7659,21 @@ async function executeClaimedRun(deps) {
|
|
|
6947
7659
|
daemonLeaseId: deps.daemonLeaseId,
|
|
6948
7660
|
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
6949
7661
|
})))(deps.loop, deps.run);
|
|
7662
|
+
const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
|
|
6950
7663
|
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
6951
7664
|
return deps.store.finalizeRun(deps.run.id, {
|
|
6952
|
-
status:
|
|
6953
|
-
finishedAt:
|
|
6954
|
-
durationMs:
|
|
6955
|
-
stdout:
|
|
6956
|
-
stderr:
|
|
6957
|
-
exitCode:
|
|
6958
|
-
error:
|
|
6959
|
-
pid:
|
|
7665
|
+
status: finalResult.status,
|
|
7666
|
+
finishedAt: finalResult.finishedAt,
|
|
7667
|
+
durationMs: finalResult.durationMs,
|
|
7668
|
+
stdout: finalResult.stdout,
|
|
7669
|
+
stderr: finalResult.stderr,
|
|
7670
|
+
exitCode: finalResult.exitCode,
|
|
7671
|
+
error: finalResult.error,
|
|
7672
|
+
pid: finalResult.pid
|
|
6960
7673
|
}, {
|
|
6961
7674
|
claimedBy: deps.runnerId,
|
|
6962
7675
|
daemonLeaseId: deps.daemonLeaseId,
|
|
6963
|
-
now: deps.now?.() ?? new Date(
|
|
7676
|
+
now: deps.now?.() ?? new Date(finalResult.finishedAt)
|
|
6964
7677
|
});
|
|
6965
7678
|
} catch (err) {
|
|
6966
7679
|
deps.onError?.(deps.loop, err);
|
|
@@ -7195,10 +7908,10 @@ async function tick(deps) {
|
|
|
7195
7908
|
}
|
|
7196
7909
|
|
|
7197
7910
|
// src/daemon/control.ts
|
|
7198
|
-
import { existsSync as existsSync4, mkdirSync as
|
|
7199
|
-
import { spawnSync as
|
|
7911
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, rmSync as rmSync4, writeFileSync as writeFileSync3 } from "fs";
|
|
7912
|
+
import { spawnSync as spawnSync5 } from "child_process";
|
|
7200
7913
|
import { hostname } from "os";
|
|
7201
|
-
import { dirname as dirname4, join as
|
|
7914
|
+
import { dirname as dirname4, join as join7 } from "path";
|
|
7202
7915
|
|
|
7203
7916
|
// src/daemon/loop.ts
|
|
7204
7917
|
function realSleep(ms) {
|
|
@@ -7228,7 +7941,7 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
7228
7941
|
return;
|
|
7229
7942
|
let raw;
|
|
7230
7943
|
try {
|
|
7231
|
-
raw =
|
|
7944
|
+
raw = readFileSync5(path, "utf8").trim();
|
|
7232
7945
|
} catch {
|
|
7233
7946
|
return;
|
|
7234
7947
|
}
|
|
@@ -7250,11 +7963,11 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
7250
7963
|
return Number.isInteger(pid) && pid > 0 ? { pid } : undefined;
|
|
7251
7964
|
}
|
|
7252
7965
|
function writePid(pid = process.pid, path = pidFilePath()) {
|
|
7253
|
-
|
|
7254
|
-
|
|
7966
|
+
mkdirSync6(dirname4(path), { recursive: true, mode: 448 });
|
|
7967
|
+
writeFileSync3(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
7255
7968
|
}
|
|
7256
7969
|
function removePid(path = pidFilePath()) {
|
|
7257
|
-
|
|
7970
|
+
rmSync4(path, { force: true });
|
|
7258
7971
|
}
|
|
7259
7972
|
function isAlive(pid, startedAt) {
|
|
7260
7973
|
try {
|
|
@@ -7284,7 +7997,7 @@ function ownProcessGroupId() {
|
|
|
7284
7997
|
return pgrp;
|
|
7285
7998
|
}
|
|
7286
7999
|
try {
|
|
7287
|
-
const run =
|
|
8000
|
+
const run = spawnSync5("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
7288
8001
|
const pgid = Number(run.stdout.trim());
|
|
7289
8002
|
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
7290
8003
|
return pgid;
|
|
@@ -7380,7 +8093,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7380
8093
|
if (!state.running || !state.pid)
|
|
7381
8094
|
return { wasRunning: false, stopped: false, forced: false };
|
|
7382
8095
|
const sleep = opts.sleep ?? realSleep;
|
|
7383
|
-
const store = new Store(
|
|
8096
|
+
const store = new Store(join7(dirname4(path), "loops.db"));
|
|
7384
8097
|
try {
|
|
7385
8098
|
const lease = store.getDaemonLease();
|
|
7386
8099
|
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
@@ -7457,7 +8170,7 @@ function rotateDaemonLog(path = daemonLogPath(), maxBytes = DAEMON_LOG_MAX_BYTES
|
|
|
7457
8170
|
if (size < maxBytes)
|
|
7458
8171
|
return false;
|
|
7459
8172
|
try {
|
|
7460
|
-
|
|
8173
|
+
rmSync5(`${path}.${keep}`, { force: true });
|
|
7461
8174
|
for (let i = keep - 1;i >= 1; i--) {
|
|
7462
8175
|
if (existsSync5(`${path}.${i}`))
|
|
7463
8176
|
renameSync2(`${path}.${i}`, `${path}.${i + 1}`);
|
|
@@ -7506,6 +8219,7 @@ async function runDaemon(opts = {}) {
|
|
|
7506
8219
|
let runAbort = new AbortController;
|
|
7507
8220
|
const activeRuns = new Map;
|
|
7508
8221
|
const activeByLane = { command: 0, agent: 0 };
|
|
8222
|
+
const isControlledStopInterruption = (result) => stopFlag && !leaseLost && result.status === "failed" && result.error?.includes("terminated by SIGTERM") === true;
|
|
7509
8223
|
const requestStop = (message) => {
|
|
7510
8224
|
stopFlag = true;
|
|
7511
8225
|
if (!runAbort.signal.aborted)
|
|
@@ -7589,6 +8303,16 @@ async function runDaemon(opts = {}) {
|
|
|
7589
8303
|
store.recordRunProcess(run.id, info, { daemonLeaseId: leaseId });
|
|
7590
8304
|
}
|
|
7591
8305
|
})),
|
|
8306
|
+
finalizeResult: (result) => {
|
|
8307
|
+
if (!isControlledStopInterruption(result))
|
|
8308
|
+
return result;
|
|
8309
|
+
return {
|
|
8310
|
+
...result,
|
|
8311
|
+
status: "skipped",
|
|
8312
|
+
exitCode: undefined,
|
|
8313
|
+
error: `${RESTART_INTERRUPTED_RUN_PREFIX}: child process terminated by SIGTERM during daemon stop/restart`
|
|
8314
|
+
};
|
|
8315
|
+
},
|
|
7592
8316
|
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
7593
8317
|
});
|
|
7594
8318
|
ensureLease();
|
|
@@ -7682,7 +8406,7 @@ async function startDaemon(opts) {
|
|
|
7682
8406
|
return { started: false, alreadyRunning: true, pid: state.pid };
|
|
7683
8407
|
if (state.stale)
|
|
7684
8408
|
removePid();
|
|
7685
|
-
const out =
|
|
8409
|
+
const out = openSync2(daemonLogPath(), "a");
|
|
7686
8410
|
const child = spawn3(opts.execPath ?? process.execPath, [opts.cliEntry, ...opts.args ?? ["daemon", "run"]], {
|
|
7687
8411
|
detached: true,
|
|
7688
8412
|
stdio: ["ignore", out, out]
|
|
@@ -7700,8 +8424,8 @@ async function startDaemon(opts) {
|
|
|
7700
8424
|
}
|
|
7701
8425
|
|
|
7702
8426
|
// src/daemon/install.ts
|
|
7703
|
-
import { chmodSync as chmodSync2, mkdirSync as
|
|
7704
|
-
import { spawnSync as
|
|
8427
|
+
import { chmodSync as chmodSync2, mkdirSync as mkdirSync7, writeFileSync as writeFileSync4 } from "fs";
|
|
8428
|
+
import { spawnSync as spawnSync6 } from "child_process";
|
|
7705
8429
|
import { dirname as dirname5 } from "path";
|
|
7706
8430
|
function systemdEscapeExecPart(part) {
|
|
7707
8431
|
const escaped = part.replaceAll("%", "%%");
|
|
@@ -7733,9 +8457,9 @@ function installStartup(cliEntry, execPath = process.execPath, args = ["daemon",
|
|
|
7733
8457
|
const dataDirPath = ensureDataDir();
|
|
7734
8458
|
if (platform === "linux") {
|
|
7735
8459
|
const path = systemdServicePath();
|
|
7736
|
-
|
|
8460
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 448 });
|
|
7737
8461
|
const execStart = [execPath, cliEntry, ...args].map(systemdEscapeExecPart).join(" ");
|
|
7738
|
-
|
|
8462
|
+
writeFileSync4(path, `[Unit]
|
|
7739
8463
|
Description=Hasna OpenLoops daemon
|
|
7740
8464
|
After=basic.target
|
|
7741
8465
|
|
|
@@ -7763,8 +8487,8 @@ WantedBy=default.target
|
|
|
7763
8487
|
}
|
|
7764
8488
|
if (platform === "darwin") {
|
|
7765
8489
|
const path = launchdPlistPath();
|
|
7766
|
-
|
|
7767
|
-
|
|
8490
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 448 });
|
|
8491
|
+
writeFileSync4(path, `<?xml version="1.0" encoding="UTF-8"?>
|
|
7768
8492
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
7769
8493
|
<plist version="1.0">
|
|
7770
8494
|
<dict>
|
|
@@ -7801,7 +8525,7 @@ ${args.map((arg) => ` <string>${xmlEscape(arg)}</string>`).join(`
|
|
|
7801
8525
|
function enableStartup(result) {
|
|
7802
8526
|
const commands = result.platform === "linux" ? ["systemctl --user daemon-reload", "systemctl --user enable --now loops-daemon.service"] : result.platform === "darwin" ? launchctlCommands(result.path) : [];
|
|
7803
8527
|
return commands.map((command) => {
|
|
7804
|
-
const run =
|
|
8528
|
+
const run = spawnSync6("sh", ["-c", command], {
|
|
7805
8529
|
encoding: "utf8",
|
|
7806
8530
|
stdio: ["ignore", "pipe", "pipe"]
|
|
7807
8531
|
});
|
|
@@ -7816,7 +8540,7 @@ function enableStartup(result) {
|
|
|
7816
8540
|
// package.json
|
|
7817
8541
|
var package_default = {
|
|
7818
8542
|
name: "@hasna/loops",
|
|
7819
|
-
version: "0.4.
|
|
8543
|
+
version: "0.4.14",
|
|
7820
8544
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
7821
8545
|
type: "module",
|
|
7822
8546
|
main: "dist/index.js",
|
|
@@ -7825,6 +8549,7 @@ var package_default = {
|
|
|
7825
8549
|
loops: "dist/cli/index.js",
|
|
7826
8550
|
"loops-daemon": "dist/daemon/index.js",
|
|
7827
8551
|
"loops-api": "dist/api/index.js",
|
|
8552
|
+
"loops-serve": "dist/serve/index.js",
|
|
7828
8553
|
"loops-runner": "dist/runner/index.js",
|
|
7829
8554
|
"loops-mcp": "dist/mcp/index.js"
|
|
7830
8555
|
},
|
|
@@ -7837,6 +8562,14 @@ var package_default = {
|
|
|
7837
8562
|
types: "./dist/sdk/index.d.ts",
|
|
7838
8563
|
import: "./dist/sdk/index.js"
|
|
7839
8564
|
},
|
|
8565
|
+
"./sdk/http": {
|
|
8566
|
+
types: "./dist/sdk/http.d.ts",
|
|
8567
|
+
import: "./dist/sdk/http.js"
|
|
8568
|
+
},
|
|
8569
|
+
"./serve": {
|
|
8570
|
+
types: "./dist/serve/index.d.ts",
|
|
8571
|
+
import: "./dist/serve/index.js"
|
|
8572
|
+
},
|
|
7840
8573
|
"./mcp": {
|
|
7841
8574
|
types: "./dist/mcp/index.d.ts",
|
|
7842
8575
|
import: "./dist/mcp/index.js"
|
|
@@ -7882,7 +8615,7 @@ var package_default = {
|
|
|
7882
8615
|
"LICENSE"
|
|
7883
8616
|
],
|
|
7884
8617
|
scripts: {
|
|
7885
|
-
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
8618
|
+
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
7886
8619
|
"build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
|
|
7887
8620
|
typecheck: "tsc --noEmit",
|
|
7888
8621
|
test: "bun test",
|
|
@@ -7917,11 +8650,13 @@ var package_default = {
|
|
|
7917
8650
|
bun: ">=1.0.0"
|
|
7918
8651
|
},
|
|
7919
8652
|
dependencies: {
|
|
8653
|
+
"@hasna/contracts": "^0.4.1",
|
|
7920
8654
|
"@hasna/events": "^0.1.9",
|
|
7921
8655
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
7922
8656
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
7923
8657
|
ai: "6.0.204",
|
|
7924
8658
|
commander: "^13.1.0",
|
|
8659
|
+
pg: "^8.13.1",
|
|
7925
8660
|
zod: "4.4.3"
|
|
7926
8661
|
},
|
|
7927
8662
|
optionalDependencies: {
|
|
@@ -7929,6 +8664,7 @@ var package_default = {
|
|
|
7929
8664
|
},
|
|
7930
8665
|
devDependencies: {
|
|
7931
8666
|
"@types/bun": "latest",
|
|
8667
|
+
"@types/pg": "^8.11.10",
|
|
7932
8668
|
typescript: "^5.7.3"
|
|
7933
8669
|
},
|
|
7934
8670
|
publishConfig: {
|