@hasna/loops 0.4.13 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -3
- package/README.md +54 -12
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +695 -10
- package/dist/cli/index.js +1633 -366
- package/dist/cli/ui.d.ts +44 -0
- package/dist/daemon/index.js +948 -212
- package/dist/generated/storage-kit/health.d.ts +19 -0
- package/dist/generated/storage-kit/index.d.ts +7 -0
- package/dist/generated/storage-kit/migrations.d.ts +47 -0
- package/dist/generated/storage-kit/mode.d.ts +47 -0
- package/dist/generated/storage-kit/pool.d.ts +33 -0
- package/dist/generated/storage-kit/query.d.ts +35 -0
- package/dist/generated/storage-kit/tls.d.ts +25 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3567 -2010
- package/dist/lib/health.d.ts +83 -3
- package/dist/lib/mode.js +14 -2
- package/dist/lib/scheduler.d.ts +5 -2
- package/dist/lib/storage/index.d.ts +1 -0
- package/dist/lib/storage/index.js +1329 -211
- package/dist/lib/storage/pg-executor.d.ts +27 -0
- package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
- package/dist/lib/storage/postgres-loop-storage.d.ts +114 -0
- package/dist/lib/storage/sqlite.js +336 -8
- package/dist/lib/store.d.ts +234 -0
- package/dist/lib/store.js +350 -8
- package/dist/mcp/index.js +1067 -306
- package/dist/runner/index.js +124 -25
- package/dist/sdk/http.d.ts +115 -0
- package/dist/sdk/http.js +145 -0
- package/dist/sdk/index.d.ts +5 -1
- package/dist/sdk/index.js +1052 -312
- package/dist/serve/index.d.ts +4 -0
- package/dist/serve/index.js +7619 -0
- package/dist/types.d.ts +3 -0
- package/docs/CUTOVER-RUNBOOK.md +61 -0
- package/docs/USAGE.md +50 -11
- package/package.json +14 -2
package/dist/cli/index.js
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
|
|
4
4
|
// src/lib/store.ts
|
|
5
5
|
import { Database } from "bun:sqlite";
|
|
6
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as
|
|
7
|
-
import { tmpdir } from "os";
|
|
8
|
-
import { basename as basename2, dirname as dirname2, join as
|
|
6
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
|
|
7
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
8
|
+
import { basename as basename2, dirname as dirname2, join as join4 } from "path";
|
|
9
9
|
|
|
10
10
|
// src/lib/errors.ts
|
|
11
11
|
class CodedError extends Error {
|
|
@@ -1204,6 +1204,196 @@ function discardWorkflowRunManifest(staged) {
|
|
|
1204
1204
|
} catch {}
|
|
1205
1205
|
}
|
|
1206
1206
|
|
|
1207
|
+
// src/lib/route/todos-cli.ts
|
|
1208
|
+
import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
|
|
1209
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1210
|
+
import { join as join3 } from "path";
|
|
1211
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
1212
|
+
|
|
1213
|
+
// src/lib/format.ts
|
|
1214
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1215
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
1216
|
+
function redact(value, visible = 0) {
|
|
1217
|
+
if (!value)
|
|
1218
|
+
return value;
|
|
1219
|
+
if (value.length <= visible)
|
|
1220
|
+
return value;
|
|
1221
|
+
if (visible <= 0)
|
|
1222
|
+
return `[redacted ${value.length} chars]`;
|
|
1223
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1224
|
+
}
|
|
1225
|
+
function truncateTextOutput(value) {
|
|
1226
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1227
|
+
return value;
|
|
1228
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1229
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1230
|
+
}
|
|
1231
|
+
function redactSensitivePayload(value, key) {
|
|
1232
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
1233
|
+
if (typeof value === "string")
|
|
1234
|
+
return redact(value);
|
|
1235
|
+
if (value === undefined || value === null)
|
|
1236
|
+
return value;
|
|
1237
|
+
return "[redacted]";
|
|
1238
|
+
}
|
|
1239
|
+
if (Array.isArray(value))
|
|
1240
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
1241
|
+
if (value && typeof value === "object") {
|
|
1242
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
1243
|
+
}
|
|
1244
|
+
return value;
|
|
1245
|
+
}
|
|
1246
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1247
|
+
const indent = opts.indent ?? "";
|
|
1248
|
+
const nested = `${indent} `;
|
|
1249
|
+
const blocks = [];
|
|
1250
|
+
for (const [label, output] of [
|
|
1251
|
+
["stdout", value.stdout],
|
|
1252
|
+
["stderr", value.stderr]
|
|
1253
|
+
]) {
|
|
1254
|
+
if (!output)
|
|
1255
|
+
continue;
|
|
1256
|
+
blocks.push(`${indent}${label}:`);
|
|
1257
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1258
|
+
blocks.push(`${nested}${line}`);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
return blocks;
|
|
1262
|
+
}
|
|
1263
|
+
function publicLoop(loop) {
|
|
1264
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1265
|
+
return {
|
|
1266
|
+
...loop,
|
|
1267
|
+
target
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
function publicRun(run, showOutput = false, opts = {}) {
|
|
1271
|
+
return {
|
|
1272
|
+
...run,
|
|
1273
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1274
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1275
|
+
error: opts.redactError ? redact(run.error) : run.error
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
1279
|
+
return {
|
|
1280
|
+
...result,
|
|
1281
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
1282
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
1283
|
+
error: redact(result.error)
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
function publicWorkflow(workflow) {
|
|
1287
|
+
return {
|
|
1288
|
+
...workflow,
|
|
1289
|
+
steps: workflow.steps.map((step) => ({
|
|
1290
|
+
...step,
|
|
1291
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
1292
|
+
}))
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
function publicWorkflowRun(run) {
|
|
1296
|
+
return { ...run, error: redact(run.error) };
|
|
1297
|
+
}
|
|
1298
|
+
function publicWorkflowInvocation(invocation) {
|
|
1299
|
+
return redactSensitivePayload(invocation);
|
|
1300
|
+
}
|
|
1301
|
+
function publicWorkflowWorkItem(item) {
|
|
1302
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
1303
|
+
}
|
|
1304
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
1305
|
+
return {
|
|
1306
|
+
...run,
|
|
1307
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1308
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1309
|
+
error: redact(run.error)
|
|
1310
|
+
};
|
|
1311
|
+
}
|
|
1312
|
+
function publicWorkflowEvent(event) {
|
|
1313
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
1314
|
+
}
|
|
1315
|
+
function publicGoal(goal) {
|
|
1316
|
+
return {
|
|
1317
|
+
...goal,
|
|
1318
|
+
objective: redact(goal.objective, 120)
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
1321
|
+
function publicGoalRun(run) {
|
|
1322
|
+
return {
|
|
1323
|
+
...run,
|
|
1324
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
1325
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// src/lib/route/todos-cli.ts
|
|
1330
|
+
function defaultLoopsProject() {
|
|
1331
|
+
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
|
|
1332
|
+
}
|
|
1333
|
+
function runLocalCommand(command, args, opts = {}) {
|
|
1334
|
+
const result = spawnSync2(command, args, {
|
|
1335
|
+
input: opts.input,
|
|
1336
|
+
encoding: "utf8",
|
|
1337
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1338
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1339
|
+
env: process.env
|
|
1340
|
+
});
|
|
1341
|
+
return {
|
|
1342
|
+
ok: result.status === 0,
|
|
1343
|
+
status: result.status,
|
|
1344
|
+
stdout: result.stdout || "",
|
|
1345
|
+
stderr: result.stderr || "",
|
|
1346
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1347
|
+
};
|
|
1348
|
+
}
|
|
1349
|
+
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
1350
|
+
const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
|
|
1351
|
+
const stdoutPath = join3(tempDir, "stdout");
|
|
1352
|
+
const stdoutFd = openSync(stdoutPath, "w");
|
|
1353
|
+
let result;
|
|
1354
|
+
try {
|
|
1355
|
+
result = spawnSync2(command, args, {
|
|
1356
|
+
input: opts.input,
|
|
1357
|
+
encoding: "utf8",
|
|
1358
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1359
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1360
|
+
env: process.env,
|
|
1361
|
+
stdio: ["pipe", stdoutFd, "pipe"]
|
|
1362
|
+
});
|
|
1363
|
+
} finally {
|
|
1364
|
+
closeSync(stdoutFd);
|
|
1365
|
+
}
|
|
1366
|
+
try {
|
|
1367
|
+
return {
|
|
1368
|
+
ok: result.status === 0,
|
|
1369
|
+
status: result.status,
|
|
1370
|
+
stdout: readFileSync3(stdoutPath, "utf8"),
|
|
1371
|
+
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
1372
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1373
|
+
};
|
|
1374
|
+
} finally {
|
|
1375
|
+
rmSync2(tempDir, { recursive: true, force: true });
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
function ensureTodosTaskList(project, slug, name, description) {
|
|
1379
|
+
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
1380
|
+
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
1381
|
+
if (!list.ok)
|
|
1382
|
+
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
1383
|
+
const values = JSON.parse(list.stdout || "[]");
|
|
1384
|
+
const found = values.find((entry) => entry.slug === slug);
|
|
1385
|
+
if (!found)
|
|
1386
|
+
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
1387
|
+
return found.id;
|
|
1388
|
+
}
|
|
1389
|
+
function todosMutationSummary(result) {
|
|
1390
|
+
return {
|
|
1391
|
+
ok: result.ok,
|
|
1392
|
+
status: result.status,
|
|
1393
|
+
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1207
1397
|
// src/lib/store.ts
|
|
1208
1398
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
1209
1399
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
@@ -1213,6 +1403,7 @@ var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "s
|
|
|
1213
1403
|
var PRUNE_BATCH_SIZE = 400;
|
|
1214
1404
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
1215
1405
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
1406
|
+
var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
|
|
1216
1407
|
function rowToLoop(row) {
|
|
1217
1408
|
return {
|
|
1218
1409
|
id: row.id,
|
|
@@ -1263,6 +1454,9 @@ function rowToRun(row) {
|
|
|
1263
1454
|
updatedAt: row.updated_at
|
|
1264
1455
|
};
|
|
1265
1456
|
}
|
|
1457
|
+
function latestRunTime(row) {
|
|
1458
|
+
return row.finished_at ?? row.started_at ?? row.created_at;
|
|
1459
|
+
}
|
|
1266
1460
|
function rowToWorkflow(row) {
|
|
1267
1461
|
return {
|
|
1268
1462
|
id: row.id,
|
|
@@ -1517,7 +1711,7 @@ class Store {
|
|
|
1517
1711
|
const file = path ?? dbPath();
|
|
1518
1712
|
if (file !== ":memory:")
|
|
1519
1713
|
ensurePrivateStorePath(file);
|
|
1520
|
-
this.rootDir = file === ":memory:" ?
|
|
1714
|
+
this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
|
|
1521
1715
|
if (file === ":memory:")
|
|
1522
1716
|
this.memoryRootDir = this.rootDir;
|
|
1523
1717
|
this.db = new Database(file);
|
|
@@ -2015,7 +2209,32 @@ class Store {
|
|
|
2015
2209
|
} else {
|
|
2016
2210
|
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2017
2211
|
}
|
|
2018
|
-
return rows.map(rowToLoop);
|
|
2212
|
+
return this.withLatestRunSummaries(rows.map(rowToLoop));
|
|
2213
|
+
}
|
|
2214
|
+
withLatestRunSummaries(loops) {
|
|
2215
|
+
if (loops.length === 0)
|
|
2216
|
+
return loops;
|
|
2217
|
+
const placeholders = loops.map(() => "?").join(",");
|
|
2218
|
+
const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
|
|
2219
|
+
FROM (
|
|
2220
|
+
SELECT loop_id, id, status, started_at, finished_at, created_at,
|
|
2221
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
|
|
2222
|
+
FROM loop_runs
|
|
2223
|
+
WHERE loop_id IN (${placeholders})
|
|
2224
|
+
)
|
|
2225
|
+
WHERE rn = 1`).all(...loops.map((loop) => loop.id));
|
|
2226
|
+
const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
|
|
2227
|
+
return loops.map((loop) => {
|
|
2228
|
+
const latest = latestByLoopId.get(loop.id);
|
|
2229
|
+
if (!latest)
|
|
2230
|
+
return loop;
|
|
2231
|
+
return {
|
|
2232
|
+
...loop,
|
|
2233
|
+
latestRunId: latest.id,
|
|
2234
|
+
latestRunStatus: latest.status,
|
|
2235
|
+
lastRunAt: latestRunTime(latest)
|
|
2236
|
+
};
|
|
2237
|
+
});
|
|
2019
2238
|
}
|
|
2020
2239
|
dueLoops(now, limit = 500) {
|
|
2021
2240
|
const rows = this.db.query(`SELECT * FROM loops
|
|
@@ -2134,6 +2353,45 @@ class Store {
|
|
|
2134
2353
|
throw error;
|
|
2135
2354
|
}
|
|
2136
2355
|
}
|
|
2356
|
+
updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
|
|
2357
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
2358
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2359
|
+
try {
|
|
2360
|
+
const current = this.requireUniqueLoop(idOrName);
|
|
2361
|
+
if (current.archivedAt)
|
|
2362
|
+
throw new LoopArchivedError(current.name || current.id);
|
|
2363
|
+
if (current.target.type !== "agent")
|
|
2364
|
+
throw new Error(`loop is not an agent loop: ${idOrName}`);
|
|
2365
|
+
if (this.hasRunningRun(current.id))
|
|
2366
|
+
throw new Error(`refusing to update running loop: ${current.id}`);
|
|
2367
|
+
const target = { ...current.target, timeoutMs };
|
|
2368
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
2369
|
+
delete target.idleTimeoutMs;
|
|
2370
|
+
const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
|
|
2371
|
+
WHERE id=$id
|
|
2372
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2373
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2374
|
+
))`).run({
|
|
2375
|
+
$id: current.id,
|
|
2376
|
+
$target: JSON.stringify(target),
|
|
2377
|
+
$updated: updated,
|
|
2378
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2379
|
+
$now: updated
|
|
2380
|
+
});
|
|
2381
|
+
if (res.changes !== 1)
|
|
2382
|
+
throw new Error("daemon lease lost");
|
|
2383
|
+
this.db.exec("COMMIT");
|
|
2384
|
+
const after = this.getLoop(current.id);
|
|
2385
|
+
if (!after)
|
|
2386
|
+
throw new Error(`loop not found after timeout update: ${current.id}`);
|
|
2387
|
+
return after;
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
try {
|
|
2390
|
+
this.db.exec("ROLLBACK");
|
|
2391
|
+
} catch {}
|
|
2392
|
+
throw error;
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2137
2395
|
createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
|
|
2138
2396
|
const normalized = normalizeCreateWorkflowInput(workflowInput);
|
|
2139
2397
|
const updated = (opts.now ?? new Date).toISOString();
|
|
@@ -2464,6 +2722,59 @@ class Store {
|
|
|
2464
2722
|
updated
|
|
2465
2723
|
});
|
|
2466
2724
|
}
|
|
2725
|
+
taskLifecycleTodosPointerContext(workflowRunId) {
|
|
2726
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
2727
|
+
if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
|
|
2728
|
+
return;
|
|
2729
|
+
const workItem = this.getWorkflowWorkItem(run.workItemId);
|
|
2730
|
+
if (!workItem || workItem.routeKey !== "todos-task")
|
|
2731
|
+
return;
|
|
2732
|
+
const invocation = this.getWorkflowInvocation(run.invocationId);
|
|
2733
|
+
if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
|
|
2734
|
+
return;
|
|
2735
|
+
const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
|
|
2736
|
+
const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
|
|
2737
|
+
if (!projectPath || !taskId)
|
|
2738
|
+
return;
|
|
2739
|
+
return {
|
|
2740
|
+
projectPath,
|
|
2741
|
+
taskId,
|
|
2742
|
+
invocationId: invocation.id,
|
|
2743
|
+
workflowRunId: run.id,
|
|
2744
|
+
manifestPath: run.manifestPath
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
|
|
2748
|
+
const context = this.taskLifecycleTodosPointerContext(workflowRunId);
|
|
2749
|
+
if (!context)
|
|
2750
|
+
return;
|
|
2751
|
+
const result = runLocalCommand("todos", [
|
|
2752
|
+
"--project",
|
|
2753
|
+
context.projectPath,
|
|
2754
|
+
"task",
|
|
2755
|
+
"workflow-pointers",
|
|
2756
|
+
context.taskId,
|
|
2757
|
+
"--clear",
|
|
2758
|
+
"--invocation",
|
|
2759
|
+
context.invocationId,
|
|
2760
|
+
"--run",
|
|
2761
|
+
context.workflowRunId,
|
|
2762
|
+
"--manifest",
|
|
2763
|
+
context.manifestPath,
|
|
2764
|
+
"--state",
|
|
2765
|
+
"succeeded",
|
|
2766
|
+
"--actor",
|
|
2767
|
+
"openloops:task-lifecycle"
|
|
2768
|
+
]);
|
|
2769
|
+
this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
|
|
2770
|
+
projectPath: context.projectPath,
|
|
2771
|
+
taskId: context.taskId,
|
|
2772
|
+
invocationId: context.invocationId,
|
|
2773
|
+
workflowRunId: context.workflowRunId,
|
|
2774
|
+
manifestPath: context.manifestPath,
|
|
2775
|
+
mutation: todosMutationSummary(result)
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2467
2778
|
createWorkflowInvocation(input) {
|
|
2468
2779
|
const now = nowIso();
|
|
2469
2780
|
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
@@ -3426,6 +3737,8 @@ class Store {
|
|
|
3426
3737
|
const run = this.getWorkflowRun(workflowRunId);
|
|
3427
3738
|
if (!run)
|
|
3428
3739
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
3740
|
+
if (changed && status === "succeeded")
|
|
3741
|
+
this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
|
|
3429
3742
|
return run;
|
|
3430
3743
|
}
|
|
3431
3744
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -3487,6 +3800,17 @@ class Store {
|
|
|
3487
3800
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
3488
3801
|
return (row?.count ?? 0) > 0;
|
|
3489
3802
|
}
|
|
3803
|
+
hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
|
|
3804
|
+
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
3805
|
+
WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
|
|
3806
|
+
return rows.some((row) => {
|
|
3807
|
+
if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
|
|
3808
|
+
return true;
|
|
3809
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at))
|
|
3810
|
+
return true;
|
|
3811
|
+
return this.hasLiveWorkflowStepProcesses(row.id);
|
|
3812
|
+
});
|
|
3813
|
+
}
|
|
3490
3814
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
3491
3815
|
const now = (opts.now ?? new Date).toISOString();
|
|
3492
3816
|
const startedMs = processStartTimeMs(pid);
|
|
@@ -3619,6 +3943,10 @@ class Store {
|
|
|
3619
3943
|
loop = currentLoop;
|
|
3620
3944
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
3621
3945
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
3946
|
+
if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
|
|
3947
|
+
this.db.exec("COMMIT");
|
|
3948
|
+
return;
|
|
3949
|
+
}
|
|
3622
3950
|
if (existing) {
|
|
3623
3951
|
if (existing.status === "running") {
|
|
3624
3952
|
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
@@ -4190,9 +4518,9 @@ class Store {
|
|
|
4190
4518
|
const runDir = dirname2(manifestPath);
|
|
4191
4519
|
try {
|
|
4192
4520
|
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
4193
|
-
|
|
4521
|
+
rmSync3(runDir, { recursive: true, force: true });
|
|
4194
4522
|
} else {
|
|
4195
|
-
|
|
4523
|
+
rmSync3(manifestPath, { force: true });
|
|
4196
4524
|
}
|
|
4197
4525
|
} catch {}
|
|
4198
4526
|
}
|
|
@@ -4259,7 +4587,7 @@ class Store {
|
|
|
4259
4587
|
this.db.close();
|
|
4260
4588
|
if (this.memoryRootDir) {
|
|
4261
4589
|
try {
|
|
4262
|
-
|
|
4590
|
+
rmSync3(this.memoryRootDir, { recursive: true, force: true });
|
|
4263
4591
|
} catch {}
|
|
4264
4592
|
this.memoryRootDir = undefined;
|
|
4265
4593
|
}
|
|
@@ -4268,7 +4596,7 @@ class Store {
|
|
|
4268
4596
|
// package.json
|
|
4269
4597
|
var package_default = {
|
|
4270
4598
|
name: "@hasna/loops",
|
|
4271
|
-
version: "0.4.
|
|
4599
|
+
version: "0.4.14",
|
|
4272
4600
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4273
4601
|
type: "module",
|
|
4274
4602
|
main: "dist/index.js",
|
|
@@ -4277,6 +4605,7 @@ var package_default = {
|
|
|
4277
4605
|
loops: "dist/cli/index.js",
|
|
4278
4606
|
"loops-daemon": "dist/daemon/index.js",
|
|
4279
4607
|
"loops-api": "dist/api/index.js",
|
|
4608
|
+
"loops-serve": "dist/serve/index.js",
|
|
4280
4609
|
"loops-runner": "dist/runner/index.js",
|
|
4281
4610
|
"loops-mcp": "dist/mcp/index.js"
|
|
4282
4611
|
},
|
|
@@ -4289,6 +4618,14 @@ var package_default = {
|
|
|
4289
4618
|
types: "./dist/sdk/index.d.ts",
|
|
4290
4619
|
import: "./dist/sdk/index.js"
|
|
4291
4620
|
},
|
|
4621
|
+
"./sdk/http": {
|
|
4622
|
+
types: "./dist/sdk/http.d.ts",
|
|
4623
|
+
import: "./dist/sdk/http.js"
|
|
4624
|
+
},
|
|
4625
|
+
"./serve": {
|
|
4626
|
+
types: "./dist/serve/index.d.ts",
|
|
4627
|
+
import: "./dist/serve/index.js"
|
|
4628
|
+
},
|
|
4292
4629
|
"./mcp": {
|
|
4293
4630
|
types: "./dist/mcp/index.d.ts",
|
|
4294
4631
|
import: "./dist/mcp/index.js"
|
|
@@ -4334,7 +4671,7 @@ var package_default = {
|
|
|
4334
4671
|
"LICENSE"
|
|
4335
4672
|
],
|
|
4336
4673
|
scripts: {
|
|
4337
|
-
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4674
|
+
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4338
4675
|
"build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
|
|
4339
4676
|
typecheck: "tsc --noEmit",
|
|
4340
4677
|
test: "bun test",
|
|
@@ -4369,11 +4706,13 @@ var package_default = {
|
|
|
4369
4706
|
bun: ">=1.0.0"
|
|
4370
4707
|
},
|
|
4371
4708
|
dependencies: {
|
|
4709
|
+
"@hasna/contracts": "^0.4.1",
|
|
4372
4710
|
"@hasna/events": "^0.1.9",
|
|
4373
4711
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
4374
4712
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
4375
4713
|
ai: "6.0.204",
|
|
4376
4714
|
commander: "^13.1.0",
|
|
4715
|
+
pg: "^8.13.1",
|
|
4377
4716
|
zod: "4.4.3"
|
|
4378
4717
|
},
|
|
4379
4718
|
optionalDependencies: {
|
|
@@ -4381,6 +4720,7 @@ var package_default = {
|
|
|
4381
4720
|
},
|
|
4382
4721
|
devDependencies: {
|
|
4383
4722
|
"@types/bun": "latest",
|
|
4723
|
+
"@types/pg": "^8.11.10",
|
|
4384
4724
|
typescript: "^5.7.3"
|
|
4385
4725
|
},
|
|
4386
4726
|
publishConfig: {
|
|
@@ -4590,136 +4930,20 @@ function deploymentStatusLine(status) {
|
|
|
4590
4930
|
|
|
4591
4931
|
// src/cli/index.ts
|
|
4592
4932
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
4593
|
-
import { existsSync as existsSync10, readdirSync as readdirSync3, readFileSync as readFileSync10, rmSync as rmSync7, statSync as statSync2, writeFileSync as
|
|
4594
|
-
import { join as
|
|
4933
|
+
import { existsSync as existsSync10, readdirSync as readdirSync3, readFileSync as readFileSync10, rmSync as rmSync7, statSync as statSync2, writeFileSync as writeFileSync7 } from "fs";
|
|
4934
|
+
import { join as join12 } from "path";
|
|
4595
4935
|
import { Database as Database3 } from "bun:sqlite";
|
|
4596
4936
|
import { Command } from "commander";
|
|
4597
4937
|
|
|
4598
|
-
// src/lib/format.ts
|
|
4599
|
-
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
4600
|
-
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
4601
|
-
function redact(value, visible = 0) {
|
|
4602
|
-
if (!value)
|
|
4603
|
-
return value;
|
|
4604
|
-
if (value.length <= visible)
|
|
4605
|
-
return value;
|
|
4606
|
-
if (visible <= 0)
|
|
4607
|
-
return `[redacted ${value.length} chars]`;
|
|
4608
|
-
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
4609
|
-
}
|
|
4610
|
-
function truncateTextOutput(value) {
|
|
4611
|
-
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
4612
|
-
return value;
|
|
4613
|
-
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
4614
|
-
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
4615
|
-
}
|
|
4616
|
-
function redactSensitivePayload(value, key) {
|
|
4617
|
-
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
4618
|
-
if (typeof value === "string")
|
|
4619
|
-
return redact(value);
|
|
4620
|
-
if (value === undefined || value === null)
|
|
4621
|
-
return value;
|
|
4622
|
-
return "[redacted]";
|
|
4623
|
-
}
|
|
4624
|
-
if (Array.isArray(value))
|
|
4625
|
-
return value.map((item) => redactSensitivePayload(item));
|
|
4626
|
-
if (value && typeof value === "object") {
|
|
4627
|
-
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
4628
|
-
}
|
|
4629
|
-
return value;
|
|
4630
|
-
}
|
|
4631
|
-
function textOutputBlocks(value, opts = {}) {
|
|
4632
|
-
const indent = opts.indent ?? "";
|
|
4633
|
-
const nested = `${indent} `;
|
|
4634
|
-
const blocks = [];
|
|
4635
|
-
for (const [label, output] of [
|
|
4636
|
-
["stdout", value.stdout],
|
|
4637
|
-
["stderr", value.stderr]
|
|
4638
|
-
]) {
|
|
4639
|
-
if (!output)
|
|
4640
|
-
continue;
|
|
4641
|
-
blocks.push(`${indent}${label}:`);
|
|
4642
|
-
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
4643
|
-
blocks.push(`${nested}${line}`);
|
|
4644
|
-
}
|
|
4645
|
-
}
|
|
4646
|
-
return blocks;
|
|
4647
|
-
}
|
|
4648
|
-
function publicLoop(loop) {
|
|
4649
|
-
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
4650
|
-
return {
|
|
4651
|
-
...loop,
|
|
4652
|
-
target
|
|
4653
|
-
};
|
|
4654
|
-
}
|
|
4655
|
-
function publicRun(run, showOutput = false, opts = {}) {
|
|
4656
|
-
return {
|
|
4657
|
-
...run,
|
|
4658
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
4659
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
4660
|
-
error: opts.redactError ? redact(run.error) : run.error
|
|
4661
|
-
};
|
|
4662
|
-
}
|
|
4663
|
-
function publicExecutorResult(result, showOutput = false) {
|
|
4664
|
-
return {
|
|
4665
|
-
...result,
|
|
4666
|
-
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
4667
|
-
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
4668
|
-
error: redact(result.error)
|
|
4669
|
-
};
|
|
4670
|
-
}
|
|
4671
|
-
function publicWorkflow(workflow) {
|
|
4672
|
-
return {
|
|
4673
|
-
...workflow,
|
|
4674
|
-
steps: workflow.steps.map((step) => ({
|
|
4675
|
-
...step,
|
|
4676
|
-
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
4677
|
-
}))
|
|
4678
|
-
};
|
|
4679
|
-
}
|
|
4680
|
-
function publicWorkflowRun(run) {
|
|
4681
|
-
return { ...run, error: redact(run.error) };
|
|
4682
|
-
}
|
|
4683
|
-
function publicWorkflowInvocation(invocation) {
|
|
4684
|
-
return redactSensitivePayload(invocation);
|
|
4685
|
-
}
|
|
4686
|
-
function publicWorkflowWorkItem(item) {
|
|
4687
|
-
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
4688
|
-
}
|
|
4689
|
-
function publicWorkflowStepRun(run, showOutput = false) {
|
|
4690
|
-
return {
|
|
4691
|
-
...run,
|
|
4692
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
4693
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
4694
|
-
error: redact(run.error)
|
|
4695
|
-
};
|
|
4696
|
-
}
|
|
4697
|
-
function publicWorkflowEvent(event) {
|
|
4698
|
-
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
4699
|
-
}
|
|
4700
|
-
function publicGoal(goal) {
|
|
4701
|
-
return {
|
|
4702
|
-
...goal,
|
|
4703
|
-
objective: redact(goal.objective, 120)
|
|
4704
|
-
};
|
|
4705
|
-
}
|
|
4706
|
-
function publicGoalRun(run) {
|
|
4707
|
-
return {
|
|
4708
|
-
...run,
|
|
4709
|
-
evidence: redactSensitivePayload(run.evidence),
|
|
4710
|
-
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
4711
|
-
};
|
|
4712
|
-
}
|
|
4713
|
-
|
|
4714
4938
|
// src/lib/executor.ts
|
|
4715
|
-
import { spawn as spawn2, spawnSync as
|
|
4939
|
+
import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
|
|
4716
4940
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
4717
4941
|
import { once } from "events";
|
|
4718
4942
|
import { lstatSync, mkdirSync as mkdirSync4, realpathSync } from "fs";
|
|
4719
4943
|
import { dirname as dirname3, resolve as resolve2 } from "path";
|
|
4720
4944
|
|
|
4721
4945
|
// src/lib/accounts.ts
|
|
4722
|
-
import { spawnSync as
|
|
4946
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
4723
4947
|
import { existsSync as existsSync2 } from "fs";
|
|
4724
4948
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4725
4949
|
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
@@ -4824,7 +5048,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4824
5048
|
if (!account)
|
|
4825
5049
|
return {};
|
|
4826
5050
|
const tool = requiredAccountTool(account, toolHint);
|
|
4827
|
-
const result =
|
|
5051
|
+
const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
|
|
4828
5052
|
encoding: "utf8",
|
|
4829
5053
|
env,
|
|
4830
5054
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -4840,8 +5064,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4840
5064
|
|
|
4841
5065
|
// src/lib/env.ts
|
|
4842
5066
|
import { accessSync, constants } from "fs";
|
|
4843
|
-
import { homedir as
|
|
4844
|
-
import { delimiter, join as
|
|
5067
|
+
import { homedir as homedir3 } from "os";
|
|
5068
|
+
import { delimiter, join as join5 } from "path";
|
|
4845
5069
|
function compactPathParts(parts) {
|
|
4846
5070
|
const seen = new Set;
|
|
4847
5071
|
const result = [];
|
|
@@ -4855,16 +5079,16 @@ function compactPathParts(parts) {
|
|
|
4855
5079
|
return result;
|
|
4856
5080
|
}
|
|
4857
5081
|
function commonExecutableDirs(env = process.env) {
|
|
4858
|
-
const home = env.HOME ||
|
|
5082
|
+
const home = env.HOME || homedir3();
|
|
4859
5083
|
return compactPathParts([
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
env.BUN_INSTALL ?
|
|
5084
|
+
join5(home, ".local", "bin"),
|
|
5085
|
+
join5(home, ".bun", "bin"),
|
|
5086
|
+
join5(home, ".cargo", "bin"),
|
|
5087
|
+
join5(home, ".npm-global", "bin"),
|
|
5088
|
+
join5(home, "bin"),
|
|
5089
|
+
env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
|
|
4866
5090
|
env.PNPM_HOME,
|
|
4867
|
-
env.NPM_CONFIG_PREFIX ?
|
|
5091
|
+
env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
4868
5092
|
"/opt/homebrew/bin",
|
|
4869
5093
|
"/usr/local/bin",
|
|
4870
5094
|
"/usr/bin",
|
|
@@ -4888,7 +5112,7 @@ function executableExists(command, env = process.env) {
|
|
|
4888
5112
|
if (command.includes("/"))
|
|
4889
5113
|
return isExecutable(command);
|
|
4890
5114
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
4891
|
-
if (dir && isExecutable(
|
|
5115
|
+
if (dir && isExecutable(join5(dir, command)))
|
|
4892
5116
|
return true;
|
|
4893
5117
|
}
|
|
4894
5118
|
return false;
|
|
@@ -5029,6 +5253,32 @@ function timeoutResult(startedAt, error, fields = {}) {
|
|
|
5029
5253
|
function successResult(startedAt, fields = {}) {
|
|
5030
5254
|
return buildResult("succeeded", startedAt, fields);
|
|
5031
5255
|
}
|
|
5256
|
+
function codewithJsonlHasTerminalSuccess(stdout) {
|
|
5257
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
5258
|
+
const trimmed = line.trim();
|
|
5259
|
+
if (!trimmed || !trimmed.startsWith("{"))
|
|
5260
|
+
continue;
|
|
5261
|
+
let event;
|
|
5262
|
+
try {
|
|
5263
|
+
event = JSON.parse(trimmed);
|
|
5264
|
+
} catch {
|
|
5265
|
+
continue;
|
|
5266
|
+
}
|
|
5267
|
+
if (!event || typeof event !== "object")
|
|
5268
|
+
continue;
|
|
5269
|
+
const record = event;
|
|
5270
|
+
if (record.type === "task_complete")
|
|
5271
|
+
return true;
|
|
5272
|
+
const payload = record.payload;
|
|
5273
|
+
if (payload && typeof payload === "object" && payload.type === "task_complete") {
|
|
5274
|
+
return true;
|
|
5275
|
+
}
|
|
5276
|
+
}
|
|
5277
|
+
return false;
|
|
5278
|
+
}
|
|
5279
|
+
function codewithJsonlReconciledSuccess(spec, fields) {
|
|
5280
|
+
return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
|
|
5281
|
+
}
|
|
5032
5282
|
function notifySpawn(pid, opts) {
|
|
5033
5283
|
if (!pid)
|
|
5034
5284
|
return;
|
|
@@ -5044,10 +5294,48 @@ function shellQuote(value) {
|
|
|
5044
5294
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
5045
5295
|
}
|
|
5046
5296
|
function codewithProfileCandidateFromLine(line) {
|
|
5047
|
-
const
|
|
5048
|
-
if (
|
|
5049
|
-
return
|
|
5050
|
-
|
|
5297
|
+
const trimmed = line.trim();
|
|
5298
|
+
if (!trimmed || trimmed === "No auth profiles saved.")
|
|
5299
|
+
return;
|
|
5300
|
+
const cols = trimmed.split(/\s+/);
|
|
5301
|
+
const candidate = cols[0] === "*" ? cols[1] : cols[0];
|
|
5302
|
+
if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
|
|
5303
|
+
return;
|
|
5304
|
+
if (candidate === '"name":') {
|
|
5305
|
+
const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
|
|
5306
|
+
if (jsonName)
|
|
5307
|
+
return jsonName;
|
|
5308
|
+
}
|
|
5309
|
+
return candidate;
|
|
5310
|
+
}
|
|
5311
|
+
function codewithProfileCandidatesFromJson(value) {
|
|
5312
|
+
const candidates = [];
|
|
5313
|
+
const root = value && typeof value === "object" ? value : undefined;
|
|
5314
|
+
const addProfile = (entry) => {
|
|
5315
|
+
if (!entry || typeof entry !== "object")
|
|
5316
|
+
return;
|
|
5317
|
+
const name = entry.name;
|
|
5318
|
+
if (typeof name === "string" && name)
|
|
5319
|
+
candidates.push(name);
|
|
5320
|
+
};
|
|
5321
|
+
const data = root?.data;
|
|
5322
|
+
if (Array.isArray(data))
|
|
5323
|
+
data.forEach(addProfile);
|
|
5324
|
+
const profiles = root?.profiles;
|
|
5325
|
+
if (Array.isArray(profiles))
|
|
5326
|
+
profiles.forEach(addProfile);
|
|
5327
|
+
return candidates;
|
|
5328
|
+
}
|
|
5329
|
+
function codewithProfileCandidatesFromOutput(output) {
|
|
5330
|
+
const trimmed = output.trim();
|
|
5331
|
+
const jsonStart = trimmed.indexOf("{");
|
|
5332
|
+
const jsonEnd = trimmed.lastIndexOf("}");
|
|
5333
|
+
if (jsonStart >= 0 && jsonEnd >= jsonStart) {
|
|
5334
|
+
try {
|
|
5335
|
+
return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
|
|
5336
|
+
} catch {}
|
|
5337
|
+
}
|
|
5338
|
+
return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
|
|
5051
5339
|
}
|
|
5052
5340
|
function codewithProfileForError(profile) {
|
|
5053
5341
|
return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
|
|
@@ -5140,6 +5428,7 @@ function commandSpec(target, opts) {
|
|
|
5140
5428
|
preflightAnyOf: invocation.preflightAnyOf,
|
|
5141
5429
|
stdin: invocation.stdin,
|
|
5142
5430
|
allowlist: agentTarget.allowlist,
|
|
5431
|
+
agentProvider: agentTarget.provider,
|
|
5143
5432
|
worktree: agentTarget.worktree
|
|
5144
5433
|
};
|
|
5145
5434
|
}
|
|
@@ -5299,7 +5588,7 @@ function remotePreflightScript(spec, metadata) {
|
|
|
5299
5588
|
}
|
|
5300
5589
|
if (spec.nativeAuthProfile?.provider === "codewith") {
|
|
5301
5590
|
const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
|
|
5302
|
-
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE",
|
|
5591
|
+
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
|
|
5303
5592
|
}
|
|
5304
5593
|
return lines.join(`
|
|
5305
5594
|
`);
|
|
@@ -5317,6 +5606,12 @@ function transportEnv(opts) {
|
|
|
5317
5606
|
return env;
|
|
5318
5607
|
}
|
|
5319
5608
|
function assertCodewithProfileListed(profile, result) {
|
|
5609
|
+
const profiles = codewithProfileSetFromResult(result);
|
|
5610
|
+
if (!profiles.has(profile)) {
|
|
5611
|
+
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5612
|
+
}
|
|
5613
|
+
}
|
|
5614
|
+
function codewithProfileSetFromResult(result) {
|
|
5320
5615
|
if (result.error) {
|
|
5321
5616
|
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
5322
5617
|
}
|
|
@@ -5324,32 +5619,42 @@ function assertCodewithProfileListed(profile, result) {
|
|
|
5324
5619
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5325
5620
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
5326
5621
|
}
|
|
5327
|
-
|
|
5328
|
-
if (!profiles.has(profile)) {
|
|
5329
|
-
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5330
|
-
}
|
|
5622
|
+
return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
|
|
5331
5623
|
}
|
|
5332
5624
|
function preflightNativeAuthProfileSync(spec, env) {
|
|
5333
5625
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5334
5626
|
return;
|
|
5335
|
-
const
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
|
|
5342
|
-
|
|
5343
|
-
|
|
5344
|
-
|
|
5345
|
-
|
|
5346
|
-
|
|
5627
|
+
const runProfileList = (args) => {
|
|
5628
|
+
const result = spawnSync4(spec.command, args, {
|
|
5629
|
+
encoding: "utf8",
|
|
5630
|
+
env,
|
|
5631
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
5632
|
+
timeout: 15000
|
|
5633
|
+
});
|
|
5634
|
+
return {
|
|
5635
|
+
status: result.status,
|
|
5636
|
+
stdout: result.stdout ?? "",
|
|
5637
|
+
stderr: result.stderr ?? "",
|
|
5638
|
+
error: result.error?.message
|
|
5639
|
+
};
|
|
5640
|
+
};
|
|
5641
|
+
const jsonResult = runProfileList(["profile", "list", "--json"]);
|
|
5642
|
+
try {
|
|
5643
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
5644
|
+
return;
|
|
5645
|
+
} catch {}
|
|
5646
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
|
|
5347
5647
|
}
|
|
5348
5648
|
async function preflightNativeAuthProfile(spec, env) {
|
|
5349
5649
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5350
5650
|
return;
|
|
5351
|
-
const
|
|
5352
|
-
|
|
5651
|
+
const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
|
|
5652
|
+
try {
|
|
5653
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
5654
|
+
return;
|
|
5655
|
+
} catch {}
|
|
5656
|
+
const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
5657
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
|
|
5353
5658
|
}
|
|
5354
5659
|
function resolvedDirEquals(left, right) {
|
|
5355
5660
|
try {
|
|
@@ -5452,7 +5757,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
|
5452
5757
|
}
|
|
5453
5758
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
5454
5759
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
5455
|
-
const result =
|
|
5760
|
+
const result = spawnSync4(plan.command, plan.args, {
|
|
5456
5761
|
encoding: "utf8",
|
|
5457
5762
|
env: transportEnv(opts),
|
|
5458
5763
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -5551,6 +5856,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
|
5551
5856
|
if (timedOut || idleTimedOut) {
|
|
5552
5857
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5553
5858
|
}
|
|
5859
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
5860
|
+
return successResult(startedAt, fields);
|
|
5861
|
+
}
|
|
5554
5862
|
if (error || exitCode !== 0) {
|
|
5555
5863
|
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5556
5864
|
}
|
|
@@ -5686,6 +5994,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
5686
5994
|
if (timedOut || idleTimedOut) {
|
|
5687
5995
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5688
5996
|
}
|
|
5997
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
5998
|
+
return successResult(startedAt, fields);
|
|
5999
|
+
}
|
|
5689
6000
|
if (error || exitCode !== 0) {
|
|
5690
6001
|
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5691
6002
|
}
|
|
@@ -6631,12 +6942,18 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
6631
6942
|
|
|
6632
6943
|
// src/lib/health.ts
|
|
6633
6944
|
import { createHash as createHash2 } from "crypto";
|
|
6634
|
-
import { existsSync as existsSync3, readFileSync as
|
|
6945
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
6946
|
+
import { join as join6 } from "path";
|
|
6635
6947
|
var EVIDENCE_CHARS = 2000;
|
|
6636
6948
|
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6949
|
+
var DEFAULT_SCAN_LIMIT = 200;
|
|
6950
|
+
var DEFAULT_SCAN_STATUSES = ["active", "paused"];
|
|
6951
|
+
var DEFAULT_MAX_FINDINGS = 100;
|
|
6952
|
+
var MIN_STALE_RUNNING_MS = 10 * 60000;
|
|
6637
6953
|
var CLASSIFICATIONS = [
|
|
6638
6954
|
"rate_limit",
|
|
6639
6955
|
"auth",
|
|
6956
|
+
"provider_unavailable",
|
|
6640
6957
|
"model_not_found",
|
|
6641
6958
|
"context_length",
|
|
6642
6959
|
"schema_response_format",
|
|
@@ -6645,10 +6962,12 @@ var CLASSIFICATIONS = [
|
|
|
6645
6962
|
"route_functional",
|
|
6646
6963
|
"timeout",
|
|
6647
6964
|
"sigsegv",
|
|
6965
|
+
"restart_interrupted",
|
|
6648
6966
|
"skipped_previous_active",
|
|
6649
6967
|
"circuit_breaker",
|
|
6650
6968
|
"unknown"
|
|
6651
6969
|
];
|
|
6970
|
+
var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
|
|
6652
6971
|
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
6653
6972
|
if (!value)
|
|
6654
6973
|
return;
|
|
@@ -6662,7 +6981,7 @@ function redactedEvidence(value) {
|
|
|
6662
6981
|
}
|
|
6663
6982
|
function searchableText(run) {
|
|
6664
6983
|
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
6665
|
-
`)
|
|
6984
|
+
`);
|
|
6666
6985
|
}
|
|
6667
6986
|
function isRecord(value) {
|
|
6668
6987
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
@@ -6687,13 +7006,36 @@ function stableFingerprint(parts) {
|
|
|
6687
7006
|
return createHash2("sha256").update(parts.join(`
|
|
6688
7007
|
`)).digest("hex").slice(0, 16);
|
|
6689
7008
|
}
|
|
6690
|
-
function
|
|
7009
|
+
function stableScanFingerprint(parts) {
|
|
7010
|
+
return `openloops:health-scan:${stableFingerprint(parts)}`;
|
|
7011
|
+
}
|
|
7012
|
+
function safeHost(value) {
|
|
7013
|
+
if (!value)
|
|
7014
|
+
return;
|
|
7015
|
+
let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
|
|
7016
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
7017
|
+
return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
|
|
7018
|
+
}
|
|
7019
|
+
function isCursorHost(host) {
|
|
7020
|
+
return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
|
|
7021
|
+
}
|
|
7022
|
+
function providerUnavailableSummary(rawText) {
|
|
7023
|
+
const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
|
|
7024
|
+
if (dns) {
|
|
7025
|
+
const host = safeHost(dns[2]);
|
|
7026
|
+
if (isCursorHost(host))
|
|
7027
|
+
return `provider DNS lookup failed: ${dns[1]} ${host}`;
|
|
7028
|
+
}
|
|
7029
|
+
return;
|
|
7030
|
+
}
|
|
7031
|
+
function stableFailureFingerprint(run, classification, summary) {
|
|
7032
|
+
const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
|
|
6691
7033
|
return stableFingerprint([
|
|
6692
7034
|
run.loopId,
|
|
6693
7035
|
classification,
|
|
6694
7036
|
String(run.status),
|
|
6695
7037
|
String(run.exitCode ?? ""),
|
|
6696
|
-
|
|
7038
|
+
evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6697
7039
|
]);
|
|
6698
7040
|
}
|
|
6699
7041
|
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
@@ -6711,13 +7053,279 @@ function healthRun(run) {
|
|
|
6711
7053
|
stderr: redactedEvidence(run.stderr)
|
|
6712
7054
|
};
|
|
6713
7055
|
}
|
|
7056
|
+
function compactText(value, limit = 500) {
|
|
7057
|
+
const text = redact(bounded(value, limit));
|
|
7058
|
+
return text?.replace(/\s+/g, " ").trim();
|
|
7059
|
+
}
|
|
7060
|
+
function publicDoctorCheck(check) {
|
|
7061
|
+
return {
|
|
7062
|
+
id: check.id,
|
|
7063
|
+
status: check.status,
|
|
7064
|
+
message: redact(bounded(check.message, 500)) ?? "",
|
|
7065
|
+
detail: redact(bounded(check.detail, 800))
|
|
7066
|
+
};
|
|
7067
|
+
}
|
|
7068
|
+
function publicDoctorReport(report) {
|
|
7069
|
+
return {
|
|
7070
|
+
ok: report.ok,
|
|
7071
|
+
checks: report.checks.map(publicDoctorCheck)
|
|
7072
|
+
};
|
|
7073
|
+
}
|
|
7074
|
+
function includedStatusSet(statuses) {
|
|
7075
|
+
const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
|
|
7076
|
+
return new Set(values);
|
|
7077
|
+
}
|
|
7078
|
+
function statusCounts(loops) {
|
|
7079
|
+
return {
|
|
7080
|
+
loops: loops.length,
|
|
7081
|
+
active: loops.filter((loop) => loop.status === "active").length,
|
|
7082
|
+
paused: loops.filter((loop) => loop.status === "paused").length,
|
|
7083
|
+
stopped: loops.filter((loop) => loop.status === "stopped").length,
|
|
7084
|
+
expired: loops.filter((loop) => loop.status === "expired").length
|
|
7085
|
+
};
|
|
7086
|
+
}
|
|
7087
|
+
function compareLoopsForScan(left, right) {
|
|
7088
|
+
const statusOrder = left.status.localeCompare(right.status);
|
|
7089
|
+
if (statusOrder !== 0)
|
|
7090
|
+
return statusOrder;
|
|
7091
|
+
return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
|
|
7092
|
+
}
|
|
7093
|
+
function scanLoops(store, statuses, opts) {
|
|
7094
|
+
const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
|
|
7095
|
+
return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
|
|
7096
|
+
}
|
|
7097
|
+
function healthReportForLoops(store, loops, generatedAt) {
|
|
7098
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
7099
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
7100
|
+
for (const expectation of expectations) {
|
|
7101
|
+
if (expectation.failure)
|
|
7102
|
+
classifications[expectation.failure.classification] += 1;
|
|
7103
|
+
}
|
|
7104
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
7105
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
7106
|
+
return {
|
|
7107
|
+
ok: unhealthy === 0,
|
|
7108
|
+
generatedAt,
|
|
7109
|
+
summary: {
|
|
7110
|
+
loops: expectations.length,
|
|
7111
|
+
healthy: expectations.length - unhealthy,
|
|
7112
|
+
unhealthy,
|
|
7113
|
+
warnings
|
|
7114
|
+
},
|
|
7115
|
+
classifications,
|
|
7116
|
+
expectations
|
|
7117
|
+
};
|
|
7118
|
+
}
|
|
7119
|
+
function ageMs(run, now) {
|
|
7120
|
+
const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
|
|
7121
|
+
const time = Date.parse(stamp);
|
|
7122
|
+
return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
|
|
7123
|
+
}
|
|
7124
|
+
function shortLoop(loop) {
|
|
7125
|
+
return {
|
|
7126
|
+
id: loop.id,
|
|
7127
|
+
name: loop.name,
|
|
7128
|
+
status: loop.status,
|
|
7129
|
+
nextRunAt: loop.nextRunAt,
|
|
7130
|
+
leaseMs: loop.leaseMs
|
|
7131
|
+
};
|
|
7132
|
+
}
|
|
7133
|
+
function priorityForSeverity(severity) {
|
|
7134
|
+
if (severity === "critical")
|
|
7135
|
+
return "critical";
|
|
7136
|
+
if (severity === "high")
|
|
7137
|
+
return "high";
|
|
7138
|
+
if (severity === "low")
|
|
7139
|
+
return "low";
|
|
7140
|
+
return "medium";
|
|
7141
|
+
}
|
|
7142
|
+
function recommendedFindingTask(finding, route) {
|
|
7143
|
+
const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
|
|
7144
|
+
if (finding.classification)
|
|
7145
|
+
tags.push(finding.classification);
|
|
7146
|
+
const description = [
|
|
7147
|
+
`OpenLoops health scan found a ${finding.kind} issue.`,
|
|
7148
|
+
finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
|
|
7149
|
+
finding.run ? `Run: ${finding.run.id}` : undefined,
|
|
7150
|
+
finding.classification ? `Classification: ${finding.classification}` : undefined,
|
|
7151
|
+
finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
|
|
7152
|
+
finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
|
|
7153
|
+
`Fingerprint: ${finding.fingerprint}`,
|
|
7154
|
+
`Severity: ${finding.severity}`,
|
|
7155
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
7156
|
+
route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
7157
|
+
route?.provider ? `Provider: ${route.provider}` : undefined,
|
|
7158
|
+
"",
|
|
7159
|
+
finding.message,
|
|
7160
|
+
finding.run?.error ? `Error:
|
|
7161
|
+
${finding.run.error}` : undefined,
|
|
7162
|
+
finding.run?.stderr ? `Stderr:
|
|
7163
|
+
${finding.run.stderr}` : undefined
|
|
7164
|
+
].filter(Boolean).join(`
|
|
7165
|
+
|
|
7166
|
+
`);
|
|
7167
|
+
return {
|
|
7168
|
+
title: finding.title,
|
|
7169
|
+
description,
|
|
7170
|
+
priority: priorityForSeverity(finding.severity),
|
|
7171
|
+
tags,
|
|
7172
|
+
dedupeKey: finding.fingerprint,
|
|
7173
|
+
search: { query: finding.fingerprint },
|
|
7174
|
+
compatibilityFallback: {
|
|
7175
|
+
search: ["todos", "search", finding.fingerprint, "--json"],
|
|
7176
|
+
add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
|
|
7177
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
7178
|
+
},
|
|
7179
|
+
futureNativeUpsert: {
|
|
7180
|
+
command: "todos task upsert",
|
|
7181
|
+
fields: {
|
|
7182
|
+
title: finding.title,
|
|
7183
|
+
description,
|
|
7184
|
+
priority: priorityForSeverity(finding.severity),
|
|
7185
|
+
tags,
|
|
7186
|
+
dedupeKey: finding.fingerprint,
|
|
7187
|
+
routeSource: route?.source ?? "openloops",
|
|
7188
|
+
routeKind: route?.kind ?? "health_scan",
|
|
7189
|
+
routeLoopId: finding.loop?.id ?? "",
|
|
7190
|
+
routeLoopName: finding.loop?.name ?? ""
|
|
7191
|
+
}
|
|
7192
|
+
}
|
|
7193
|
+
};
|
|
7194
|
+
}
|
|
7195
|
+
function daemonFinding(daemon) {
|
|
7196
|
+
if (daemon.running && !daemon.stale)
|
|
7197
|
+
return;
|
|
7198
|
+
const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
|
|
7199
|
+
const severity = "critical";
|
|
7200
|
+
const finding = {
|
|
7201
|
+
kind: "daemon",
|
|
7202
|
+
severity,
|
|
7203
|
+
fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
|
|
7204
|
+
title: "OpenLoops daemon health issue",
|
|
7205
|
+
message: reason
|
|
7206
|
+
};
|
|
7207
|
+
return {
|
|
7208
|
+
...finding,
|
|
7209
|
+
recommendedTask: recommendedFindingTask(finding, undefined)
|
|
7210
|
+
};
|
|
7211
|
+
}
|
|
7212
|
+
function doctorSeverity(check) {
|
|
7213
|
+
if (check.status === "fail" && check.id === "data-dir")
|
|
7214
|
+
return "critical";
|
|
7215
|
+
if (check.status === "fail")
|
|
7216
|
+
return "high";
|
|
7217
|
+
return "medium";
|
|
7218
|
+
}
|
|
7219
|
+
function doctorFinding(check, loop, route) {
|
|
7220
|
+
if (check.status === "ok" || check.id === "loop-runs")
|
|
7221
|
+
return;
|
|
7222
|
+
const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
|
|
7223
|
+
const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
|
|
7224
|
+
const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
|
|
7225
|
+
const finding = {
|
|
7226
|
+
kind,
|
|
7227
|
+
severity,
|
|
7228
|
+
fingerprint,
|
|
7229
|
+
title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
|
|
7230
|
+
message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
|
|
7231
|
+
loop: loop ? shortLoop(loop) : undefined,
|
|
7232
|
+
route,
|
|
7233
|
+
doctorCheck: publicDoctorCheck(check)
|
|
7234
|
+
};
|
|
7235
|
+
return {
|
|
7236
|
+
...finding,
|
|
7237
|
+
recommendedTask: recommendedFindingTask(finding, route)
|
|
7238
|
+
};
|
|
7239
|
+
}
|
|
7240
|
+
function latestRunFinding(expectation) {
|
|
7241
|
+
if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
|
|
7242
|
+
return;
|
|
7243
|
+
const failure = expectation.failure;
|
|
7244
|
+
if (!failure)
|
|
7245
|
+
return;
|
|
7246
|
+
const severity = expectation.loop.status === "active" ? "high" : "medium";
|
|
7247
|
+
return {
|
|
7248
|
+
kind: "latest-run",
|
|
7249
|
+
severity,
|
|
7250
|
+
fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
|
|
7251
|
+
title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
|
|
7252
|
+
message: expectation.check.message,
|
|
7253
|
+
loop: expectation.loop,
|
|
7254
|
+
run: expectation.latestRun,
|
|
7255
|
+
route: expectation.route,
|
|
7256
|
+
classification: failure.classification,
|
|
7257
|
+
doctorCheck: undefined,
|
|
7258
|
+
recommendedTask: expectation.recommendedTask
|
|
7259
|
+
};
|
|
7260
|
+
}
|
|
7261
|
+
function staleRunningFinding(loop, expectation, now, staleRunningMs) {
|
|
7262
|
+
const run = expectation.latestRun;
|
|
7263
|
+
if (loop.status !== "active" || run?.status !== "running")
|
|
7264
|
+
return;
|
|
7265
|
+
const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
|
|
7266
|
+
const age = ageMs(run, now);
|
|
7267
|
+
if (age <= threshold)
|
|
7268
|
+
return;
|
|
7269
|
+
const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
|
|
7270
|
+
const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
|
|
7271
|
+
const finding = {
|
|
7272
|
+
kind: "stale-running",
|
|
7273
|
+
severity: "critical",
|
|
7274
|
+
fingerprint,
|
|
7275
|
+
title: `OpenLoops stale running run - ${loop.name}`,
|
|
7276
|
+
message,
|
|
7277
|
+
loop: shortLoop(loop),
|
|
7278
|
+
run,
|
|
7279
|
+
route: expectation.route,
|
|
7280
|
+
ageMs: age,
|
|
7281
|
+
staleThresholdMs: threshold
|
|
7282
|
+
};
|
|
7283
|
+
return {
|
|
7284
|
+
...finding,
|
|
7285
|
+
recommendedTask: recommendedFindingTask(finding, expectation.route)
|
|
7286
|
+
};
|
|
7287
|
+
}
|
|
7288
|
+
function scanStatus(findings) {
|
|
7289
|
+
if (findings.some((finding) => finding.severity === "critical"))
|
|
7290
|
+
return "critical";
|
|
7291
|
+
if (findings.length > 0)
|
|
7292
|
+
return "degraded";
|
|
7293
|
+
return "ok";
|
|
7294
|
+
}
|
|
7295
|
+
function timestampDir(root, generatedAt) {
|
|
7296
|
+
const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
|
|
7297
|
+
return join6(root, stamp);
|
|
7298
|
+
}
|
|
7299
|
+
function healthScanMarkdown(scan) {
|
|
7300
|
+
return [
|
|
7301
|
+
"# OpenLoops Health Scan",
|
|
7302
|
+
"",
|
|
7303
|
+
`- status: ${scan.status}`,
|
|
7304
|
+
`- generated_at: ${scan.generatedAt}`,
|
|
7305
|
+
`- included_statuses: ${scan.includedStatuses.join(",")}`,
|
|
7306
|
+
`- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
|
|
7307
|
+
`- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
|
|
7308
|
+
scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
|
|
7309
|
+
scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
|
|
7310
|
+
scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
|
|
7311
|
+
"",
|
|
7312
|
+
"## Findings",
|
|
7313
|
+
scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
|
|
7314
|
+
`) : "None."
|
|
7315
|
+
].join(`
|
|
7316
|
+
`);
|
|
7317
|
+
}
|
|
6714
7318
|
function classifyRunFailure(run) {
|
|
6715
7319
|
if (run.status === "succeeded" || run.status === "running")
|
|
6716
7320
|
return;
|
|
6717
|
-
const
|
|
7321
|
+
const rawText = searchableText(run);
|
|
7322
|
+
const text = rawText.toLowerCase();
|
|
6718
7323
|
let classification = "unknown";
|
|
7324
|
+
let summary;
|
|
6719
7325
|
if (run.status === "timed_out")
|
|
6720
7326
|
classification = "timeout";
|
|
7327
|
+
else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
|
|
7328
|
+
classification = "restart_interrupted";
|
|
6721
7329
|
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
6722
7330
|
classification = "circuit_breaker";
|
|
6723
7331
|
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
@@ -6726,8 +7334,10 @@ function classifyRunFailure(run) {
|
|
|
6726
7334
|
classification = "preflight";
|
|
6727
7335
|
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
6728
7336
|
classification = "rate_limit";
|
|
6729
|
-
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
7337
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
|
|
6730
7338
|
classification = "auth";
|
|
7339
|
+
else if (summary = providerUnavailableSummary(rawText))
|
|
7340
|
+
classification = "provider_unavailable";
|
|
6731
7341
|
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
6732
7342
|
classification = "model_not_found";
|
|
6733
7343
|
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
@@ -6740,8 +7350,9 @@ function classifyRunFailure(run) {
|
|
|
6740
7350
|
classification = "sigsegv";
|
|
6741
7351
|
return {
|
|
6742
7352
|
classification,
|
|
6743
|
-
fingerprint: stableFailureFingerprint(run, classification),
|
|
7353
|
+
fingerprint: stableFailureFingerprint(run, classification, summary),
|
|
6744
7354
|
evidence: {
|
|
7355
|
+
summary,
|
|
6745
7356
|
error: redactedEvidence(run.error),
|
|
6746
7357
|
stdout: redactedEvidence(run.stdout),
|
|
6747
7358
|
stderr: redactedEvidence(run.stderr),
|
|
@@ -6786,7 +7397,7 @@ function routeEvidenceReport(run) {
|
|
|
6786
7397
|
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
6787
7398
|
if (evidencePath && existsSync3(evidencePath)) {
|
|
6788
7399
|
try {
|
|
6789
|
-
return parseJsonObject(
|
|
7400
|
+
return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
|
|
6790
7401
|
} catch {
|
|
6791
7402
|
return stdoutReport;
|
|
6792
7403
|
}
|
|
@@ -6918,6 +7529,12 @@ function targetRoute(loop) {
|
|
|
6918
7529
|
loopName: loop.name
|
|
6919
7530
|
};
|
|
6920
7531
|
}
|
|
7532
|
+
function expectationLoop(loop) {
|
|
7533
|
+
return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
|
|
7534
|
+
}
|
|
7535
|
+
function hasPendingRetry(loop, run) {
|
|
7536
|
+
return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
|
|
7537
|
+
}
|
|
6921
7538
|
function recommendedTask(loop, run, failure, route) {
|
|
6922
7539
|
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
6923
7540
|
const description = [
|
|
@@ -6929,6 +7546,7 @@ function recommendedTask(loop, run, failure, route) {
|
|
|
6929
7546
|
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6930
7547
|
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6931
7548
|
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
7549
|
+
failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
|
|
6932
7550
|
failure.evidence.error ? `Error:
|
|
6933
7551
|
${failure.evidence.error}` : undefined,
|
|
6934
7552
|
failure.evidence.stderr ? `Stderr:
|
|
@@ -6938,7 +7556,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6938
7556
|
`);
|
|
6939
7557
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6940
7558
|
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6941
|
-
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
7559
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
|
|
6942
7560
|
return {
|
|
6943
7561
|
title,
|
|
6944
7562
|
description,
|
|
@@ -6972,7 +7590,7 @@ function expectationForLoop(store, loop) {
|
|
|
6972
7590
|
const route = targetRoute(loop);
|
|
6973
7591
|
if (!latestRun) {
|
|
6974
7592
|
return {
|
|
6975
|
-
loop:
|
|
7593
|
+
loop: expectationLoop(loop),
|
|
6976
7594
|
ok: true,
|
|
6977
7595
|
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
6978
7596
|
route
|
|
@@ -6981,7 +7599,7 @@ function expectationForLoop(store, loop) {
|
|
|
6981
7599
|
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
6982
7600
|
if (routeFailure) {
|
|
6983
7601
|
return {
|
|
6984
|
-
loop:
|
|
7602
|
+
loop: expectationLoop(loop),
|
|
6985
7603
|
ok: false,
|
|
6986
7604
|
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
6987
7605
|
latestRun: healthRun(latestRun),
|
|
@@ -6992,7 +7610,7 @@ function expectationForLoop(store, loop) {
|
|
|
6992
7610
|
}
|
|
6993
7611
|
if (latestRun.status === "succeeded") {
|
|
6994
7612
|
return {
|
|
6995
|
-
loop:
|
|
7613
|
+
loop: expectationLoop(loop),
|
|
6996
7614
|
ok: true,
|
|
6997
7615
|
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
6998
7616
|
latestRun: healthRun(latestRun),
|
|
@@ -7003,7 +7621,7 @@ function expectationForLoop(store, loop) {
|
|
|
7003
7621
|
if (failure?.classification === "circuit_breaker") {
|
|
7004
7622
|
if (loop.status !== "paused") {
|
|
7005
7623
|
return {
|
|
7006
|
-
loop:
|
|
7624
|
+
loop: expectationLoop(loop),
|
|
7007
7625
|
ok: true,
|
|
7008
7626
|
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
7009
7627
|
latestRun: healthRun(latestRun),
|
|
@@ -7011,7 +7629,7 @@ function expectationForLoop(store, loop) {
|
|
|
7011
7629
|
};
|
|
7012
7630
|
}
|
|
7013
7631
|
return {
|
|
7014
|
-
loop:
|
|
7632
|
+
loop: expectationLoop(loop),
|
|
7015
7633
|
ok: false,
|
|
7016
7634
|
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
7017
7635
|
latestRun: healthRun(latestRun),
|
|
@@ -7020,8 +7638,33 @@ function expectationForLoop(store, loop) {
|
|
|
7020
7638
|
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
7021
7639
|
};
|
|
7022
7640
|
}
|
|
7641
|
+
if (failure?.classification === "restart_interrupted") {
|
|
7642
|
+
return {
|
|
7643
|
+
loop: expectationLoop(loop),
|
|
7644
|
+
ok: true,
|
|
7645
|
+
check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
|
|
7646
|
+
latestRun: healthRun(latestRun),
|
|
7647
|
+
failure,
|
|
7648
|
+
route
|
|
7649
|
+
};
|
|
7650
|
+
}
|
|
7651
|
+
if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
|
|
7652
|
+
const message = [
|
|
7653
|
+
"provider unavailable/network failure; retry is scheduled",
|
|
7654
|
+
loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
|
|
7655
|
+
failure.evidence.summary
|
|
7656
|
+
].filter(Boolean).join("; ");
|
|
7657
|
+
return {
|
|
7658
|
+
loop: expectationLoop(loop),
|
|
7659
|
+
ok: true,
|
|
7660
|
+
check: { id: "latest-run-succeeded", status: "warn", message },
|
|
7661
|
+
latestRun: healthRun(latestRun),
|
|
7662
|
+
failure,
|
|
7663
|
+
route
|
|
7664
|
+
};
|
|
7665
|
+
}
|
|
7023
7666
|
return {
|
|
7024
|
-
loop:
|
|
7667
|
+
loop: expectationLoop(loop),
|
|
7025
7668
|
ok: false,
|
|
7026
7669
|
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
7027
7670
|
latestRun: healthRun(latestRun),
|
|
@@ -7041,18 +7684,101 @@ function buildHealthReport(store, opts = {}) {
|
|
|
7041
7684
|
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
7042
7685
|
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
7043
7686
|
return {
|
|
7044
|
-
ok: unhealthy === 0,
|
|
7045
|
-
generatedAt: new Date().toISOString(),
|
|
7046
|
-
summary: {
|
|
7047
|
-
loops: expectations.length,
|
|
7048
|
-
healthy: expectations.length - unhealthy,
|
|
7049
|
-
unhealthy,
|
|
7050
|
-
warnings
|
|
7687
|
+
ok: unhealthy === 0,
|
|
7688
|
+
generatedAt: new Date().toISOString(),
|
|
7689
|
+
summary: {
|
|
7690
|
+
loops: expectations.length,
|
|
7691
|
+
healthy: expectations.length - unhealthy,
|
|
7692
|
+
unhealthy,
|
|
7693
|
+
warnings
|
|
7694
|
+
},
|
|
7695
|
+
classifications,
|
|
7696
|
+
expectations
|
|
7697
|
+
};
|
|
7698
|
+
}
|
|
7699
|
+
function buildHealthScan(store, opts = {}) {
|
|
7700
|
+
const generatedAt = (opts.now ?? new Date).toISOString();
|
|
7701
|
+
const now = opts.now ?? new Date(generatedAt);
|
|
7702
|
+
const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
|
|
7703
|
+
const loops = scanLoops(store, includeStatuses, opts);
|
|
7704
|
+
const health = healthReportForLoops(store, loops, generatedAt);
|
|
7705
|
+
const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
|
|
7706
|
+
const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
|
|
7707
|
+
const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
|
|
7708
|
+
const allFindings = [];
|
|
7709
|
+
const pushFinding = (finding) => {
|
|
7710
|
+
if (!finding)
|
|
7711
|
+
return;
|
|
7712
|
+
allFindings.push(finding);
|
|
7713
|
+
};
|
|
7714
|
+
if (opts.daemon)
|
|
7715
|
+
pushFinding(daemonFinding(opts.daemon));
|
|
7716
|
+
if (opts.latestRun !== false) {
|
|
7717
|
+
for (const loop of loops) {
|
|
7718
|
+
const expectation = expectationsByLoopId.get(loop.id);
|
|
7719
|
+
if (!expectation)
|
|
7720
|
+
continue;
|
|
7721
|
+
pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
|
|
7722
|
+
pushFinding(latestRunFinding(expectation));
|
|
7723
|
+
}
|
|
7724
|
+
}
|
|
7725
|
+
const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
|
|
7726
|
+
if (doctor) {
|
|
7727
|
+
for (const check of doctor.checks) {
|
|
7728
|
+
const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
|
|
7729
|
+
const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
|
|
7730
|
+
const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
|
|
7731
|
+
pushFinding(doctorFinding(check, loop, route));
|
|
7732
|
+
}
|
|
7733
|
+
}
|
|
7734
|
+
const findings = allFindings.slice(0, maxFindings);
|
|
7735
|
+
const status = scanStatus(allFindings);
|
|
7736
|
+
const baseCounts = statusCounts(loops);
|
|
7737
|
+
return {
|
|
7738
|
+
ok: status === "ok",
|
|
7739
|
+
status,
|
|
7740
|
+
generatedAt,
|
|
7741
|
+
includedStatuses: includeStatuses,
|
|
7742
|
+
counts: {
|
|
7743
|
+
...baseCounts,
|
|
7744
|
+
latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
|
|
7745
|
+
staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
|
|
7746
|
+
daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
|
|
7747
|
+
doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
|
|
7748
|
+
preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
|
|
7749
|
+
findings: allFindings.length,
|
|
7750
|
+
reportedFindings: findings.length,
|
|
7751
|
+
truncatedFindings: Math.max(0, allFindings.length - findings.length)
|
|
7051
7752
|
},
|
|
7052
|
-
|
|
7053
|
-
|
|
7753
|
+
daemon: opts.daemon ? {
|
|
7754
|
+
running: opts.daemon.running,
|
|
7755
|
+
stale: opts.daemon.stale,
|
|
7756
|
+
pid: opts.daemon.pid,
|
|
7757
|
+
host: opts.daemon.host,
|
|
7758
|
+
loops: opts.daemon.loops,
|
|
7759
|
+
runs: opts.daemon.runs,
|
|
7760
|
+
logPath: opts.daemon.logPath
|
|
7761
|
+
} : undefined,
|
|
7762
|
+
doctor,
|
|
7763
|
+
health,
|
|
7764
|
+
selfHeals: opts.selfHeals ?? [],
|
|
7765
|
+
findings
|
|
7054
7766
|
};
|
|
7055
7767
|
}
|
|
7768
|
+
function writeHealthScanReports(scan, opts = {}) {
|
|
7769
|
+
const root = opts.reportDir ?? join6(dataDir(), "reports", "health-scan");
|
|
7770
|
+
const dir = timestampDir(root, scan.generatedAt);
|
|
7771
|
+
mkdirSync5(dir, { recursive: true, mode: 448 });
|
|
7772
|
+
const json = join6(dir, "summary.json");
|
|
7773
|
+
const markdown = join6(dir, "report.md");
|
|
7774
|
+
const withReports = {
|
|
7775
|
+
...scan,
|
|
7776
|
+
reports: { dir, json, markdown }
|
|
7777
|
+
};
|
|
7778
|
+
writeFileSync2(json, JSON.stringify(withReports, null, 2), { mode: 384 });
|
|
7779
|
+
writeFileSync2(markdown, healthScanMarkdown(withReports), { mode: 384 });
|
|
7780
|
+
return withReports;
|
|
7781
|
+
}
|
|
7056
7782
|
|
|
7057
7783
|
// src/lib/scheduler.ts
|
|
7058
7784
|
function loopLane(loop) {
|
|
@@ -7139,7 +7865,7 @@ var MAX_RETRY_EXPONENT = 20;
|
|
|
7139
7865
|
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
7140
7866
|
const attempt = Math.max(1, run.attempt);
|
|
7141
7867
|
const failure = classifyRunFailure(run);
|
|
7142
|
-
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
7868
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
|
|
7143
7869
|
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
7144
7870
|
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
7145
7871
|
const jitter = 0.5 + random();
|
|
@@ -7266,20 +7992,21 @@ async function executeClaimedRun(deps) {
|
|
|
7266
7992
|
daemonLeaseId: deps.daemonLeaseId,
|
|
7267
7993
|
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
7268
7994
|
})))(deps.loop, deps.run);
|
|
7995
|
+
const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
|
|
7269
7996
|
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
7270
7997
|
return deps.store.finalizeRun(deps.run.id, {
|
|
7271
|
-
status:
|
|
7272
|
-
finishedAt:
|
|
7273
|
-
durationMs:
|
|
7274
|
-
stdout:
|
|
7275
|
-
stderr:
|
|
7276
|
-
exitCode:
|
|
7277
|
-
error:
|
|
7278
|
-
pid:
|
|
7998
|
+
status: finalResult.status,
|
|
7999
|
+
finishedAt: finalResult.finishedAt,
|
|
8000
|
+
durationMs: finalResult.durationMs,
|
|
8001
|
+
stdout: finalResult.stdout,
|
|
8002
|
+
stderr: finalResult.stderr,
|
|
8003
|
+
exitCode: finalResult.exitCode,
|
|
8004
|
+
error: finalResult.error,
|
|
8005
|
+
pid: finalResult.pid
|
|
7279
8006
|
}, {
|
|
7280
8007
|
claimedBy: deps.runnerId,
|
|
7281
8008
|
daemonLeaseId: deps.daemonLeaseId,
|
|
7282
|
-
now: deps.now?.() ?? new Date(
|
|
8009
|
+
now: deps.now?.() ?? new Date(finalResult.finishedAt)
|
|
7283
8010
|
});
|
|
7284
8011
|
} catch (err) {
|
|
7285
8012
|
deps.onError?.(deps.loop, err);
|
|
@@ -7514,10 +8241,10 @@ async function tick(deps) {
|
|
|
7514
8241
|
}
|
|
7515
8242
|
|
|
7516
8243
|
// src/daemon/control.ts
|
|
7517
|
-
import { existsSync as existsSync4, mkdirSync as
|
|
7518
|
-
import { spawnSync as
|
|
8244
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, rmSync as rmSync4, writeFileSync as writeFileSync3 } from "fs";
|
|
8245
|
+
import { spawnSync as spawnSync5 } from "child_process";
|
|
7519
8246
|
import { hostname } from "os";
|
|
7520
|
-
import { dirname as dirname4, join as
|
|
8247
|
+
import { dirname as dirname4, join as join7 } from "path";
|
|
7521
8248
|
|
|
7522
8249
|
// src/daemon/loop.ts
|
|
7523
8250
|
function realSleep(ms) {
|
|
@@ -7547,7 +8274,7 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
7547
8274
|
return;
|
|
7548
8275
|
let raw;
|
|
7549
8276
|
try {
|
|
7550
|
-
raw =
|
|
8277
|
+
raw = readFileSync5(path, "utf8").trim();
|
|
7551
8278
|
} catch {
|
|
7552
8279
|
return;
|
|
7553
8280
|
}
|
|
@@ -7569,11 +8296,11 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
7569
8296
|
return Number.isInteger(pid) && pid > 0 ? { pid } : undefined;
|
|
7570
8297
|
}
|
|
7571
8298
|
function writePid(pid = process.pid, path = pidFilePath()) {
|
|
7572
|
-
|
|
7573
|
-
|
|
8299
|
+
mkdirSync6(dirname4(path), { recursive: true, mode: 448 });
|
|
8300
|
+
writeFileSync3(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
7574
8301
|
}
|
|
7575
8302
|
function removePid(path = pidFilePath()) {
|
|
7576
|
-
|
|
8303
|
+
rmSync4(path, { force: true });
|
|
7577
8304
|
}
|
|
7578
8305
|
function isAlive(pid, startedAt) {
|
|
7579
8306
|
try {
|
|
@@ -7603,7 +8330,7 @@ function ownProcessGroupId() {
|
|
|
7603
8330
|
return pgrp;
|
|
7604
8331
|
}
|
|
7605
8332
|
try {
|
|
7606
|
-
const run =
|
|
8333
|
+
const run = spawnSync5("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
7607
8334
|
const pgid = Number(run.stdout.trim());
|
|
7608
8335
|
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
7609
8336
|
return pgid;
|
|
@@ -7699,7 +8426,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7699
8426
|
if (!state.running || !state.pid)
|
|
7700
8427
|
return { wasRunning: false, stopped: false, forced: false };
|
|
7701
8428
|
const sleep = opts.sleep ?? realSleep;
|
|
7702
|
-
const store = new Store(
|
|
8429
|
+
const store = new Store(join7(dirname4(path), "loops.db"));
|
|
7703
8430
|
try {
|
|
7704
8431
|
const lease = store.getDaemonLease();
|
|
7705
8432
|
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
@@ -7737,7 +8464,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7737
8464
|
}
|
|
7738
8465
|
|
|
7739
8466
|
// src/daemon/daemon.ts
|
|
7740
|
-
import { copyFileSync, existsSync as existsSync5, openSync, renameSync as renameSync2, rmSync as
|
|
8467
|
+
import { copyFileSync, existsSync as existsSync5, openSync as openSync2, renameSync as renameSync2, rmSync as rmSync5, statSync, truncateSync } from "fs";
|
|
7741
8468
|
import { hostname as hostname2 } from "os";
|
|
7742
8469
|
import { spawn as spawn3 } from "child_process";
|
|
7743
8470
|
function intervalFromEnv() {
|
|
@@ -7779,7 +8506,7 @@ function rotateDaemonLog(path = daemonLogPath(), maxBytes = DAEMON_LOG_MAX_BYTES
|
|
|
7779
8506
|
if (size < maxBytes)
|
|
7780
8507
|
return false;
|
|
7781
8508
|
try {
|
|
7782
|
-
|
|
8509
|
+
rmSync5(`${path}.${keep}`, { force: true });
|
|
7783
8510
|
for (let i = keep - 1;i >= 1; i--) {
|
|
7784
8511
|
if (existsSync5(`${path}.${i}`))
|
|
7785
8512
|
renameSync2(`${path}.${i}`, `${path}.${i + 1}`);
|
|
@@ -7828,6 +8555,7 @@ async function runDaemon(opts = {}) {
|
|
|
7828
8555
|
let runAbort = new AbortController;
|
|
7829
8556
|
const activeRuns = new Map;
|
|
7830
8557
|
const activeByLane = { command: 0, agent: 0 };
|
|
8558
|
+
const isControlledStopInterruption = (result) => stopFlag && !leaseLost && result.status === "failed" && result.error?.includes("terminated by SIGTERM") === true;
|
|
7831
8559
|
const requestStop = (message) => {
|
|
7832
8560
|
stopFlag = true;
|
|
7833
8561
|
if (!runAbort.signal.aborted)
|
|
@@ -7911,6 +8639,16 @@ async function runDaemon(opts = {}) {
|
|
|
7911
8639
|
store.recordRunProcess(run.id, info, { daemonLeaseId: leaseId });
|
|
7912
8640
|
}
|
|
7913
8641
|
})),
|
|
8642
|
+
finalizeResult: (result) => {
|
|
8643
|
+
if (!isControlledStopInterruption(result))
|
|
8644
|
+
return result;
|
|
8645
|
+
return {
|
|
8646
|
+
...result,
|
|
8647
|
+
status: "skipped",
|
|
8648
|
+
exitCode: undefined,
|
|
8649
|
+
error: `${RESTART_INTERRUPTED_RUN_PREFIX}: child process terminated by SIGTERM during daemon stop/restart`
|
|
8650
|
+
};
|
|
8651
|
+
},
|
|
7914
8652
|
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
7915
8653
|
});
|
|
7916
8654
|
ensureLease();
|
|
@@ -8004,7 +8742,7 @@ async function startDaemon(opts) {
|
|
|
8004
8742
|
return { started: false, alreadyRunning: true, pid: state.pid };
|
|
8005
8743
|
if (state.stale)
|
|
8006
8744
|
removePid();
|
|
8007
|
-
const out =
|
|
8745
|
+
const out = openSync2(daemonLogPath(), "a");
|
|
8008
8746
|
const child = spawn3(opts.execPath ?? process.execPath, [opts.cliEntry, ...opts.args ?? ["daemon", "run"]], {
|
|
8009
8747
|
detached: true,
|
|
8010
8748
|
stdio: ["ignore", out, out]
|
|
@@ -8022,8 +8760,8 @@ async function startDaemon(opts) {
|
|
|
8022
8760
|
}
|
|
8023
8761
|
|
|
8024
8762
|
// src/daemon/install.ts
|
|
8025
|
-
import { chmodSync as chmodSync2, mkdirSync as
|
|
8026
|
-
import { spawnSync as
|
|
8763
|
+
import { chmodSync as chmodSync2, mkdirSync as mkdirSync7, writeFileSync as writeFileSync4 } from "fs";
|
|
8764
|
+
import { spawnSync as spawnSync6 } from "child_process";
|
|
8027
8765
|
import { dirname as dirname5 } from "path";
|
|
8028
8766
|
function systemdEscapeExecPart(part) {
|
|
8029
8767
|
const escaped = part.replaceAll("%", "%%");
|
|
@@ -8055,9 +8793,9 @@ function installStartup(cliEntry, execPath = process.execPath, args = ["daemon",
|
|
|
8055
8793
|
const dataDirPath = ensureDataDir();
|
|
8056
8794
|
if (platform === "linux") {
|
|
8057
8795
|
const path = systemdServicePath();
|
|
8058
|
-
|
|
8796
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 448 });
|
|
8059
8797
|
const execStart = [execPath, cliEntry, ...args].map(systemdEscapeExecPart).join(" ");
|
|
8060
|
-
|
|
8798
|
+
writeFileSync4(path, `[Unit]
|
|
8061
8799
|
Description=Hasna OpenLoops daemon
|
|
8062
8800
|
After=basic.target
|
|
8063
8801
|
|
|
@@ -8085,8 +8823,8 @@ WantedBy=default.target
|
|
|
8085
8823
|
}
|
|
8086
8824
|
if (platform === "darwin") {
|
|
8087
8825
|
const path = launchdPlistPath();
|
|
8088
|
-
|
|
8089
|
-
|
|
8826
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 448 });
|
|
8827
|
+
writeFileSync4(path, `<?xml version="1.0" encoding="UTF-8"?>
|
|
8090
8828
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
8091
8829
|
<plist version="1.0">
|
|
8092
8830
|
<dict>
|
|
@@ -8123,7 +8861,7 @@ ${args.map((arg) => ` <string>${xmlEscape(arg)}</string>`).join(`
|
|
|
8123
8861
|
function enableStartup(result) {
|
|
8124
8862
|
const commands = result.platform === "linux" ? ["systemctl --user daemon-reload", "systemctl --user enable --now loops-daemon.service"] : result.platform === "darwin" ? launchctlCommands(result.path) : [];
|
|
8125
8863
|
return commands.map((command) => {
|
|
8126
|
-
const run =
|
|
8864
|
+
const run = spawnSync6("sh", ["-c", command], {
|
|
8127
8865
|
encoding: "utf8",
|
|
8128
8866
|
stdio: ["ignore", "pipe", "pipe"]
|
|
8129
8867
|
});
|
|
@@ -8137,7 +8875,7 @@ function enableStartup(result) {
|
|
|
8137
8875
|
}
|
|
8138
8876
|
|
|
8139
8877
|
// src/lib/doctor.ts
|
|
8140
|
-
import { spawnSync as
|
|
8878
|
+
import { spawnSync as spawnSync7 } from "child_process";
|
|
8141
8879
|
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
8142
8880
|
var PROVIDER_COMMANDS = [
|
|
8143
8881
|
"claude",
|
|
@@ -8148,11 +8886,11 @@ var PROVIDER_COMMANDS = [
|
|
|
8148
8886
|
"codex"
|
|
8149
8887
|
];
|
|
8150
8888
|
function hasCommand(command) {
|
|
8151
|
-
const result =
|
|
8889
|
+
const result = spawnSync7("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
8152
8890
|
return (result.status ?? 1) === 0;
|
|
8153
8891
|
}
|
|
8154
8892
|
function commandVersion(command) {
|
|
8155
|
-
const result =
|
|
8893
|
+
const result = spawnSync7(command, ["--version"], {
|
|
8156
8894
|
encoding: "utf8",
|
|
8157
8895
|
stdio: ["ignore", "pipe", "pipe"]
|
|
8158
8896
|
});
|
|
@@ -8201,7 +8939,15 @@ function runDoctor(store) {
|
|
|
8201
8939
|
const status = daemonStatus(store);
|
|
8202
8940
|
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
8203
8941
|
const failedRuns = store.countRuns("failed");
|
|
8942
|
+
const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
|
|
8204
8943
|
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
8944
|
+
if (restartInterruptedRuns > 0) {
|
|
8945
|
+
checks.push({
|
|
8946
|
+
id: "loop-runs:restart-interrupted",
|
|
8947
|
+
status: "warn",
|
|
8948
|
+
message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
|
|
8949
|
+
});
|
|
8950
|
+
}
|
|
8205
8951
|
const deployment = buildDeploymentStatus();
|
|
8206
8952
|
const schedulerState = deployment.schedulerState;
|
|
8207
8953
|
checks.push({
|
|
@@ -8247,6 +8993,365 @@ function runDoctor(store) {
|
|
|
8247
8993
|
};
|
|
8248
8994
|
}
|
|
8249
8995
|
|
|
8996
|
+
// src/cli/ui.ts
|
|
8997
|
+
var CLEAR_SCREEN = "\x1B[2J\x1B[H";
|
|
8998
|
+
var ENTER_ALT_SCREEN = "\x1B[?1049h";
|
|
8999
|
+
var EXIT_ALT_SCREEN = "\x1B[?1049l";
|
|
9000
|
+
var HIDE_CURSOR = "\x1B[?25l";
|
|
9001
|
+
var SHOW_CURSOR = "\x1B[?25h";
|
|
9002
|
+
var RESET = "\x1B[0m";
|
|
9003
|
+
var BOLD = "\x1B[1m";
|
|
9004
|
+
var DIM = "\x1B[2m";
|
|
9005
|
+
var CYAN = "\x1B[36m";
|
|
9006
|
+
var GREEN = "\x1B[32m";
|
|
9007
|
+
var YELLOW = "\x1B[33m";
|
|
9008
|
+
var RED = "\x1B[31m";
|
|
9009
|
+
var MAGENTA = "\x1B[35m";
|
|
9010
|
+
var DEFAULT_REFRESH_MS = 2000;
|
|
9011
|
+
var MIN_REFRESH_MS = 500;
|
|
9012
|
+
var MAX_ACTIVE_LOOPS = 1e4;
|
|
9013
|
+
var MAX_RUNNING_RUNS = 1e5;
|
|
9014
|
+
function buildLoopUiSnapshot(store, opts = {}) {
|
|
9015
|
+
const now = opts.now ?? new Date;
|
|
9016
|
+
const activeLoops = store.listLoops({ status: "active", limit: opts.limit ?? MAX_ACTIVE_LOOPS });
|
|
9017
|
+
const runningRuns = store.listRuns({ status: "running", limit: MAX_RUNNING_RUNS });
|
|
9018
|
+
const runningByLoop = new Map;
|
|
9019
|
+
for (const run of runningRuns) {
|
|
9020
|
+
runningByLoop.set(run.loopId, (runningByLoop.get(run.loopId) ?? 0) + 1);
|
|
9021
|
+
}
|
|
9022
|
+
return {
|
|
9023
|
+
rows: activeLoops.map((loop) => {
|
|
9024
|
+
const latest = store.listRuns({ loopId: loop.id, limit: 1 })[0];
|
|
9025
|
+
return {
|
|
9026
|
+
id: loop.id,
|
|
9027
|
+
name: loop.name,
|
|
9028
|
+
status: loop.status,
|
|
9029
|
+
cadence: scheduleLabel(loop.schedule),
|
|
9030
|
+
nextRun: nextRunLabel(loop.nextRunAt, now),
|
|
9031
|
+
lastRunOutcome: runOutcomeLabel(latest),
|
|
9032
|
+
provider: providerLabel(loop.target),
|
|
9033
|
+
activeRuns: runningByLoop.get(loop.id) ?? 0
|
|
9034
|
+
};
|
|
9035
|
+
}),
|
|
9036
|
+
stats: {
|
|
9037
|
+
activeLoops: store.countLoops("active"),
|
|
9038
|
+
pausedLoops: store.countLoops("paused"),
|
|
9039
|
+
stoppedLoops: store.countLoops("stopped"),
|
|
9040
|
+
runningRuns: store.countRuns("running"),
|
|
9041
|
+
failedRuns: store.countRuns("failed"),
|
|
9042
|
+
updatedAt: now.toISOString()
|
|
9043
|
+
}
|
|
9044
|
+
};
|
|
9045
|
+
}
|
|
9046
|
+
function renderLoopUiFrame(snapshot, opts = {}) {
|
|
9047
|
+
const columns = Math.max(40, opts.columns ?? 100);
|
|
9048
|
+
const rows = opts.rows ?? snapshot.rows.length + 8;
|
|
9049
|
+
const refreshMs = opts.refreshMs ?? DEFAULT_REFRESH_MS;
|
|
9050
|
+
const color = opts.color ?? false;
|
|
9051
|
+
const compact2 = columns < 88;
|
|
9052
|
+
const widths = tableWidths(columns, compact2);
|
|
9053
|
+
const maxBodyRows = Math.max(0, rows - 7);
|
|
9054
|
+
const shownRows = snapshot.rows.slice(0, maxBodyRows);
|
|
9055
|
+
const hidden = snapshot.rows.length - shownRows.length;
|
|
9056
|
+
const updated = timeOnly(snapshot.stats.updatedAt);
|
|
9057
|
+
const statLine = [
|
|
9058
|
+
`active loops ${snapshot.stats.activeLoops}`,
|
|
9059
|
+
`running runs ${snapshot.stats.runningRuns}`,
|
|
9060
|
+
`failed runs ${snapshot.stats.failedRuns}`,
|
|
9061
|
+
`paused ${snapshot.stats.pausedLoops}`,
|
|
9062
|
+
`refresh ${durationLabel(refreshMs) || `${refreshMs}ms`}`,
|
|
9063
|
+
`updated ${updated}`
|
|
9064
|
+
].join(" | ");
|
|
9065
|
+
const lines = [
|
|
9066
|
+
paint(`OpenLoops`, `${BOLD}${CYAN}`, color) + paint(" live loops", DIM, color),
|
|
9067
|
+
paint(fitLine(statLine, columns), DIM, color),
|
|
9068
|
+
"",
|
|
9069
|
+
paint(tableHeader(widths, compact2), `${BOLD}${CYAN}`, color),
|
|
9070
|
+
paint("-".repeat(Math.min(columns, tableWidth(widths))), DIM, color)
|
|
9071
|
+
];
|
|
9072
|
+
for (const row of shownRows) {
|
|
9073
|
+
lines.push(tableRow(row, widths, color));
|
|
9074
|
+
}
|
|
9075
|
+
if (snapshot.rows.length === 0) {
|
|
9076
|
+
lines.push(paint("No active loops.", DIM, color));
|
|
9077
|
+
} else if (hidden > 0) {
|
|
9078
|
+
lines.push(paint(`showing ${shownRows.length} of ${snapshot.rows.length} active loops`, DIM, color));
|
|
9079
|
+
}
|
|
9080
|
+
lines.push("");
|
|
9081
|
+
lines.push(paint("q quit | Ctrl-C exit", DIM, color));
|
|
9082
|
+
return `${lines.map((line) => fitLine(line, columns, color)).join(`
|
|
9083
|
+
`)}
|
|
9084
|
+
`;
|
|
9085
|
+
}
|
|
9086
|
+
async function runLoopsUiApp(opts = {}) {
|
|
9087
|
+
const refreshMs = Math.max(MIN_REFRESH_MS, opts.refreshMs ?? DEFAULT_REFRESH_MS);
|
|
9088
|
+
const input = opts.input ?? process.stdin;
|
|
9089
|
+
const output = opts.output ?? process.stdout;
|
|
9090
|
+
const storeFactory = opts.storeFactory ?? (() => new Store);
|
|
9091
|
+
const rawInput = input;
|
|
9092
|
+
const signalExitCodes = { SIGINT: 130, SIGTERM: 143 };
|
|
9093
|
+
const signalHandlers = [];
|
|
9094
|
+
let interval;
|
|
9095
|
+
let closed = false;
|
|
9096
|
+
let cleaned = false;
|
|
9097
|
+
let rawModeEnabled = false;
|
|
9098
|
+
let terminalEntered = false;
|
|
9099
|
+
let store;
|
|
9100
|
+
let stopApp;
|
|
9101
|
+
let renderHandler;
|
|
9102
|
+
const render = () => {
|
|
9103
|
+
if (closed || !store)
|
|
9104
|
+
return;
|
|
9105
|
+
const snapshot = buildLoopUiSnapshot(store);
|
|
9106
|
+
output.write(`${CLEAR_SCREEN}${renderLoopUiFrame(snapshot, {
|
|
9107
|
+
columns: output.columns,
|
|
9108
|
+
rows: output.rows,
|
|
9109
|
+
refreshMs,
|
|
9110
|
+
color: true
|
|
9111
|
+
})}`);
|
|
9112
|
+
};
|
|
9113
|
+
const onData = (chunk) => {
|
|
9114
|
+
const value = chunk.toString("utf8");
|
|
9115
|
+
if (value.toLowerCase().includes("q") || value.includes("\x03") || value.includes("\x1B"))
|
|
9116
|
+
stopApp?.();
|
|
9117
|
+
};
|
|
9118
|
+
const cleanup = () => {
|
|
9119
|
+
if (cleaned)
|
|
9120
|
+
return;
|
|
9121
|
+
cleaned = true;
|
|
9122
|
+
if (interval)
|
|
9123
|
+
clearInterval(interval);
|
|
9124
|
+
input.off("data", onData);
|
|
9125
|
+
if (renderHandler)
|
|
9126
|
+
output.off("resize", renderHandler);
|
|
9127
|
+
for (const [signal, handler] of signalHandlers)
|
|
9128
|
+
process.removeListener(signal, handler);
|
|
9129
|
+
if (rawModeEnabled)
|
|
9130
|
+
rawInput.setRawMode?.(false);
|
|
9131
|
+
if (terminalEntered) {
|
|
9132
|
+
try {
|
|
9133
|
+
output.write(`${SHOW_CURSOR}${CLEAR_SCREEN}${EXIT_ALT_SCREEN}`);
|
|
9134
|
+
} catch {}
|
|
9135
|
+
}
|
|
9136
|
+
try {
|
|
9137
|
+
store?.close();
|
|
9138
|
+
} catch {}
|
|
9139
|
+
};
|
|
9140
|
+
try {
|
|
9141
|
+
store = storeFactory();
|
|
9142
|
+
output.write(`${ENTER_ALT_SCREEN}${HIDE_CURSOR}${CLEAR_SCREEN}`);
|
|
9143
|
+
terminalEntered = true;
|
|
9144
|
+
await new Promise((resolve3, reject) => {
|
|
9145
|
+
const fail = (error) => {
|
|
9146
|
+
if (closed)
|
|
9147
|
+
return;
|
|
9148
|
+
closed = true;
|
|
9149
|
+
cleanup();
|
|
9150
|
+
reject(error);
|
|
9151
|
+
};
|
|
9152
|
+
const stop = () => {
|
|
9153
|
+
if (closed)
|
|
9154
|
+
return;
|
|
9155
|
+
closed = true;
|
|
9156
|
+
cleanup();
|
|
9157
|
+
resolve3();
|
|
9158
|
+
};
|
|
9159
|
+
const safeRender = () => {
|
|
9160
|
+
try {
|
|
9161
|
+
render();
|
|
9162
|
+
} catch (error) {
|
|
9163
|
+
fail(error);
|
|
9164
|
+
}
|
|
9165
|
+
};
|
|
9166
|
+
stopApp = stop;
|
|
9167
|
+
renderHandler = safeRender;
|
|
9168
|
+
for (const signal of ["SIGINT", "SIGTERM"]) {
|
|
9169
|
+
const handler = () => {
|
|
9170
|
+
process.exitCode = signalExitCodes[signal] ?? 1;
|
|
9171
|
+
stop();
|
|
9172
|
+
};
|
|
9173
|
+
signalHandlers.push([signal, handler]);
|
|
9174
|
+
process.once(signal, handler);
|
|
9175
|
+
}
|
|
9176
|
+
if (typeof rawInput.setRawMode === "function") {
|
|
9177
|
+
rawInput.setRawMode(true);
|
|
9178
|
+
rawModeEnabled = true;
|
|
9179
|
+
}
|
|
9180
|
+
input.resume();
|
|
9181
|
+
input.on("data", onData);
|
|
9182
|
+
output.on("resize", safeRender);
|
|
9183
|
+
safeRender();
|
|
9184
|
+
if (!closed)
|
|
9185
|
+
interval = setInterval(safeRender, refreshMs);
|
|
9186
|
+
});
|
|
9187
|
+
} finally {
|
|
9188
|
+
cleanup();
|
|
9189
|
+
}
|
|
9190
|
+
}
|
|
9191
|
+
function tableWidths(columns, compact2) {
|
|
9192
|
+
const widths = {
|
|
9193
|
+
name: 20,
|
|
9194
|
+
status: compact2 ? 6 : 7,
|
|
9195
|
+
cadence: compact2 ? 10 : 14,
|
|
9196
|
+
nextRun: compact2 ? 8 : 10,
|
|
9197
|
+
lastRunOutcome: compact2 ? 8 : 11,
|
|
9198
|
+
provider: compact2 ? 7 : 10,
|
|
9199
|
+
activeRuns: compact2 ? 3 : 11
|
|
9200
|
+
};
|
|
9201
|
+
const fixed = widths.status + widths.cadence + widths.nextRun + widths.lastRunOutcome + widths.provider + widths.activeRuns + 12;
|
|
9202
|
+
widths.name = Math.max(6, columns - fixed);
|
|
9203
|
+
return widths;
|
|
9204
|
+
}
|
|
9205
|
+
function tableWidth(widths) {
|
|
9206
|
+
return widths.name + widths.status + widths.cadence + widths.nextRun + widths.lastRunOutcome + widths.provider + widths.activeRuns + 12;
|
|
9207
|
+
}
|
|
9208
|
+
function tableHeader(widths, compact2) {
|
|
9209
|
+
return [
|
|
9210
|
+
cell("NAME", widths.name),
|
|
9211
|
+
cell("STATUS", widths.status),
|
|
9212
|
+
cell("CADENCE", widths.cadence),
|
|
9213
|
+
cell("NEXT-RUN", widths.nextRun),
|
|
9214
|
+
cell("LAST-RUN", widths.lastRunOutcome),
|
|
9215
|
+
cell("PROVIDER", widths.provider),
|
|
9216
|
+
cell(compact2 ? "RUN" : "ACTIVE-RUNS", widths.activeRuns, "right")
|
|
9217
|
+
].join(" ");
|
|
9218
|
+
}
|
|
9219
|
+
function tableRow(row, widths, color) {
|
|
9220
|
+
return [
|
|
9221
|
+
cell(row.name, widths.name),
|
|
9222
|
+
paint(cell(row.status, widths.status), statusColor(row.status), color),
|
|
9223
|
+
cell(row.cadence, widths.cadence),
|
|
9224
|
+
cell(row.nextRun, widths.nextRun),
|
|
9225
|
+
paint(cell(row.lastRunOutcome, widths.lastRunOutcome), outcomeColor(row.lastRunOutcome), color),
|
|
9226
|
+
cell(row.provider, widths.provider),
|
|
9227
|
+
cell(String(row.activeRuns), widths.activeRuns, "right")
|
|
9228
|
+
].join(" ");
|
|
9229
|
+
}
|
|
9230
|
+
function cell(value, width, align = "left") {
|
|
9231
|
+
const clipped = clip(value, width);
|
|
9232
|
+
return align === "right" ? clipped.padStart(width) : clipped.padEnd(width);
|
|
9233
|
+
}
|
|
9234
|
+
function clip(value, width) {
|
|
9235
|
+
if (width <= 0)
|
|
9236
|
+
return "";
|
|
9237
|
+
if (value.length <= width)
|
|
9238
|
+
return value;
|
|
9239
|
+
if (width === 1)
|
|
9240
|
+
return "~";
|
|
9241
|
+
return `${value.slice(0, width - 1)}~`;
|
|
9242
|
+
}
|
|
9243
|
+
function fitLine(value, columns, hasAnsi = false) {
|
|
9244
|
+
if (!hasAnsi)
|
|
9245
|
+
return clip(value, columns).trimEnd();
|
|
9246
|
+
let visible = 0;
|
|
9247
|
+
let index = 0;
|
|
9248
|
+
let out = "";
|
|
9249
|
+
while (index < value.length) {
|
|
9250
|
+
const char = value[index];
|
|
9251
|
+
if (char === "\x1B" && value[index + 1] === "[") {
|
|
9252
|
+
let end = index + 2;
|
|
9253
|
+
while (end < value.length && !/[A-Za-z]/.test(value[end]))
|
|
9254
|
+
end += 1;
|
|
9255
|
+
out += value.slice(index, Math.min(end + 1, value.length));
|
|
9256
|
+
index = end + 1;
|
|
9257
|
+
continue;
|
|
9258
|
+
}
|
|
9259
|
+
if (visible < columns) {
|
|
9260
|
+
out += char;
|
|
9261
|
+
visible += 1;
|
|
9262
|
+
}
|
|
9263
|
+
index += 1;
|
|
9264
|
+
}
|
|
9265
|
+
return out;
|
|
9266
|
+
}
|
|
9267
|
+
function paint(value, code, enabled) {
|
|
9268
|
+
return enabled ? `${code}${value}${RESET}` : value;
|
|
9269
|
+
}
|
|
9270
|
+
function statusColor(status) {
|
|
9271
|
+
if (status === "active")
|
|
9272
|
+
return GREEN;
|
|
9273
|
+
if (status === "paused")
|
|
9274
|
+
return YELLOW;
|
|
9275
|
+
if (status === "expired")
|
|
9276
|
+
return MAGENTA;
|
|
9277
|
+
return DIM;
|
|
9278
|
+
}
|
|
9279
|
+
function outcomeColor(outcome) {
|
|
9280
|
+
if (outcome.startsWith("succeeded"))
|
|
9281
|
+
return GREEN;
|
|
9282
|
+
if (outcome.startsWith("running"))
|
|
9283
|
+
return CYAN;
|
|
9284
|
+
if (outcome.startsWith("failed") || outcome.startsWith("timed_out") || outcome.startsWith("abandoned"))
|
|
9285
|
+
return RED;
|
|
9286
|
+
if (outcome.startsWith("skipped"))
|
|
9287
|
+
return YELLOW;
|
|
9288
|
+
return DIM;
|
|
9289
|
+
}
|
|
9290
|
+
function runOutcomeLabel(run) {
|
|
9291
|
+
if (!run)
|
|
9292
|
+
return "-";
|
|
9293
|
+
if (run.status === "failed" && run.exitCode !== undefined)
|
|
9294
|
+
return `failed(${run.exitCode})`;
|
|
9295
|
+
return run.status;
|
|
9296
|
+
}
|
|
9297
|
+
function providerLabel(target) {
|
|
9298
|
+
if (target.type === "command")
|
|
9299
|
+
return "command";
|
|
9300
|
+
if (target.type === "agent")
|
|
9301
|
+
return target.provider;
|
|
9302
|
+
return "workflow";
|
|
9303
|
+
}
|
|
9304
|
+
function scheduleLabel(schedule) {
|
|
9305
|
+
if (schedule.type === "once")
|
|
9306
|
+
return "once";
|
|
9307
|
+
if (schedule.type === "interval")
|
|
9308
|
+
return `every:${durationLabel(schedule.everyMs) || `${schedule.everyMs}ms`}`;
|
|
9309
|
+
if (schedule.type === "cron")
|
|
9310
|
+
return `cron:${schedule.expression}`;
|
|
9311
|
+
return schedule.minIntervalMs ? `dynamic:${durationLabel(schedule.minIntervalMs) || `${schedule.minIntervalMs}ms`}` : "dynamic";
|
|
9312
|
+
}
|
|
9313
|
+
function nextRunLabel(nextRunAt, now) {
|
|
9314
|
+
if (!nextRunAt)
|
|
9315
|
+
return "-";
|
|
9316
|
+
const date = new Date(nextRunAt);
|
|
9317
|
+
if (Number.isNaN(date.getTime()))
|
|
9318
|
+
return "invalid";
|
|
9319
|
+
return relativeTime(date.getTime() - now.getTime());
|
|
9320
|
+
}
|
|
9321
|
+
function relativeTime(deltaMs) {
|
|
9322
|
+
const past = deltaMs < 0;
|
|
9323
|
+
const abs = Math.abs(deltaMs);
|
|
9324
|
+
const value = durationLabel(abs);
|
|
9325
|
+
if (!value || value === "0ms")
|
|
9326
|
+
return "now";
|
|
9327
|
+
return past ? `${value} ago` : `in ${value}`;
|
|
9328
|
+
}
|
|
9329
|
+
function durationLabel(ms) {
|
|
9330
|
+
if (ms === undefined || !Number.isFinite(ms))
|
|
9331
|
+
return "";
|
|
9332
|
+
if (ms < 1000)
|
|
9333
|
+
return `${Math.max(0, Math.round(ms))}ms`;
|
|
9334
|
+
const units = [
|
|
9335
|
+
[7 * 24 * 60 * 60 * 1000, "w"],
|
|
9336
|
+
[24 * 60 * 60 * 1000, "d"],
|
|
9337
|
+
[60 * 60 * 1000, "h"],
|
|
9338
|
+
[60 * 1000, "m"],
|
|
9339
|
+
[1000, "s"]
|
|
9340
|
+
];
|
|
9341
|
+
for (const [unitMs, label] of units) {
|
|
9342
|
+
const value = ms / unitMs;
|
|
9343
|
+
if (value >= 1)
|
|
9344
|
+
return `${Math.round(value)}${label}`;
|
|
9345
|
+
}
|
|
9346
|
+
return `${ms}ms`;
|
|
9347
|
+
}
|
|
9348
|
+
function timeOnly(value) {
|
|
9349
|
+
const date = new Date(value);
|
|
9350
|
+
if (Number.isNaN(date.getTime()))
|
|
9351
|
+
return value;
|
|
9352
|
+
return date.toISOString().slice(11, 19);
|
|
9353
|
+
}
|
|
9354
|
+
|
|
8250
9355
|
// src/lib/migration.ts
|
|
8251
9356
|
import { createHash as createHash3 } from "crypto";
|
|
8252
9357
|
import { hostname as hostname3 } from "os";
|
|
@@ -8758,7 +9863,7 @@ function selfHostedControlPlaneSummary(env = process.env) {
|
|
|
8758
9863
|
|
|
8759
9864
|
// src/lib/hygiene.ts
|
|
8760
9865
|
import { basename as basename3 } from "path";
|
|
8761
|
-
import { homedir as
|
|
9866
|
+
import { homedir as homedir4 } from "os";
|
|
8762
9867
|
var PROVIDER_TOKENS = new Set([
|
|
8763
9868
|
"codewith",
|
|
8764
9869
|
"claude",
|
|
@@ -8774,7 +9879,7 @@ var REPO_GENERIC_TOKENS = new Set(["repo", "repoops"]);
|
|
|
8774
9879
|
var CADENCE_SUFFIX_TOKENS = new Set(["hourly", "daily", "weekly", "monthly"]);
|
|
8775
9880
|
var CADENCE_SUFFIX_PATTERN = /^(?:every-?)?\d+(?:s|m|h|d|w)$/;
|
|
8776
9881
|
function userHome() {
|
|
8777
|
-
return process.env.HOME ||
|
|
9882
|
+
return process.env.HOME || homedir4();
|
|
8778
9883
|
}
|
|
8779
9884
|
function slugify(value) {
|
|
8780
9885
|
return value.normalize("NFKD").replace(/[^\w\s.-]/g, "-").replace(/[_\s.:/]+/g, "-").replace(/[^a-zA-Z0-9-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").toLowerCase();
|
|
@@ -9012,14 +10117,14 @@ function buildScriptInventoryReport(store, opts = {}) {
|
|
|
9012
10117
|
// src/lib/templates.ts
|
|
9013
10118
|
import { execFileSync } from "child_process";
|
|
9014
10119
|
import { existsSync as existsSync7 } from "fs";
|
|
9015
|
-
import { homedir as
|
|
9016
|
-
import { basename as basename4, isAbsolute as isAbsolute2, join as
|
|
10120
|
+
import { homedir as homedir5 } from "os";
|
|
10121
|
+
import { basename as basename4, isAbsolute as isAbsolute2, join as join9, relative, resolve as resolve4 } from "path";
|
|
9017
10122
|
|
|
9018
10123
|
// src/lib/template-kit.ts
|
|
9019
10124
|
var TODOS_TASK_WORKER_VERIFIER_TEMPLATE_ID = "todos-task-worker-verifier";
|
|
9020
10125
|
var EVENT_WORKER_VERIFIER_TEMPLATE_ID = "event-worker-verifier";
|
|
9021
10126
|
var BOUNDED_AGENT_WORKER_VERIFIER_TEMPLATE_ID = "bounded-agent-worker-verifier";
|
|
9022
|
-
var
|
|
10127
|
+
var TASK_LIFECYCLE_TEMPLATE_ID2 = "task-lifecycle";
|
|
9023
10128
|
var PR_REVIEW_TEMPLATE_ID = "pr-review";
|
|
9024
10129
|
var SCHEDULED_AUDIT_TEMPLATE_ID = "scheduled-audit";
|
|
9025
10130
|
var KNOWLEDGE_REFRESH_TEMPLATE_ID = "knowledge-refresh";
|
|
@@ -9126,7 +10231,7 @@ var BUILTIN_TEMPLATE_SUMMARIES = [
|
|
|
9126
10231
|
]
|
|
9127
10232
|
},
|
|
9128
10233
|
{
|
|
9129
|
-
id:
|
|
10234
|
+
id: TASK_LIFECYCLE_TEMPLATE_ID2,
|
|
9130
10235
|
name: "Task Lifecycle",
|
|
9131
10236
|
description: "Run the standard task-created lifecycle: triage/dedupe, plan, worker execution, independent verification, and todos closure/follow-up evidence.",
|
|
9132
10237
|
kind: "workflow",
|
|
@@ -9685,8 +10790,8 @@ function assignPoolAuthProfiles(input) {
|
|
|
9685
10790
|
return { profiles, deferred: false, minLoad: Number.isFinite(minLoad) ? minLoad : 0 };
|
|
9686
10791
|
}
|
|
9687
10792
|
// src/lib/templates-custom.ts
|
|
9688
|
-
import { existsSync as existsSync6, lstatSync as lstatSync2, mkdirSync as
|
|
9689
|
-
import { join as
|
|
10793
|
+
import { existsSync as existsSync6, lstatSync as lstatSync2, mkdirSync as mkdirSync8, readdirSync, readFileSync as readFileSync6, writeFileSync as writeFileSync5 } from "fs";
|
|
10794
|
+
import { join as join8, resolve as resolve3 } from "path";
|
|
9690
10795
|
var CUSTOM_TEMPLATE_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
|
9691
10796
|
var CUSTOM_TEMPLATE_VARIABLE_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
9692
10797
|
var CUSTOM_TEMPLATE_VARIABLE_TYPES = new Set(["string", "number", "boolean", "json", "string[]"]);
|
|
@@ -9713,11 +10818,11 @@ function assertTemplateKind(value, label) {
|
|
|
9713
10818
|
return kind;
|
|
9714
10819
|
}
|
|
9715
10820
|
function customLoopTemplatesDir() {
|
|
9716
|
-
return
|
|
10821
|
+
return join8(dataDir(), "templates");
|
|
9717
10822
|
}
|
|
9718
10823
|
function ensureCustomLoopTemplatesDir() {
|
|
9719
10824
|
const dir = customLoopTemplatesDir();
|
|
9720
|
-
|
|
10825
|
+
mkdirSync8(dir, { recursive: true, mode: 448 });
|
|
9721
10826
|
return dir;
|
|
9722
10827
|
}
|
|
9723
10828
|
function validateCustomTemplateId(id, label) {
|
|
@@ -9848,7 +10953,7 @@ function customTemplateSummary(definition, sourcePath) {
|
|
|
9848
10953
|
function readCustomTemplateFile(file) {
|
|
9849
10954
|
let parsed;
|
|
9850
10955
|
try {
|
|
9851
|
-
parsed = JSON.parse(
|
|
10956
|
+
parsed = JSON.parse(readFileSync6(file, "utf8"));
|
|
9852
10957
|
} catch (error) {
|
|
9853
10958
|
const message = error instanceof Error ? error.message : String(error);
|
|
9854
10959
|
throw new Error(`failed to read custom template ${file}: ${message}`);
|
|
@@ -9876,7 +10981,7 @@ function loadCustomLoopTemplatesRaw() {
|
|
|
9876
10981
|
if (!existsSync6(dir))
|
|
9877
10982
|
return [];
|
|
9878
10983
|
return readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.name.endsWith(".json")).sort((left, right) => left.name.localeCompare(right.name)).map((entry) => {
|
|
9879
|
-
const file =
|
|
10984
|
+
const file = join8(dir, entry.name);
|
|
9880
10985
|
if (entry.isSymbolicLink())
|
|
9881
10986
|
throw new Error(`refusing symlinked custom template file: ${file}`);
|
|
9882
10987
|
if (!entry.isFile())
|
|
@@ -9996,7 +11101,7 @@ function importCustomLoopTemplate(file, reservedKeys, opts = {}) {
|
|
|
9996
11101
|
const source = resolve3(file);
|
|
9997
11102
|
const entry = readCustomTemplateFile(source);
|
|
9998
11103
|
const dir = ensureCustomLoopTemplatesDir();
|
|
9999
|
-
const destination =
|
|
11104
|
+
const destination = join8(dir, `${entry.definition.id}.json`);
|
|
10000
11105
|
const replaced = existsSync6(destination);
|
|
10001
11106
|
const existing = loadCustomLoopTemplatesRaw().filter((template) => resolve3(template.path) !== resolve3(destination));
|
|
10002
11107
|
assertNoTemplateCollisions([...existing, { ...entry, path: destination, summary: customTemplateSummary(entry.definition, destination) }], reservedKeys);
|
|
@@ -10007,7 +11112,7 @@ function importCustomLoopTemplate(file, reservedKeys, opts = {}) {
|
|
|
10007
11112
|
if (!opts.replace)
|
|
10008
11113
|
throw new Error(`custom template already exists: ${entry.definition.id}; use --replace to overwrite it`);
|
|
10009
11114
|
}
|
|
10010
|
-
|
|
11115
|
+
writeFileSync5(destination, `${JSON.stringify(entry.definition, null, 2)}
|
|
10011
11116
|
`, { mode: 384 });
|
|
10012
11117
|
const imported = readCustomTemplateFile(destination);
|
|
10013
11118
|
return { template: structuredClone(imported.summary), path: destination, replaced };
|
|
@@ -10125,10 +11230,10 @@ function normalizeWorktreeMode(mode) {
|
|
|
10125
11230
|
}
|
|
10126
11231
|
function defaultWorktreeRoot(root) {
|
|
10127
11232
|
if (root?.trim()) {
|
|
10128
|
-
const expanded = root.trim().replace(/^~(?=$|\/)/,
|
|
11233
|
+
const expanded = root.trim().replace(/^~(?=$|\/)/, homedir5());
|
|
10129
11234
|
return isAbsolute2(expanded) ? expanded : resolve4(expanded);
|
|
10130
11235
|
}
|
|
10131
|
-
return
|
|
11236
|
+
return join9(homedir5(), ".hasna", "loops", "worktrees");
|
|
10132
11237
|
}
|
|
10133
11238
|
function gitRootFor(path) {
|
|
10134
11239
|
if (!existsSync7(path))
|
|
@@ -10185,9 +11290,9 @@ function worktreePlan(input, seed) {
|
|
|
10185
11290
|
const root = defaultWorktreeRoot(input.worktreeRoot);
|
|
10186
11291
|
const repoSlug = slugSegment(basename4(repoRoot), "repo");
|
|
10187
11292
|
const seedSlug = `${slugSegment(seed, "run").slice(0, 48)}-${stableHex(`${repoRoot}:${seed}`)}`;
|
|
10188
|
-
const worktreePath =
|
|
11293
|
+
const worktreePath = join9(root, repoSlug, seedSlug);
|
|
10189
11294
|
const relativeCwd = relative(repoRoot, originalCwd);
|
|
10190
|
-
const cwd = relativeCwd && !relativeCwd.startsWith("..") && !isAbsolute2(relativeCwd) ?
|
|
11295
|
+
const cwd = relativeCwd && !relativeCwd.startsWith("..") && !isAbsolute2(relativeCwd) ? join9(worktreePath, relativeCwd) : worktreePath;
|
|
10191
11296
|
const branchPrefix = (input.worktreeBranchPrefix?.trim() || "openloops").replace(/^\/+|\/+$/g, "") || "openloops";
|
|
10192
11297
|
const branch = `${branchPrefix}/${repoSlug}/${seedSlug}`;
|
|
10193
11298
|
return {
|
|
@@ -10306,7 +11411,7 @@ function lifecycleGateStep(opts) {
|
|
|
10306
11411
|
});
|
|
10307
11412
|
}
|
|
10308
11413
|
function prHandoffArtifactPath(plan, taskId) {
|
|
10309
|
-
return
|
|
11414
|
+
return join9(plan.cwd, ".openloops", "pr-handoff", `${slugSegment(taskId, "task")}.json`);
|
|
10310
11415
|
}
|
|
10311
11416
|
function prHandoffStep(input, plan, todosProjectPath) {
|
|
10312
11417
|
return commandStep({
|
|
@@ -10823,7 +11928,7 @@ var BOUNDED_LIFECYCLE_TEMPLATES = {
|
|
|
10823
11928
|
};
|
|
10824
11929
|
function renderLifecycleBoundedTemplate(id, values) {
|
|
10825
11930
|
const base = agentTemplateInput(values);
|
|
10826
|
-
if (id ===
|
|
11931
|
+
if (id === TASK_LIFECYCLE_TEMPLATE_ID2) {
|
|
10827
11932
|
const taskId = values.taskId ?? "";
|
|
10828
11933
|
if (!taskId.trim())
|
|
10829
11934
|
throw new Error("taskId is required");
|
|
@@ -10939,8 +12044,8 @@ function renderLoopTemplate(id, values, opts = {}) {
|
|
|
10939
12044
|
|
|
10940
12045
|
// src/lib/backup.ts
|
|
10941
12046
|
import { Database as Database2 } from "bun:sqlite";
|
|
10942
|
-
import { chmodSync as chmodSync3, existsSync as existsSync8, mkdirSync as
|
|
10943
|
-
import { dirname as dirname6, join as
|
|
12047
|
+
import { chmodSync as chmodSync3, existsSync as existsSync8, mkdirSync as mkdirSync9, readdirSync as readdirSync2, rmSync as rmSync6 } from "fs";
|
|
12048
|
+
import { dirname as dirname6, join as join10 } from "path";
|
|
10944
12049
|
var DEFAULT_KEEP = 3;
|
|
10945
12050
|
var DEBOUNCE_MS = 60 * 60 * 1000;
|
|
10946
12051
|
function reasonSlug(reason) {
|
|
@@ -10971,10 +12076,10 @@ function backupDatabase(opts) {
|
|
|
10971
12076
|
if (!existsSync8(file)) {
|
|
10972
12077
|
return { skipped: true, skipReason: `database file not found: ${file}`, prunedPaths: [] };
|
|
10973
12078
|
}
|
|
10974
|
-
const dir = opts.backupsDir ??
|
|
12079
|
+
const dir = opts.backupsDir ?? join10(dirname6(file), "backups");
|
|
10975
12080
|
const slug = reasonSlug(opts.reason);
|
|
10976
12081
|
const now = opts.now ?? new Date;
|
|
10977
|
-
|
|
12082
|
+
mkdirSync9(dir, { recursive: true, mode: 448 });
|
|
10978
12083
|
const existing = listBackups(dir, slug);
|
|
10979
12084
|
const newest = existing[0];
|
|
10980
12085
|
if (newest && now.getTime() - newest.timeMs < DEBOUNCE_MS) {
|
|
@@ -10984,12 +12089,12 @@ function backupDatabase(opts) {
|
|
|
10984
12089
|
prunedPaths: []
|
|
10985
12090
|
};
|
|
10986
12091
|
}
|
|
10987
|
-
const target =
|
|
12092
|
+
const target = join10(dir, `loops-${slug}-${backupStamp(now)}.db`);
|
|
10988
12093
|
const db = new Database2(file, { readonly: true });
|
|
10989
12094
|
try {
|
|
10990
12095
|
db.query("VACUUM INTO ?").run(target);
|
|
10991
12096
|
} catch (error) {
|
|
10992
|
-
|
|
12097
|
+
rmSync6(target, { force: true });
|
|
10993
12098
|
throw error;
|
|
10994
12099
|
} finally {
|
|
10995
12100
|
db.close();
|
|
@@ -10997,8 +12102,8 @@ function backupDatabase(opts) {
|
|
|
10997
12102
|
chmodSync3(target, 384);
|
|
10998
12103
|
const prunedPaths = [];
|
|
10999
12104
|
for (const entry of listBackups(dir, slug).slice(keep)) {
|
|
11000
|
-
const path =
|
|
11001
|
-
|
|
12105
|
+
const path = join10(dir, entry.name);
|
|
12106
|
+
rmSync6(path, { force: true });
|
|
11002
12107
|
prunedPaths.push(path);
|
|
11003
12108
|
}
|
|
11004
12109
|
return { path: target, skipped: false, prunedPaths };
|
|
@@ -11103,6 +12208,14 @@ function taskEventField(data, keys) {
|
|
|
11103
12208
|
if (direct)
|
|
11104
12209
|
return direct;
|
|
11105
12210
|
}
|
|
12211
|
+
const payloadTask = payload.task;
|
|
12212
|
+
if (payloadTask && typeof payloadTask === "object" && !Array.isArray(payloadTask)) {
|
|
12213
|
+
for (const key of keys) {
|
|
12214
|
+
const direct = stringField(payloadTask[key]);
|
|
12215
|
+
if (direct)
|
|
12216
|
+
return direct;
|
|
12217
|
+
}
|
|
12218
|
+
}
|
|
11106
12219
|
}
|
|
11107
12220
|
return;
|
|
11108
12221
|
}
|
|
@@ -11315,17 +12428,17 @@ function routeFieldList(records, fields) {
|
|
|
11315
12428
|
}
|
|
11316
12429
|
// src/lib/route/cursors.ts
|
|
11317
12430
|
import { randomUUID } from "crypto";
|
|
11318
|
-
import { existsSync as existsSync9, mkdirSync as
|
|
11319
|
-
import { join as
|
|
12431
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync10, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "fs";
|
|
12432
|
+
import { join as join11 } from "path";
|
|
11320
12433
|
function routeCursorsPath() {
|
|
11321
|
-
return
|
|
12434
|
+
return join11(dataDir(), "route-cursors.json");
|
|
11322
12435
|
}
|
|
11323
12436
|
function readRouteCursors() {
|
|
11324
12437
|
const path = routeCursorsPath();
|
|
11325
12438
|
if (!existsSync9(path))
|
|
11326
12439
|
return {};
|
|
11327
12440
|
try {
|
|
11328
|
-
const parsed = JSON.parse(
|
|
12441
|
+
const parsed = JSON.parse(readFileSync7(path, "utf8"));
|
|
11329
12442
|
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
11330
12443
|
} catch {
|
|
11331
12444
|
return {};
|
|
@@ -11336,15 +12449,15 @@ function writeRouteCursor(key, lastFingerprint) {
|
|
|
11336
12449
|
return;
|
|
11337
12450
|
const cursors = readRouteCursors();
|
|
11338
12451
|
cursors[key] = { lastFingerprint, updatedAt: new Date().toISOString() };
|
|
11339
|
-
|
|
12452
|
+
writeFileSync6(routeCursorsPath(), JSON.stringify(cursors, null, 2), { mode: 384 });
|
|
11340
12453
|
}
|
|
11341
12454
|
function writeRouteEvidence(kind, value, evidenceDir) {
|
|
11342
12455
|
if (!evidenceDir)
|
|
11343
12456
|
return;
|
|
11344
|
-
|
|
12457
|
+
mkdirSync10(evidenceDir, { recursive: true, mode: 448 });
|
|
11345
12458
|
const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\./g, "");
|
|
11346
|
-
const evidencePath =
|
|
11347
|
-
|
|
12459
|
+
const evidencePath = join11(evidenceDir, `${kind}-${stamp}-${randomUUID().slice(0, 8)}.json`);
|
|
12460
|
+
writeFileSync6(evidencePath, JSON.stringify(value, null, 2), { mode: 384, flag: "wx" });
|
|
11348
12461
|
return evidencePath;
|
|
11349
12462
|
}
|
|
11350
12463
|
function selectRouteItems(items, maxActions, cursorKey, fingerprintOf) {
|
|
@@ -11378,7 +12491,7 @@ function routeCursorKey(kind, parts, opts) {
|
|
|
11378
12491
|
return `${kind}:${stableHash([...parts, ...routeMode])}`;
|
|
11379
12492
|
}
|
|
11380
12493
|
// src/lib/route/gates.ts
|
|
11381
|
-
import { readFileSync as
|
|
12494
|
+
import { readFileSync as readFileSync8 } from "fs";
|
|
11382
12495
|
import { dirname as dirname7, resolve as resolve5 } from "path";
|
|
11383
12496
|
class GateError extends CodedError {
|
|
11384
12497
|
gate;
|
|
@@ -11414,7 +12527,7 @@ function normalizeWorkflowForStorage(body, context) {
|
|
|
11414
12527
|
function workflowBodyFromFile(file, fallbackName, context) {
|
|
11415
12528
|
try {
|
|
11416
12529
|
const resolved = resolve5(file);
|
|
11417
|
-
return workflowBodyFromJson(JSON.parse(
|
|
12530
|
+
return workflowBodyFromJson(JSON.parse(readFileSync8(resolved, "utf8")), fallbackName, { baseDir: dirname7(resolved) });
|
|
11418
12531
|
} catch (error) {
|
|
11419
12532
|
gateFailure("validation", error, context);
|
|
11420
12533
|
}
|
|
@@ -11603,7 +12716,7 @@ function permissionModeFromOpts(opts, provider) {
|
|
|
11603
12716
|
return mode;
|
|
11604
12717
|
}
|
|
11605
12718
|
// src/lib/route/pr-review.ts
|
|
11606
|
-
import { spawnSync as
|
|
12719
|
+
import { spawnSync as spawnSync8 } from "child_process";
|
|
11607
12720
|
var PR_AUTHOR_FIELDS = [
|
|
11608
12721
|
"github_author",
|
|
11609
12722
|
"githubAuthor",
|
|
@@ -11787,13 +12900,13 @@ function prFingerprintFromTask(data, metadata) {
|
|
|
11787
12900
|
return ref ? prFingerprint(ref) : undefined;
|
|
11788
12901
|
}
|
|
11789
12902
|
function ghAuthorResolver(ref) {
|
|
11790
|
-
const result =
|
|
12903
|
+
const result = spawnSync8("gh", ["pr", "view", String(ref.number), "--repo", `${ref.owner}/${ref.repo}`, "--json", "author", "-q", ".author.login"], { encoding: "utf8", timeout: 20000 });
|
|
11791
12904
|
if (result.error || result.status !== 0)
|
|
11792
12905
|
return;
|
|
11793
12906
|
return githubLogin((result.stdout ?? "").trim());
|
|
11794
12907
|
}
|
|
11795
12908
|
function ghStateResolver(ref) {
|
|
11796
|
-
const result =
|
|
12909
|
+
const result = spawnSync8("gh", ["pr", "view", String(ref.number), "--repo", `${ref.owner}/${ref.repo}`, "--json", "state,mergeStateStatus"], { encoding: "utf8", timeout: 20000 });
|
|
11797
12910
|
if (result.error || result.status !== 0)
|
|
11798
12911
|
return;
|
|
11799
12912
|
try {
|
|
@@ -11922,7 +13035,7 @@ function prReviewRoutingDecision(data, metadata, opts, resolveAuthor = ghAuthorR
|
|
|
11922
13035
|
}
|
|
11923
13036
|
// src/lib/route/throttle.ts
|
|
11924
13037
|
import { realpathSync as realpathSync2 } from "fs";
|
|
11925
|
-
import { spawnSync as
|
|
13038
|
+
import { spawnSync as spawnSync9 } from "child_process";
|
|
11926
13039
|
import { resolve as resolve6 } from "path";
|
|
11927
13040
|
function routeThrottleLimitsFromOpts(opts) {
|
|
11928
13041
|
return {
|
|
@@ -11944,7 +13057,7 @@ function normalizeRoutePath(value) {
|
|
|
11944
13057
|
} catch {
|
|
11945
13058
|
return canonical;
|
|
11946
13059
|
}
|
|
11947
|
-
const gitRoot =
|
|
13060
|
+
const gitRoot = spawnSync9("git", ["-C", canonical, "rev-parse", "--show-toplevel"], { encoding: "utf8" });
|
|
11948
13061
|
if (gitRoot.status === 0 && gitRoot.stdout.trim()) {
|
|
11949
13062
|
try {
|
|
11950
13063
|
return realpathSync2(gitRoot.stdout.trim());
|
|
@@ -11995,7 +13108,7 @@ function routeThrottleDryRunPreview(args) {
|
|
|
11995
13108
|
};
|
|
11996
13109
|
}
|
|
11997
13110
|
function isExistingGitProjectPath(path) {
|
|
11998
|
-
const result =
|
|
13111
|
+
const result = spawnSync9("git", ["-C", path, "rev-parse", "--is-inside-work-tree"], { encoding: "utf8" });
|
|
11999
13112
|
return result.status === 0;
|
|
12000
13113
|
}
|
|
12001
13114
|
function validateRequiredRouteWorktreeProjectPath(opts, projectPath) {
|
|
@@ -12006,7 +13119,7 @@ function validateRequiredRouteWorktreeProjectPath(opts, projectPath) {
|
|
|
12006
13119
|
}
|
|
12007
13120
|
}
|
|
12008
13121
|
// src/lib/route/route-event.ts
|
|
12009
|
-
import { readFileSync as
|
|
13122
|
+
import { readFileSync as readFileSync9 } from "fs";
|
|
12010
13123
|
import { resolve as resolve7 } from "path";
|
|
12011
13124
|
function generatedRouteSandboxPreflight(workflow) {
|
|
12012
13125
|
const checks = [];
|
|
@@ -12062,7 +13175,7 @@ function routeWorkflowForStorage(store, workflowBody) {
|
|
|
12062
13175
|
}
|
|
12063
13176
|
var TODOS_TASK_ROUTE_TEMPLATE_IDS = new Set([
|
|
12064
13177
|
TODOS_TASK_WORKER_VERIFIER_TEMPLATE_ID,
|
|
12065
|
-
|
|
13178
|
+
TASK_LIFECYCLE_TEMPLATE_ID2
|
|
12066
13179
|
]);
|
|
12067
13180
|
var UNCLEARED_ROUTE_WORK_ITEM_STATUSES = new Set([
|
|
12068
13181
|
"admitted",
|
|
@@ -12103,6 +13216,36 @@ function reactivateStaleTodosTaskWorkItem(store, routeKey, item, now = Date.now(
|
|
|
12103
13216
|
reason: `re-admitted from ${item.status}: todos task still actionable after prior run (attempt ${item.attempts + 1}/${MAX_TODOS_TASK_ROUTE_REDISPATCHES})`
|
|
12104
13217
|
});
|
|
12105
13218
|
}
|
|
13219
|
+
function findRouteWorkItemByKeys(store, routeKey, idempotencyKeys) {
|
|
13220
|
+
const [primaryKey, ...aliasKeys] = idempotencyKeys;
|
|
13221
|
+
if (primaryKey) {
|
|
13222
|
+
const existingItem = store.findWorkflowWorkItem(routeKey, primaryKey);
|
|
13223
|
+
if (existingItem && isUnclearedRouteWorkItem(existingItem))
|
|
13224
|
+
return existingItem;
|
|
13225
|
+
}
|
|
13226
|
+
for (const key of aliasKeys) {
|
|
13227
|
+
const existingItem = store.findWorkflowWorkItem(routeKey, key);
|
|
13228
|
+
if (existingItem && (isUnclearedRouteWorkItem(existingItem) || existingItem.status === "queued" || existingItem.status === "deferred"))
|
|
13229
|
+
return existingItem;
|
|
13230
|
+
}
|
|
13231
|
+
return;
|
|
13232
|
+
}
|
|
13233
|
+
function isPrBacklogTask(data, metadata) {
|
|
13234
|
+
const explicitFingerprint = taskEventField(data, [
|
|
13235
|
+
"pr_fingerprint",
|
|
13236
|
+
"prFingerprint",
|
|
13237
|
+
"github_pr",
|
|
13238
|
+
"githubPr",
|
|
13239
|
+
"github_pr_fingerprint",
|
|
13240
|
+
"githubPrFingerprint",
|
|
13241
|
+
"pull_request_fingerprint",
|
|
13242
|
+
"pullRequestFingerprint"
|
|
13243
|
+
]);
|
|
13244
|
+
if (explicitFingerprint)
|
|
13245
|
+
return true;
|
|
13246
|
+
const tags = new Set(taskEventTags(taskEventRecords(data, metadata)).map((tag) => tag.toLowerCase()));
|
|
13247
|
+
return tags.has("github-pr") && tags.has("pr-merge-queue");
|
|
13248
|
+
}
|
|
12106
13249
|
function todosTaskRouteTemplateId(opts) {
|
|
12107
13250
|
const id = (opts.template ?? TODOS_TASK_WORKER_VERIFIER_TEMPLATE_ID).trim();
|
|
12108
13251
|
if (!TODOS_TASK_ROUTE_TEMPLATE_IDS.has(id)) {
|
|
@@ -12111,7 +13254,7 @@ function todosTaskRouteTemplateId(opts) {
|
|
|
12111
13254
|
return id;
|
|
12112
13255
|
}
|
|
12113
13256
|
async function readEventEnvelopeInput(opts = {}) {
|
|
12114
|
-
const raw = opts.eventJson ?? (opts.eventFile ?
|
|
13257
|
+
const raw = opts.eventJson ?? (opts.eventFile ? readFileSync9(opts.eventFile, "utf8") : process.env.HASNA_EVENT_JSON || await Bun.stdin.text());
|
|
12115
13258
|
const event = JSON.parse(raw);
|
|
12116
13259
|
if (!event || typeof event !== "object" || Array.isArray(event))
|
|
12117
13260
|
throw new ValidationError("event JSON must be an object");
|
|
@@ -12170,6 +13313,7 @@ function dedupedRoutePrint(plan, outcome) {
|
|
|
12170
13313
|
function routeEvent(plan) {
|
|
12171
13314
|
const { event, opts, idempotencyKey, workflowBody } = plan;
|
|
12172
13315
|
const sandboxPreflight = generatedRouteSandboxPreflight(workflowBody);
|
|
13316
|
+
const dedupeKeys = [idempotencyKey, ...plan.dedupeAliases ?? []];
|
|
12173
13317
|
const workItemInput = {
|
|
12174
13318
|
routeKey: plan.routeKey,
|
|
12175
13319
|
idempotencyKey,
|
|
@@ -12222,8 +13366,8 @@ function routeEvent(plan) {
|
|
|
12222
13366
|
let poolAssignment;
|
|
12223
13367
|
const outcome = store.writeTransaction(() => {
|
|
12224
13368
|
const invocation = store.createWorkflowInvocation(plan.invocationInput);
|
|
12225
|
-
const existingItem = store
|
|
12226
|
-
if (existingItem
|
|
13369
|
+
const existingItem = findRouteWorkItemByKeys(store, plan.routeKey, dedupeKeys);
|
|
13370
|
+
if (existingItem) {
|
|
12227
13371
|
if (!reactivateStaleTodosTaskWorkItem(store, plan.routeKey, existingItem)) {
|
|
12228
13372
|
const existingLoop = existingItem.loopId ? store.getLoop(existingItem.loopId) : undefined;
|
|
12229
13373
|
const existingWorkflow = existingItem.workflowId ? store.getWorkflow(existingItem.workflowId) : undefined;
|
|
@@ -12386,8 +13530,10 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12386
13530
|
const projectGroup = routeProjectGroup(opts.projectGroup, data, metadata);
|
|
12387
13531
|
const throttleLimits = routeThrottleLimitsFromOpts(opts);
|
|
12388
13532
|
const sourceProjectIdempotencyPrefix = sourceTodosProjectPath ? normalizeRoutePath(sourceTodosProjectPath) ?? resolve7(sourceTodosProjectPath) : undefined;
|
|
12389
|
-
const prFingerprint2 = prFingerprintFromTask(data, metadata);
|
|
13533
|
+
const prFingerprint2 = isPrBacklogTask(data, metadata) ? prFingerprintFromTask(data, metadata) : undefined;
|
|
12390
13534
|
const idempotencyKey = prFingerprint2 ? `todos-task:pr:${prFingerprint2}` : sourceProjectIdempotencyPrefix ? `todos-task:${sourceProjectIdempotencyPrefix}:${taskId}` : `todos-task:${taskId}`;
|
|
13535
|
+
const legacyTaskIdempotencyKey = `todos-task:${taskId}`;
|
|
13536
|
+
const dedupeAliases = legacyTaskIdempotencyKey === idempotencyKey ? [] : [legacyTaskIdempotencyKey];
|
|
12391
13537
|
const idempotencySuffix = stableSuffix(idempotencyKey);
|
|
12392
13538
|
const namePrefix = opts.namePrefix ?? "event:todos-task";
|
|
12393
13539
|
const workflowName = `${namePrefix}:${taskId.slice(0, 8)}:${idempotencySuffix}:workflow`;
|
|
@@ -12395,8 +13541,8 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12395
13541
|
if (!opts.dryRun) {
|
|
12396
13542
|
const store = new Store;
|
|
12397
13543
|
try {
|
|
12398
|
-
const existingItem = store
|
|
12399
|
-
if (existingItem
|
|
13544
|
+
const existingItem = findRouteWorkItemByKeys(store, "todos-task", [idempotencyKey, ...dedupeAliases]);
|
|
13545
|
+
if (existingItem) {
|
|
12400
13546
|
if (!reactivateStaleTodosTaskWorkItem(store, "todos-task", existingItem)) {
|
|
12401
13547
|
const existingLoop = existingItem.loopId ? store.getLoop(existingItem.loopId) : undefined;
|
|
12402
13548
|
const existingWorkflow = existingItem.workflowId ? store.getWorkflow(existingItem.workflowId) : undefined;
|
|
@@ -12463,7 +13609,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12463
13609
|
worktreeMode: opts.worktreeMode ?? "auto",
|
|
12464
13610
|
worktreeRoot: opts.worktreeRoot,
|
|
12465
13611
|
worktreeBranchPrefix: opts.worktreeBranchPrefix ?? "openloops",
|
|
12466
|
-
prHandoff: templateId ===
|
|
13612
|
+
prHandoff: templateId === TASK_LIFECYCLE_TEMPLATE_ID2 ? Boolean(opts.prHandoff) : false,
|
|
12467
13613
|
prReviewRouting: prReviewRouting.required ? prReviewRouting : undefined,
|
|
12468
13614
|
eventId: event.id,
|
|
12469
13615
|
eventType: event.type,
|
|
@@ -12474,7 +13620,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12474
13620
|
type: "todos-task-event-workflow",
|
|
12475
13621
|
event: event.id
|
|
12476
13622
|
};
|
|
12477
|
-
let workflowBody = templateId ===
|
|
13623
|
+
let workflowBody = templateId === TASK_LIFECYCLE_TEMPLATE_ID2 ? renderTaskLifecycleWorkflow(workflowInput) : renderTodosTaskWorkerVerifierWorkflow(workflowInput);
|
|
12478
13624
|
workflowBody.name = workflowName;
|
|
12479
13625
|
workflowBody.description = `Task-triggered ${templateId} workflow for ${taskTitle ?? taskId} from ${event.source}/${event.type}; ` + `idempotency=${idempotencyKey}; event=${event.id}; project=${projectPath}; projectGroup=${projectGroup ?? "-"}`;
|
|
12480
13626
|
workflowBody = normalizeWorkflowForStorage(workflowBody, workflowContext);
|
|
@@ -12500,7 +13646,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12500
13646
|
worktreePolicy: opts.worktreeMode ?? "auto",
|
|
12501
13647
|
permissions: permissionMode,
|
|
12502
13648
|
manualBreakGlass: Boolean(opts.manualBreakGlass),
|
|
12503
|
-
prHandoff: templateId ===
|
|
13649
|
+
prHandoff: templateId === TASK_LIFECYCLE_TEMPLATE_ID2 ? Boolean(opts.prHandoff) : false,
|
|
12504
13650
|
accountPolicy: providerRouting.authProfilePool?.length || providerRouting.accountPool?.length ? "pool" : hasExplicitRoleAccount ? "role-explicit" : "single",
|
|
12505
13651
|
providerRouting: providerRoutingPublic(providerRouting),
|
|
12506
13652
|
prReviewRouting: prReviewRouting.required ? prReviewRouting : undefined,
|
|
@@ -12516,6 +13662,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12516
13662
|
event,
|
|
12517
13663
|
opts,
|
|
12518
13664
|
idempotencyKey,
|
|
13665
|
+
dedupeAliases,
|
|
12519
13666
|
workflowBody,
|
|
12520
13667
|
workflowContext,
|
|
12521
13668
|
invocationInput,
|
|
@@ -12657,80 +13804,6 @@ function routeEventByKind(kind, event, opts) {
|
|
|
12657
13804
|
}
|
|
12658
13805
|
// src/lib/route/drain.ts
|
|
12659
13806
|
import { resolve as resolve8 } from "path";
|
|
12660
|
-
|
|
12661
|
-
// src/lib/route/todos-cli.ts
|
|
12662
|
-
import { closeSync, mkdtempSync as mkdtempSync2, openSync as openSync2, readFileSync as readFileSync9, rmSync as rmSync6 } from "fs";
|
|
12663
|
-
import { spawnSync as spawnSync9 } from "child_process";
|
|
12664
|
-
import { join as join10 } from "path";
|
|
12665
|
-
import { homedir as homedir5, tmpdir as tmpdir2 } from "os";
|
|
12666
|
-
function defaultLoopsProject() {
|
|
12667
|
-
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir5()}/.hasna/loops`;
|
|
12668
|
-
}
|
|
12669
|
-
function runLocalCommand(command, args, opts = {}) {
|
|
12670
|
-
const result = spawnSync9(command, args, {
|
|
12671
|
-
input: opts.input,
|
|
12672
|
-
encoding: "utf8",
|
|
12673
|
-
timeout: opts.timeoutMs ?? 30000,
|
|
12674
|
-
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
12675
|
-
env: process.env
|
|
12676
|
-
});
|
|
12677
|
-
return {
|
|
12678
|
-
ok: result.status === 0,
|
|
12679
|
-
status: result.status,
|
|
12680
|
-
stdout: result.stdout || "",
|
|
12681
|
-
stderr: result.stderr || "",
|
|
12682
|
-
error: result.error ? String(result.error.message || result.error) : ""
|
|
12683
|
-
};
|
|
12684
|
-
}
|
|
12685
|
-
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
12686
|
-
const tempDir = mkdtempSync2(join10(tmpdir2(), "loops-command-output-"));
|
|
12687
|
-
const stdoutPath = join10(tempDir, "stdout");
|
|
12688
|
-
const stdoutFd = openSync2(stdoutPath, "w");
|
|
12689
|
-
let result;
|
|
12690
|
-
try {
|
|
12691
|
-
result = spawnSync9(command, args, {
|
|
12692
|
-
input: opts.input,
|
|
12693
|
-
encoding: "utf8",
|
|
12694
|
-
timeout: opts.timeoutMs ?? 30000,
|
|
12695
|
-
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
12696
|
-
env: process.env,
|
|
12697
|
-
stdio: ["pipe", stdoutFd, "pipe"]
|
|
12698
|
-
});
|
|
12699
|
-
} finally {
|
|
12700
|
-
closeSync(stdoutFd);
|
|
12701
|
-
}
|
|
12702
|
-
try {
|
|
12703
|
-
return {
|
|
12704
|
-
ok: result.status === 0,
|
|
12705
|
-
status: result.status,
|
|
12706
|
-
stdout: readFileSync9(stdoutPath, "utf8"),
|
|
12707
|
-
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
12708
|
-
error: result.error ? String(result.error.message || result.error) : ""
|
|
12709
|
-
};
|
|
12710
|
-
} finally {
|
|
12711
|
-
rmSync6(tempDir, { recursive: true, force: true });
|
|
12712
|
-
}
|
|
12713
|
-
}
|
|
12714
|
-
function ensureTodosTaskList(project, slug, name, description) {
|
|
12715
|
-
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
12716
|
-
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
12717
|
-
if (!list.ok)
|
|
12718
|
-
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
12719
|
-
const values = JSON.parse(list.stdout || "[]");
|
|
12720
|
-
const found = values.find((entry) => entry.slug === slug);
|
|
12721
|
-
if (!found)
|
|
12722
|
-
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
12723
|
-
return found.id;
|
|
12724
|
-
}
|
|
12725
|
-
function todosMutationSummary(result) {
|
|
12726
|
-
return {
|
|
12727
|
-
ok: result.ok,
|
|
12728
|
-
status: result.status,
|
|
12729
|
-
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
12730
|
-
};
|
|
12731
|
-
}
|
|
12732
|
-
|
|
12733
|
-
// src/lib/route/drain.ts
|
|
12734
13807
|
function taskField(task, keys) {
|
|
12735
13808
|
for (const key of keys) {
|
|
12736
13809
|
const value = stringField(task[key]);
|
|
@@ -13633,6 +14706,24 @@ function print(value, human) {
|
|
|
13633
14706
|
else
|
|
13634
14707
|
console.log(human);
|
|
13635
14708
|
}
|
|
14709
|
+
var LOOP_STATUS_VALUES = ["active", "paused", "stopped", "expired"];
|
|
14710
|
+
function parseLoopStatuses(value, label = "--include") {
|
|
14711
|
+
const raw = splitList(value) ?? ["active", "paused"];
|
|
14712
|
+
const expanded = raw.flatMap((entry) => entry === "all" ? LOOP_STATUS_VALUES : [entry]);
|
|
14713
|
+
const invalid = expanded.filter((entry) => !LOOP_STATUS_VALUES.includes(entry));
|
|
14714
|
+
if (invalid.length > 0)
|
|
14715
|
+
throw new ValidationError(`${label} has invalid loop status: ${invalid.join(", ")}`);
|
|
14716
|
+
return [...new Set(expanded)];
|
|
14717
|
+
}
|
|
14718
|
+
function compactHealthScanOutput(scan) {
|
|
14719
|
+
if (!scan || typeof scan !== "object" || !("health" in scan))
|
|
14720
|
+
return scan;
|
|
14721
|
+
const healthValue = scan.health;
|
|
14722
|
+
if (!healthValue || typeof healthValue !== "object")
|
|
14723
|
+
return scan;
|
|
14724
|
+
const { expectations: _expectations, ...health } = healthValue;
|
|
14725
|
+
return { ...scan, health };
|
|
14726
|
+
}
|
|
13636
14727
|
function reportCliError(error) {
|
|
13637
14728
|
const message = error instanceof Error ? error.message : String(error);
|
|
13638
14729
|
process.exitCode = 1;
|
|
@@ -13723,6 +14814,15 @@ function workflowWithAgentTimeouts(workflow, timeoutMs, opts = {}) {
|
|
|
13723
14814
|
agentStepIds
|
|
13724
14815
|
};
|
|
13725
14816
|
}
|
|
14817
|
+
function agentLoopTargetWithTimeout(loop, timeoutMs) {
|
|
14818
|
+
if (loop.target.type !== "agent")
|
|
14819
|
+
throw new Error(`loop is not an agent loop: ${loop.name || loop.id}`);
|
|
14820
|
+
const target = { ...loop.target, timeoutMs };
|
|
14821
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
14822
|
+
delete target.idleTimeoutMs;
|
|
14823
|
+
const changed = loop.target.timeoutMs !== timeoutMs || timeoutMs === null && loop.target.idleTimeoutMs !== undefined;
|
|
14824
|
+
return { changed, target };
|
|
14825
|
+
}
|
|
13726
14826
|
function workflowTimeoutMigrationName(workflow, timeoutMs) {
|
|
13727
14827
|
const policy = timeoutMs === null ? "agent-timeout-unlimited" : `agent-timeout-${timeoutMs}ms`;
|
|
13728
14828
|
const suffix = randomUUID2().replace(/-/g, "").slice(0, 8);
|
|
@@ -13801,7 +14901,7 @@ function parsePolicy(opts) {
|
|
|
13801
14901
|
leaseMs: positiveDuration(opts.lease, "--lease")
|
|
13802
14902
|
};
|
|
13803
14903
|
}
|
|
13804
|
-
function
|
|
14904
|
+
function durationLabel2(ms) {
|
|
13805
14905
|
if (!ms || !Number.isFinite(ms))
|
|
13806
14906
|
return "";
|
|
13807
14907
|
const units = [
|
|
@@ -13817,14 +14917,14 @@ function durationLabel(ms) {
|
|
|
13817
14917
|
}
|
|
13818
14918
|
return `${ms}ms`;
|
|
13819
14919
|
}
|
|
13820
|
-
function
|
|
14920
|
+
function scheduleLabel2(schedule) {
|
|
13821
14921
|
if (schedule.type === "once")
|
|
13822
14922
|
return `once:${schedule.at}`;
|
|
13823
14923
|
if (schedule.type === "interval")
|
|
13824
|
-
return `every:${
|
|
14924
|
+
return `every:${durationLabel2(schedule.everyMs) || `${schedule.everyMs}ms`}`;
|
|
13825
14925
|
if (schedule.type === "cron")
|
|
13826
14926
|
return `cron:${schedule.expression}`;
|
|
13827
|
-
return schedule.minIntervalMs ? `dynamic:min-${
|
|
14927
|
+
return schedule.minIntervalMs ? `dynamic:min-${durationLabel2(schedule.minIntervalMs) || `${schedule.minIntervalMs}ms`}` : "dynamic";
|
|
13828
14928
|
}
|
|
13829
14929
|
function targetLabel(target) {
|
|
13830
14930
|
if (target.type === "command")
|
|
@@ -13836,7 +14936,7 @@ function targetLabel(target) {
|
|
|
13836
14936
|
function defaultLoopDescription(name, schedule, target) {
|
|
13837
14937
|
return [
|
|
13838
14938
|
`Why: keep ${name} running as an OpenLoops scheduled automation.`,
|
|
13839
|
-
`How: ${targetLabel(target)} on cadence ${
|
|
14939
|
+
`How: ${targetLabel(target)} on cadence ${scheduleLabel2(schedule)}.`,
|
|
13840
14940
|
"Outcome: record each run, status, retries, and evidence in OpenLoops for operator review."
|
|
13841
14941
|
].join(" ");
|
|
13842
14942
|
}
|
|
@@ -14066,7 +15166,7 @@ program.command("export").description("export a local OpenLoops migration bundle
|
|
|
14066
15166
|
throw new ValidationError("export is not no-loss because redactions/blockers are present; rerun with --allow-redacted to write a redacted bundle");
|
|
14067
15167
|
}
|
|
14068
15168
|
if (!opts.dryRun)
|
|
14069
|
-
|
|
15169
|
+
writeFileSync7(opts.file, `${JSON.stringify(bundle, null, 2)}
|
|
14070
15170
|
`, { mode: 384 });
|
|
14071
15171
|
const output = {
|
|
14072
15172
|
ok: true,
|
|
@@ -14364,6 +15464,7 @@ addScheduleOptions(addTodosDrainOptions(routes.command("schedule <kind> <name>")
|
|
|
14364
15464
|
})).action(runAction((kind, name, opts) => {
|
|
14365
15465
|
if (kind !== "todos-task")
|
|
14366
15466
|
throw new ValidationError("route schedule currently supports kind todos-task");
|
|
15467
|
+
todosTaskRouteTemplateId(opts);
|
|
14367
15468
|
const store = new Store;
|
|
14368
15469
|
try {
|
|
14369
15470
|
const target = {
|
|
@@ -14603,7 +15704,7 @@ workflows.command("recover <runId>").description("reset interrupted running work
|
|
|
14603
15704
|
store.close();
|
|
14604
15705
|
}
|
|
14605
15706
|
}));
|
|
14606
|
-
workflows.command("migrate-agent-timeouts").description("
|
|
15707
|
+
workflows.command("migrate-agent-timeouts").description("migrate workflow loops, or a direct agent loop selected with --loop, to a new agent timeout policy").option("--loop <idOrName>", "migrate only one loop; required for direct agent loops").option("--timeout <duration>", "agent timeout policy; use none/unlimited for no timeout", "none").option("--apply", "create new workflow specs or update direct agent targets for eligible loops").option("--archive-old", "archive old workflow specs after retargeting when no active loops still reference them").action(runAction((opts) => {
|
|
14607
15708
|
const store = new Store;
|
|
14608
15709
|
try {
|
|
14609
15710
|
const timeoutMs = timeoutDuration(opts.timeout, "--timeout") ?? null;
|
|
@@ -14614,8 +15715,45 @@ workflows.command("migrate-agent-timeouts").description("append-only migrate act
|
|
|
14614
15715
|
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "loop is archived" });
|
|
14615
15716
|
continue;
|
|
14616
15717
|
}
|
|
15718
|
+
if (loop.target.type === "agent") {
|
|
15719
|
+
const migration2 = agentLoopTargetWithTimeout(loop, timeoutMs);
|
|
15720
|
+
if (!migration2.changed) {
|
|
15721
|
+
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "agent timeout policy already matches" });
|
|
15722
|
+
continue;
|
|
15723
|
+
}
|
|
15724
|
+
if (store.hasRunningRun(loop.id)) {
|
|
15725
|
+
rows.push({ loop: publicLoop(loop), status: "blocked", reason: "loop has a running run; retry after it finishes" });
|
|
15726
|
+
continue;
|
|
15727
|
+
}
|
|
15728
|
+
if (!opts.apply) {
|
|
15729
|
+
rows.push({
|
|
15730
|
+
loop: publicLoop(loop),
|
|
15731
|
+
status: "would_update",
|
|
15732
|
+
target: { ...migration2.target, prompt: redact(migration2.target.prompt) },
|
|
15733
|
+
timeoutMs
|
|
15734
|
+
});
|
|
15735
|
+
continue;
|
|
15736
|
+
}
|
|
15737
|
+
try {
|
|
15738
|
+
const updated = store.updateAgentLoopTimeout(loop.id, timeoutMs);
|
|
15739
|
+
rows.push({
|
|
15740
|
+
loop: publicLoop(updated),
|
|
15741
|
+
previousLoop: publicLoop(loop),
|
|
15742
|
+
status: "updated",
|
|
15743
|
+
timeoutMs
|
|
15744
|
+
});
|
|
15745
|
+
} catch (error) {
|
|
15746
|
+
rows.push({
|
|
15747
|
+
loop: publicLoop(loop),
|
|
15748
|
+
status: "blocked",
|
|
15749
|
+
reason: migrationErrorReason(error),
|
|
15750
|
+
timeoutMs
|
|
15751
|
+
});
|
|
15752
|
+
}
|
|
15753
|
+
continue;
|
|
15754
|
+
}
|
|
14617
15755
|
if (loop.target.type !== "workflow") {
|
|
14618
|
-
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "loop is not
|
|
15756
|
+
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "loop is not an agent or workflow loop" });
|
|
14619
15757
|
continue;
|
|
14620
15758
|
}
|
|
14621
15759
|
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
@@ -14674,11 +15812,13 @@ workflows.command("migrate-agent-timeouts").description("append-only migrate act
|
|
|
14674
15812
|
timeoutMs,
|
|
14675
15813
|
total: rows.length,
|
|
14676
15814
|
migrated: rows.filter((row) => row.status === "migrated").length,
|
|
15815
|
+
updated: rows.filter((row) => row.status === "updated").length,
|
|
14677
15816
|
wouldMigrate: rows.filter((row) => row.status === "would_migrate").length,
|
|
15817
|
+
wouldUpdate: rows.filter((row) => row.status === "would_update").length,
|
|
14678
15818
|
blocked: rows.filter((row) => row.status === "blocked").length,
|
|
14679
15819
|
skipped: rows.filter((row) => row.status === "skipped").length
|
|
14680
15820
|
};
|
|
14681
|
-
print({ summary, rows }, opts.apply ? `migrated=${summary.migrated} blocked=${summary.blocked} skipped=${summary.skipped}` : `would_migrate=${summary.wouldMigrate} blocked=${summary.blocked} skipped=${summary.skipped}`);
|
|
15821
|
+
print({ summary, rows }, opts.apply ? `migrated=${summary.migrated} updated=${summary.updated} blocked=${summary.blocked} skipped=${summary.skipped}` : `would_migrate=${summary.wouldMigrate} would_update=${summary.wouldUpdate} blocked=${summary.blocked} skipped=${summary.skipped}`);
|
|
14682
15822
|
} finally {
|
|
14683
15823
|
store.close();
|
|
14684
15824
|
}
|
|
@@ -14792,13 +15932,23 @@ program.command("list").alias("ls").description("list loops with schedule and ne
|
|
|
14792
15932
|
for (const loop of loops) {
|
|
14793
15933
|
const machine = loop.machine ? ` machine=${loop.machine.id}` : "";
|
|
14794
15934
|
const archive = loop.archivedAt ? ` archived=${loop.archivedAt} from=${loop.archivedFromStatus ?? "-"}` : "";
|
|
14795
|
-
console.log(`${loop.id} ${loop.status.padEnd(7)} cadence=${
|
|
15935
|
+
console.log(`${loop.id} ${loop.status.padEnd(7)} cadence=${scheduleLabel2(loop.schedule)} next=${loop.nextRunAt ?? "-"} ${loop.name}${machine}${archive}`);
|
|
14796
15936
|
}
|
|
14797
15937
|
}
|
|
14798
15938
|
} finally {
|
|
14799
15939
|
store.close();
|
|
14800
15940
|
}
|
|
14801
15941
|
}));
|
|
15942
|
+
program.command("ui").description("open a live table of active loops").option("--refresh <duration>", "refresh interval", "2s").action(runAction(async (opts) => {
|
|
15943
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
15944
|
+
console.error("OpenLoops UI requires a TTY terminal.");
|
|
15945
|
+
console.error("Use `loops list`, `loops runs`, or `loops daemon status` non-interactively.");
|
|
15946
|
+
process.exitCode = 1;
|
|
15947
|
+
return;
|
|
15948
|
+
}
|
|
15949
|
+
const refreshMs = Math.max(500, parseDuration(opts.refresh ?? "2s"));
|
|
15950
|
+
await runLoopsUiApp({ refreshMs });
|
|
15951
|
+
}));
|
|
14802
15952
|
program.command("show <idOrName>").description("show one loop by id or name").action(runAction((idOrName) => {
|
|
14803
15953
|
const store = new Store;
|
|
14804
15954
|
try {
|
|
@@ -14853,8 +16003,9 @@ var health = program.command("health").description("summarize loop health and la
|
|
|
14853
16003
|
console.log(JSON.stringify(report, null, 2));
|
|
14854
16004
|
else {
|
|
14855
16005
|
console.log(`loops=${report.summary.loops} healthy=${report.summary.healthy} unhealthy=${report.summary.unhealthy} warnings=${report.summary.warnings}`);
|
|
14856
|
-
for (const expectation of report.expectations.filter((entry) => !entry.ok)) {
|
|
14857
|
-
|
|
16006
|
+
for (const expectation of report.expectations.filter((entry) => !entry.ok || entry.check.status === "warn")) {
|
|
16007
|
+
const status = expectation.ok ? "warn" : "fail";
|
|
16008
|
+
console.log(`${status} ${expectation.loop.name} ${expectation.failure?.classification ?? "unknown"} ${expectation.failure?.fingerprint ?? "-"}`);
|
|
14858
16009
|
}
|
|
14859
16010
|
}
|
|
14860
16011
|
if (!report.ok)
|
|
@@ -14863,6 +16014,122 @@ var health = program.command("health").description("summarize loop health and la
|
|
|
14863
16014
|
store.close();
|
|
14864
16015
|
}
|
|
14865
16016
|
}));
|
|
16017
|
+
health.command("scan").description("scan OpenLoops health, write bounded reports, and optionally upsert deduped todos findings").option("--include <statuses>", "comma-separated loop statuses to inventory: active,paused,stopped,expired,all", "active,paused").option("--limit <n>", "maximum loops to inspect", "200").option("--max-findings <n>", "maximum findings to include in output", "100").option("--latest-run", "include latest-run and stale-running checks").option("--no-latest-run", "skip latest-run and stale-running checks").option("--stale-running-after <duration>", "minimum age before a running latest run is stale; loop lease and 10m still apply").option("--doctor", "include doctor/preflight checks").option("--daemon", "include daemon status").option("--start-daemon", "safe self-heal: start the daemon if it is not running").option("--report-dir <path>", "write summary.json and report.md under this reports root").option("--evidence-dir <path>", "alias for --report-dir and todo evidence output").option("--upsert-todos", "upsert deduped todos tasks for scan findings").option("--project <path>", "todos project path for --upsert-todos", defaultLoopsProject()).option("--task-list <slug>", "todos task-list slug for --upsert-todos", "loop-error-self-heal").option("--max-actions <n>", "maximum todos tasks to upsert", "5").option("--dry-run", "with --upsert-todos, print intended task upserts without mutating todos").option("--auto-route", "with --upsert-todos, opt routed tasks into task-created headless worker/verifier automation").option("--route-project-path <path>", "fallback project path for --auto-route when the finding has no cwd").option("-j, --json", "print JSON for this command").action(runAction(async (opts) => {
|
|
16018
|
+
const store = new Store;
|
|
16019
|
+
try {
|
|
16020
|
+
const includeStatuses = parseLoopStatuses(opts.include, "--include");
|
|
16021
|
+
let daemon = opts.daemon || opts.startDaemon ? daemonStatus(store) : undefined;
|
|
16022
|
+
const selfHeals = [];
|
|
16023
|
+
if (opts.startDaemon) {
|
|
16024
|
+
if (daemon?.running) {
|
|
16025
|
+
selfHeals.push({
|
|
16026
|
+
kind: "daemon-start",
|
|
16027
|
+
attempted: false,
|
|
16028
|
+
ok: true,
|
|
16029
|
+
reason: "daemon already running"
|
|
16030
|
+
});
|
|
16031
|
+
} else {
|
|
16032
|
+
const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops" });
|
|
16033
|
+
const ok = Boolean(result.started || result.alreadyRunning);
|
|
16034
|
+
selfHeals.push({
|
|
16035
|
+
kind: "daemon-start",
|
|
16036
|
+
attempted: true,
|
|
16037
|
+
ok,
|
|
16038
|
+
reason: daemon?.stale ? "daemon pid file was stale" : "daemon was not running",
|
|
16039
|
+
result
|
|
16040
|
+
});
|
|
16041
|
+
daemon = daemonStatus(store);
|
|
16042
|
+
}
|
|
16043
|
+
}
|
|
16044
|
+
let scan = writeHealthScanReports(buildHealthScan(store, {
|
|
16045
|
+
includeStatuses,
|
|
16046
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 200,
|
|
16047
|
+
maxFindings: nonNegativeInteger(opts.maxFindings, "--max-findings") ?? 100,
|
|
16048
|
+
latestRun: opts.latestRun !== false,
|
|
16049
|
+
staleRunningMs: opts.staleRunningAfter ? positiveDuration(opts.staleRunningAfter, "--stale-running-after") : undefined,
|
|
16050
|
+
doctor: opts.doctor ? runDoctor(store) : undefined,
|
|
16051
|
+
daemon,
|
|
16052
|
+
selfHeals
|
|
16053
|
+
}), { reportDir: opts.reportDir ?? opts.evidenceDir });
|
|
16054
|
+
if (opts.upsertTodos) {
|
|
16055
|
+
const tasks = scan.findings.filter((finding) => finding.recommendedTask).map((finding) => {
|
|
16056
|
+
const task = finding.recommendedTask;
|
|
16057
|
+
const description = [
|
|
16058
|
+
task.description,
|
|
16059
|
+
scan.reports ? `Report: ${scan.reports.markdown}` : undefined
|
|
16060
|
+
].filter(Boolean).join(`
|
|
16061
|
+
|
|
16062
|
+
`);
|
|
16063
|
+
return {
|
|
16064
|
+
title: task.title,
|
|
16065
|
+
description,
|
|
16066
|
+
priority: task.priority,
|
|
16067
|
+
tags: task.tags,
|
|
16068
|
+
fingerprint: task.dedupeKey,
|
|
16069
|
+
extra: {
|
|
16070
|
+
kind: finding.kind,
|
|
16071
|
+
severity: finding.severity,
|
|
16072
|
+
classification: finding.classification
|
|
16073
|
+
},
|
|
16074
|
+
metadata: {
|
|
16075
|
+
source: "openloops.health.scan",
|
|
16076
|
+
kind: finding.kind,
|
|
16077
|
+
severity: finding.severity,
|
|
16078
|
+
loop_id: finding.loop?.id,
|
|
16079
|
+
loop_name: finding.loop?.name,
|
|
16080
|
+
loop_status: finding.loop?.status,
|
|
16081
|
+
run_id: finding.run?.id,
|
|
16082
|
+
classification: finding.classification,
|
|
16083
|
+
fingerprint: task.dedupeKey,
|
|
16084
|
+
cwd: finding.route?.cwd,
|
|
16085
|
+
provider: finding.route?.provider,
|
|
16086
|
+
report_dir: scan.reports?.dir,
|
|
16087
|
+
no_tmux_dispatch: true
|
|
16088
|
+
}
|
|
16089
|
+
};
|
|
16090
|
+
});
|
|
16091
|
+
const result = upsertRouteTasks({
|
|
16092
|
+
project: opts.project,
|
|
16093
|
+
taskList: {
|
|
16094
|
+
slug: opts.taskList,
|
|
16095
|
+
name: "Loop Error Self Heal",
|
|
16096
|
+
description: "Deduped OpenLoops health scan findings for daemon, doctor, preflight, latest-run, and stale-running issues."
|
|
16097
|
+
},
|
|
16098
|
+
cursorKey: routeCursorKey("health", ["scan", opts.project, opts.taskList, includeStatuses.join(","), opts.limit, Boolean(opts.doctor), Boolean(opts.daemon)], { autoRoute: Boolean(opts.autoRoute), routeProjectPath: opts.routeProjectPath }),
|
|
16099
|
+
maxActions: positiveInteger(opts.maxActions, "--max-actions") ?? 5,
|
|
16100
|
+
dryRun: Boolean(opts.dryRun),
|
|
16101
|
+
autoRoute: Boolean(opts.autoRoute),
|
|
16102
|
+
routeProjectPath: opts.routeProjectPath,
|
|
16103
|
+
source: "openloops.health.scan",
|
|
16104
|
+
evidence: { kind: "health-scan-route-tasks", dir: opts.evidenceDir ?? opts.reportDir },
|
|
16105
|
+
summary: {
|
|
16106
|
+
status: scan.status,
|
|
16107
|
+
inspected: scan.counts.loops,
|
|
16108
|
+
findings: scan.counts.findings,
|
|
16109
|
+
reportDir: scan.reports?.dir
|
|
16110
|
+
},
|
|
16111
|
+
tasks
|
|
16112
|
+
});
|
|
16113
|
+
scan = { ...scan, todos: result.output };
|
|
16114
|
+
if (!result.ok)
|
|
16115
|
+
process.exitCode = 1;
|
|
16116
|
+
}
|
|
16117
|
+
const jsonMode = isJson() || Boolean(opts.json);
|
|
16118
|
+
if (jsonMode)
|
|
16119
|
+
console.log(JSON.stringify(compactHealthScanOutput(scan), null, 2));
|
|
16120
|
+
else {
|
|
16121
|
+
const actions = Array.isArray(scan.todos?.actions) ? scan.todos.actions : [];
|
|
16122
|
+
console.log(`health_scan status=${scan.status} loops=${scan.counts.loops} findings=${scan.counts.findings} ` + `reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} ` + `latest=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} ` + `daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings} ` + `report=${scan.reports?.markdown ?? "none"} todos_actions=${actions.length}`);
|
|
16123
|
+
for (const finding of scan.findings) {
|
|
16124
|
+
console.log(`${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop?.name ?? ""} ${finding.message}`);
|
|
16125
|
+
}
|
|
16126
|
+
}
|
|
16127
|
+
if (!process.exitCode && scan.status !== "ok")
|
|
16128
|
+
process.exitCode = scan.status === "critical" ? 2 : 1;
|
|
16129
|
+
} finally {
|
|
16130
|
+
store.close();
|
|
16131
|
+
}
|
|
16132
|
+
}));
|
|
14866
16133
|
health.command("route-tasks").description("upsert deduped todos tasks for failed loop health expectations").option("--project <path>", "todos project path", defaultLoopsProject()).option("--task-list <slug>", "todos task-list slug", "loop-error-self-heal").option("--limit <n>", "maximum loops to inspect", "200").option("--max-actions <n>", "maximum todos tasks to upsert", "5").option("--include-inactive", "also route stopped or expired loops").option("--auto-route", "opt routed tasks into task-created headless worker/verifier automation").option("--route-project-path <path>", "fallback project path for --auto-route when the failed loop has no cwd").option("--evidence-dir <path>", "write the route result JSON to this directory").option("--dry-run", "print intended task upserts without mutating todos").action(runAction((opts) => {
|
|
14867
16134
|
const store = new Store;
|
|
14868
16135
|
try {
|
|
@@ -15221,7 +16488,7 @@ function listManagedBackups(dir) {
|
|
|
15221
16488
|
const group = modern ? `reason:${modern[1]}` : legacy ? `legacy:${legacy[1]}` : undefined;
|
|
15222
16489
|
if (!group)
|
|
15223
16490
|
continue;
|
|
15224
|
-
const path =
|
|
16491
|
+
const path = join12(dir, name);
|
|
15225
16492
|
try {
|
|
15226
16493
|
entries.push({ path, group, timeMs: statSync2(path).mtimeMs });
|
|
15227
16494
|
} catch {}
|
|
@@ -15236,7 +16503,7 @@ function listStrayTempFiles(dirs) {
|
|
|
15236
16503
|
for (const name of readdirSync3(dir)) {
|
|
15237
16504
|
if (!name.endsWith(".tmp"))
|
|
15238
16505
|
continue;
|
|
15239
|
-
const path =
|
|
16506
|
+
const path = join12(dir, name);
|
|
15240
16507
|
try {
|
|
15241
16508
|
if (statSync2(path).isFile())
|
|
15242
16509
|
stray.push(path);
|
|
@@ -15259,7 +16526,7 @@ program.command("gc").description("prune old run history, rotate database backup
|
|
|
15259
16526
|
} finally {
|
|
15260
16527
|
store.close();
|
|
15261
16528
|
}
|
|
15262
|
-
const backupsDir =
|
|
16529
|
+
const backupsDir = join12(dataDir(), "backups");
|
|
15263
16530
|
const grouped = new Map;
|
|
15264
16531
|
for (const entry of listManagedBackups(backupsDir)) {
|
|
15265
16532
|
const group = grouped.get(entry.group) ?? [];
|