@hasna/loops 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +106 -3
- package/README.md +70 -17
- package/dist/api/index.d.ts +35 -0
- package/dist/api/index.js +750 -10
- package/dist/cli/index.js +1744 -361
- package/dist/cli/ui.d.ts +44 -0
- package/dist/daemon/index.js +948 -212
- package/dist/generated/storage-kit/health.d.ts +19 -0
- package/dist/generated/storage-kit/index.d.ts +7 -0
- package/dist/generated/storage-kit/migrations.d.ts +47 -0
- package/dist/generated/storage-kit/mode.d.ts +47 -0
- package/dist/generated/storage-kit/pool.d.ts +33 -0
- package/dist/generated/storage-kit/query.d.ts +35 -0
- package/dist/generated/storage-kit/tls.d.ts +25 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +6513 -4840
- package/dist/lib/health.d.ts +83 -3
- package/dist/lib/mode.d.ts +27 -0
- package/dist/lib/mode.js +69 -2
- package/dist/lib/scheduler.d.ts +5 -2
- package/dist/lib/storage/index.d.ts +1 -0
- package/dist/lib/storage/index.js +1329 -211
- package/dist/lib/storage/pg-executor.d.ts +27 -0
- package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
- package/dist/lib/storage/postgres-loop-storage.d.ts +114 -0
- package/dist/lib/storage/sqlite.js +336 -8
- package/dist/lib/store.d.ts +234 -0
- package/dist/lib/store.js +350 -8
- package/dist/mcp/index.js +1447 -479
- package/dist/runner/index.js +179 -25
- package/dist/sdk/http.d.ts +115 -0
- package/dist/sdk/http.js +145 -0
- package/dist/sdk/index.d.ts +5 -1
- package/dist/sdk/index.js +1106 -296
- package/dist/serve/index.d.ts +4 -0
- package/dist/serve/index.js +7619 -0
- package/dist/types.d.ts +3 -0
- package/docs/CUTOVER-RUNBOOK.md +61 -0
- package/docs/DEPLOYMENT_MODES.md +23 -1
- package/docs/USAGE.md +66 -16
- package/package.json +14 -2
package/dist/cli/index.js
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
|
|
4
4
|
// src/lib/store.ts
|
|
5
5
|
import { Database } from "bun:sqlite";
|
|
6
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as
|
|
7
|
-
import { tmpdir } from "os";
|
|
8
|
-
import { basename as basename2, dirname as dirname2, join as
|
|
6
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
|
|
7
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
8
|
+
import { basename as basename2, dirname as dirname2, join as join4 } from "path";
|
|
9
9
|
|
|
10
10
|
// src/lib/errors.ts
|
|
11
11
|
class CodedError extends Error {
|
|
@@ -1204,6 +1204,196 @@ function discardWorkflowRunManifest(staged) {
|
|
|
1204
1204
|
} catch {}
|
|
1205
1205
|
}
|
|
1206
1206
|
|
|
1207
|
+
// src/lib/route/todos-cli.ts
|
|
1208
|
+
import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
|
|
1209
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1210
|
+
import { join as join3 } from "path";
|
|
1211
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
1212
|
+
|
|
1213
|
+
// src/lib/format.ts
|
|
1214
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1215
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
1216
|
+
function redact(value, visible = 0) {
|
|
1217
|
+
if (!value)
|
|
1218
|
+
return value;
|
|
1219
|
+
if (value.length <= visible)
|
|
1220
|
+
return value;
|
|
1221
|
+
if (visible <= 0)
|
|
1222
|
+
return `[redacted ${value.length} chars]`;
|
|
1223
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1224
|
+
}
|
|
1225
|
+
function truncateTextOutput(value) {
|
|
1226
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1227
|
+
return value;
|
|
1228
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1229
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1230
|
+
}
|
|
1231
|
+
function redactSensitivePayload(value, key) {
|
|
1232
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
1233
|
+
if (typeof value === "string")
|
|
1234
|
+
return redact(value);
|
|
1235
|
+
if (value === undefined || value === null)
|
|
1236
|
+
return value;
|
|
1237
|
+
return "[redacted]";
|
|
1238
|
+
}
|
|
1239
|
+
if (Array.isArray(value))
|
|
1240
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
1241
|
+
if (value && typeof value === "object") {
|
|
1242
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
1243
|
+
}
|
|
1244
|
+
return value;
|
|
1245
|
+
}
|
|
1246
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1247
|
+
const indent = opts.indent ?? "";
|
|
1248
|
+
const nested = `${indent} `;
|
|
1249
|
+
const blocks = [];
|
|
1250
|
+
for (const [label, output] of [
|
|
1251
|
+
["stdout", value.stdout],
|
|
1252
|
+
["stderr", value.stderr]
|
|
1253
|
+
]) {
|
|
1254
|
+
if (!output)
|
|
1255
|
+
continue;
|
|
1256
|
+
blocks.push(`${indent}${label}:`);
|
|
1257
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1258
|
+
blocks.push(`${nested}${line}`);
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
return blocks;
|
|
1262
|
+
}
|
|
1263
|
+
function publicLoop(loop) {
|
|
1264
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1265
|
+
return {
|
|
1266
|
+
...loop,
|
|
1267
|
+
target
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
function publicRun(run, showOutput = false, opts = {}) {
|
|
1271
|
+
return {
|
|
1272
|
+
...run,
|
|
1273
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1274
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1275
|
+
error: opts.redactError ? redact(run.error) : run.error
|
|
1276
|
+
};
|
|
1277
|
+
}
|
|
1278
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
1279
|
+
return {
|
|
1280
|
+
...result,
|
|
1281
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
1282
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
1283
|
+
error: redact(result.error)
|
|
1284
|
+
};
|
|
1285
|
+
}
|
|
1286
|
+
function publicWorkflow(workflow) {
|
|
1287
|
+
return {
|
|
1288
|
+
...workflow,
|
|
1289
|
+
steps: workflow.steps.map((step) => ({
|
|
1290
|
+
...step,
|
|
1291
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
1292
|
+
}))
|
|
1293
|
+
};
|
|
1294
|
+
}
|
|
1295
|
+
function publicWorkflowRun(run) {
|
|
1296
|
+
return { ...run, error: redact(run.error) };
|
|
1297
|
+
}
|
|
1298
|
+
function publicWorkflowInvocation(invocation) {
|
|
1299
|
+
return redactSensitivePayload(invocation);
|
|
1300
|
+
}
|
|
1301
|
+
function publicWorkflowWorkItem(item) {
|
|
1302
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
1303
|
+
}
|
|
1304
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
1305
|
+
return {
|
|
1306
|
+
...run,
|
|
1307
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1308
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1309
|
+
error: redact(run.error)
|
|
1310
|
+
};
|
|
1311
|
+
}
|
|
1312
|
+
function publicWorkflowEvent(event) {
|
|
1313
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
1314
|
+
}
|
|
1315
|
+
function publicGoal(goal) {
|
|
1316
|
+
return {
|
|
1317
|
+
...goal,
|
|
1318
|
+
objective: redact(goal.objective, 120)
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
1321
|
+
function publicGoalRun(run) {
|
|
1322
|
+
return {
|
|
1323
|
+
...run,
|
|
1324
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
1325
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// src/lib/route/todos-cli.ts
|
|
1330
|
+
function defaultLoopsProject() {
|
|
1331
|
+
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
|
|
1332
|
+
}
|
|
1333
|
+
function runLocalCommand(command, args, opts = {}) {
|
|
1334
|
+
const result = spawnSync2(command, args, {
|
|
1335
|
+
input: opts.input,
|
|
1336
|
+
encoding: "utf8",
|
|
1337
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1338
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1339
|
+
env: process.env
|
|
1340
|
+
});
|
|
1341
|
+
return {
|
|
1342
|
+
ok: result.status === 0,
|
|
1343
|
+
status: result.status,
|
|
1344
|
+
stdout: result.stdout || "",
|
|
1345
|
+
stderr: result.stderr || "",
|
|
1346
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1347
|
+
};
|
|
1348
|
+
}
|
|
1349
|
+
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
1350
|
+
const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
|
|
1351
|
+
const stdoutPath = join3(tempDir, "stdout");
|
|
1352
|
+
const stdoutFd = openSync(stdoutPath, "w");
|
|
1353
|
+
let result;
|
|
1354
|
+
try {
|
|
1355
|
+
result = spawnSync2(command, args, {
|
|
1356
|
+
input: opts.input,
|
|
1357
|
+
encoding: "utf8",
|
|
1358
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1359
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1360
|
+
env: process.env,
|
|
1361
|
+
stdio: ["pipe", stdoutFd, "pipe"]
|
|
1362
|
+
});
|
|
1363
|
+
} finally {
|
|
1364
|
+
closeSync(stdoutFd);
|
|
1365
|
+
}
|
|
1366
|
+
try {
|
|
1367
|
+
return {
|
|
1368
|
+
ok: result.status === 0,
|
|
1369
|
+
status: result.status,
|
|
1370
|
+
stdout: readFileSync3(stdoutPath, "utf8"),
|
|
1371
|
+
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
1372
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1373
|
+
};
|
|
1374
|
+
} finally {
|
|
1375
|
+
rmSync2(tempDir, { recursive: true, force: true });
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
function ensureTodosTaskList(project, slug, name, description) {
|
|
1379
|
+
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
1380
|
+
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
1381
|
+
if (!list.ok)
|
|
1382
|
+
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
1383
|
+
const values = JSON.parse(list.stdout || "[]");
|
|
1384
|
+
const found = values.find((entry) => entry.slug === slug);
|
|
1385
|
+
if (!found)
|
|
1386
|
+
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
1387
|
+
return found.id;
|
|
1388
|
+
}
|
|
1389
|
+
function todosMutationSummary(result) {
|
|
1390
|
+
return {
|
|
1391
|
+
ok: result.ok,
|
|
1392
|
+
status: result.status,
|
|
1393
|
+
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1207
1397
|
// src/lib/store.ts
|
|
1208
1398
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
1209
1399
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
@@ -1213,6 +1403,7 @@ var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "s
|
|
|
1213
1403
|
var PRUNE_BATCH_SIZE = 400;
|
|
1214
1404
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
1215
1405
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
1406
|
+
var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
|
|
1216
1407
|
function rowToLoop(row) {
|
|
1217
1408
|
return {
|
|
1218
1409
|
id: row.id,
|
|
@@ -1263,6 +1454,9 @@ function rowToRun(row) {
|
|
|
1263
1454
|
updatedAt: row.updated_at
|
|
1264
1455
|
};
|
|
1265
1456
|
}
|
|
1457
|
+
function latestRunTime(row) {
|
|
1458
|
+
return row.finished_at ?? row.started_at ?? row.created_at;
|
|
1459
|
+
}
|
|
1266
1460
|
function rowToWorkflow(row) {
|
|
1267
1461
|
return {
|
|
1268
1462
|
id: row.id,
|
|
@@ -1517,7 +1711,7 @@ class Store {
|
|
|
1517
1711
|
const file = path ?? dbPath();
|
|
1518
1712
|
if (file !== ":memory:")
|
|
1519
1713
|
ensurePrivateStorePath(file);
|
|
1520
|
-
this.rootDir = file === ":memory:" ?
|
|
1714
|
+
this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
|
|
1521
1715
|
if (file === ":memory:")
|
|
1522
1716
|
this.memoryRootDir = this.rootDir;
|
|
1523
1717
|
this.db = new Database(file);
|
|
@@ -2015,7 +2209,32 @@ class Store {
|
|
|
2015
2209
|
} else {
|
|
2016
2210
|
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2017
2211
|
}
|
|
2018
|
-
return rows.map(rowToLoop);
|
|
2212
|
+
return this.withLatestRunSummaries(rows.map(rowToLoop));
|
|
2213
|
+
}
|
|
2214
|
+
withLatestRunSummaries(loops) {
|
|
2215
|
+
if (loops.length === 0)
|
|
2216
|
+
return loops;
|
|
2217
|
+
const placeholders = loops.map(() => "?").join(",");
|
|
2218
|
+
const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
|
|
2219
|
+
FROM (
|
|
2220
|
+
SELECT loop_id, id, status, started_at, finished_at, created_at,
|
|
2221
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
|
|
2222
|
+
FROM loop_runs
|
|
2223
|
+
WHERE loop_id IN (${placeholders})
|
|
2224
|
+
)
|
|
2225
|
+
WHERE rn = 1`).all(...loops.map((loop) => loop.id));
|
|
2226
|
+
const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
|
|
2227
|
+
return loops.map((loop) => {
|
|
2228
|
+
const latest = latestByLoopId.get(loop.id);
|
|
2229
|
+
if (!latest)
|
|
2230
|
+
return loop;
|
|
2231
|
+
return {
|
|
2232
|
+
...loop,
|
|
2233
|
+
latestRunId: latest.id,
|
|
2234
|
+
latestRunStatus: latest.status,
|
|
2235
|
+
lastRunAt: latestRunTime(latest)
|
|
2236
|
+
};
|
|
2237
|
+
});
|
|
2019
2238
|
}
|
|
2020
2239
|
dueLoops(now, limit = 500) {
|
|
2021
2240
|
const rows = this.db.query(`SELECT * FROM loops
|
|
@@ -2134,6 +2353,45 @@ class Store {
|
|
|
2134
2353
|
throw error;
|
|
2135
2354
|
}
|
|
2136
2355
|
}
|
|
2356
|
+
updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
|
|
2357
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
2358
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2359
|
+
try {
|
|
2360
|
+
const current = this.requireUniqueLoop(idOrName);
|
|
2361
|
+
if (current.archivedAt)
|
|
2362
|
+
throw new LoopArchivedError(current.name || current.id);
|
|
2363
|
+
if (current.target.type !== "agent")
|
|
2364
|
+
throw new Error(`loop is not an agent loop: ${idOrName}`);
|
|
2365
|
+
if (this.hasRunningRun(current.id))
|
|
2366
|
+
throw new Error(`refusing to update running loop: ${current.id}`);
|
|
2367
|
+
const target = { ...current.target, timeoutMs };
|
|
2368
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
2369
|
+
delete target.idleTimeoutMs;
|
|
2370
|
+
const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
|
|
2371
|
+
WHERE id=$id
|
|
2372
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2373
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2374
|
+
))`).run({
|
|
2375
|
+
$id: current.id,
|
|
2376
|
+
$target: JSON.stringify(target),
|
|
2377
|
+
$updated: updated,
|
|
2378
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2379
|
+
$now: updated
|
|
2380
|
+
});
|
|
2381
|
+
if (res.changes !== 1)
|
|
2382
|
+
throw new Error("daemon lease lost");
|
|
2383
|
+
this.db.exec("COMMIT");
|
|
2384
|
+
const after = this.getLoop(current.id);
|
|
2385
|
+
if (!after)
|
|
2386
|
+
throw new Error(`loop not found after timeout update: ${current.id}`);
|
|
2387
|
+
return after;
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
try {
|
|
2390
|
+
this.db.exec("ROLLBACK");
|
|
2391
|
+
} catch {}
|
|
2392
|
+
throw error;
|
|
2393
|
+
}
|
|
2394
|
+
}
|
|
2137
2395
|
createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
|
|
2138
2396
|
const normalized = normalizeCreateWorkflowInput(workflowInput);
|
|
2139
2397
|
const updated = (opts.now ?? new Date).toISOString();
|
|
@@ -2464,6 +2722,59 @@ class Store {
|
|
|
2464
2722
|
updated
|
|
2465
2723
|
});
|
|
2466
2724
|
}
|
|
2725
|
+
taskLifecycleTodosPointerContext(workflowRunId) {
|
|
2726
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
2727
|
+
if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
|
|
2728
|
+
return;
|
|
2729
|
+
const workItem = this.getWorkflowWorkItem(run.workItemId);
|
|
2730
|
+
if (!workItem || workItem.routeKey !== "todos-task")
|
|
2731
|
+
return;
|
|
2732
|
+
const invocation = this.getWorkflowInvocation(run.invocationId);
|
|
2733
|
+
if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
|
|
2734
|
+
return;
|
|
2735
|
+
const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
|
|
2736
|
+
const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
|
|
2737
|
+
if (!projectPath || !taskId)
|
|
2738
|
+
return;
|
|
2739
|
+
return {
|
|
2740
|
+
projectPath,
|
|
2741
|
+
taskId,
|
|
2742
|
+
invocationId: invocation.id,
|
|
2743
|
+
workflowRunId: run.id,
|
|
2744
|
+
manifestPath: run.manifestPath
|
|
2745
|
+
};
|
|
2746
|
+
}
|
|
2747
|
+
syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
|
|
2748
|
+
const context = this.taskLifecycleTodosPointerContext(workflowRunId);
|
|
2749
|
+
if (!context)
|
|
2750
|
+
return;
|
|
2751
|
+
const result = runLocalCommand("todos", [
|
|
2752
|
+
"--project",
|
|
2753
|
+
context.projectPath,
|
|
2754
|
+
"task",
|
|
2755
|
+
"workflow-pointers",
|
|
2756
|
+
context.taskId,
|
|
2757
|
+
"--clear",
|
|
2758
|
+
"--invocation",
|
|
2759
|
+
context.invocationId,
|
|
2760
|
+
"--run",
|
|
2761
|
+
context.workflowRunId,
|
|
2762
|
+
"--manifest",
|
|
2763
|
+
context.manifestPath,
|
|
2764
|
+
"--state",
|
|
2765
|
+
"succeeded",
|
|
2766
|
+
"--actor",
|
|
2767
|
+
"openloops:task-lifecycle"
|
|
2768
|
+
]);
|
|
2769
|
+
this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
|
|
2770
|
+
projectPath: context.projectPath,
|
|
2771
|
+
taskId: context.taskId,
|
|
2772
|
+
invocationId: context.invocationId,
|
|
2773
|
+
workflowRunId: context.workflowRunId,
|
|
2774
|
+
manifestPath: context.manifestPath,
|
|
2775
|
+
mutation: todosMutationSummary(result)
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2467
2778
|
createWorkflowInvocation(input) {
|
|
2468
2779
|
const now = nowIso();
|
|
2469
2780
|
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
@@ -3426,6 +3737,8 @@ class Store {
|
|
|
3426
3737
|
const run = this.getWorkflowRun(workflowRunId);
|
|
3427
3738
|
if (!run)
|
|
3428
3739
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
3740
|
+
if (changed && status === "succeeded")
|
|
3741
|
+
this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
|
|
3429
3742
|
return run;
|
|
3430
3743
|
}
|
|
3431
3744
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -3487,6 +3800,17 @@ class Store {
|
|
|
3487
3800
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
3488
3801
|
return (row?.count ?? 0) > 0;
|
|
3489
3802
|
}
|
|
3803
|
+
hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
|
|
3804
|
+
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
3805
|
+
WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
|
|
3806
|
+
return rows.some((row) => {
|
|
3807
|
+
if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
|
|
3808
|
+
return true;
|
|
3809
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at))
|
|
3810
|
+
return true;
|
|
3811
|
+
return this.hasLiveWorkflowStepProcesses(row.id);
|
|
3812
|
+
});
|
|
3813
|
+
}
|
|
3490
3814
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
3491
3815
|
const now = (opts.now ?? new Date).toISOString();
|
|
3492
3816
|
const startedMs = processStartTimeMs(pid);
|
|
@@ -3619,6 +3943,10 @@ class Store {
|
|
|
3619
3943
|
loop = currentLoop;
|
|
3620
3944
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
3621
3945
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
3946
|
+
if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
|
|
3947
|
+
this.db.exec("COMMIT");
|
|
3948
|
+
return;
|
|
3949
|
+
}
|
|
3622
3950
|
if (existing) {
|
|
3623
3951
|
if (existing.status === "running") {
|
|
3624
3952
|
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
@@ -4190,9 +4518,9 @@ class Store {
|
|
|
4190
4518
|
const runDir = dirname2(manifestPath);
|
|
4191
4519
|
try {
|
|
4192
4520
|
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
4193
|
-
|
|
4521
|
+
rmSync3(runDir, { recursive: true, force: true });
|
|
4194
4522
|
} else {
|
|
4195
|
-
|
|
4523
|
+
rmSync3(manifestPath, { force: true });
|
|
4196
4524
|
}
|
|
4197
4525
|
} catch {}
|
|
4198
4526
|
}
|
|
@@ -4259,7 +4587,7 @@ class Store {
|
|
|
4259
4587
|
this.db.close();
|
|
4260
4588
|
if (this.memoryRootDir) {
|
|
4261
4589
|
try {
|
|
4262
|
-
|
|
4590
|
+
rmSync3(this.memoryRootDir, { recursive: true, force: true });
|
|
4263
4591
|
} catch {}
|
|
4264
4592
|
this.memoryRootDir = undefined;
|
|
4265
4593
|
}
|
|
@@ -4268,7 +4596,7 @@ class Store {
|
|
|
4268
4596
|
// package.json
|
|
4269
4597
|
var package_default = {
|
|
4270
4598
|
name: "@hasna/loops",
|
|
4271
|
-
version: "0.4.
|
|
4599
|
+
version: "0.4.14",
|
|
4272
4600
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4273
4601
|
type: "module",
|
|
4274
4602
|
main: "dist/index.js",
|
|
@@ -4277,6 +4605,7 @@ var package_default = {
|
|
|
4277
4605
|
loops: "dist/cli/index.js",
|
|
4278
4606
|
"loops-daemon": "dist/daemon/index.js",
|
|
4279
4607
|
"loops-api": "dist/api/index.js",
|
|
4608
|
+
"loops-serve": "dist/serve/index.js",
|
|
4280
4609
|
"loops-runner": "dist/runner/index.js",
|
|
4281
4610
|
"loops-mcp": "dist/mcp/index.js"
|
|
4282
4611
|
},
|
|
@@ -4289,6 +4618,14 @@ var package_default = {
|
|
|
4289
4618
|
types: "./dist/sdk/index.d.ts",
|
|
4290
4619
|
import: "./dist/sdk/index.js"
|
|
4291
4620
|
},
|
|
4621
|
+
"./sdk/http": {
|
|
4622
|
+
types: "./dist/sdk/http.d.ts",
|
|
4623
|
+
import: "./dist/sdk/http.js"
|
|
4624
|
+
},
|
|
4625
|
+
"./serve": {
|
|
4626
|
+
types: "./dist/serve/index.d.ts",
|
|
4627
|
+
import: "./dist/serve/index.js"
|
|
4628
|
+
},
|
|
4292
4629
|
"./mcp": {
|
|
4293
4630
|
types: "./dist/mcp/index.d.ts",
|
|
4294
4631
|
import: "./dist/mcp/index.js"
|
|
@@ -4334,7 +4671,7 @@ var package_default = {
|
|
|
4334
4671
|
"LICENSE"
|
|
4335
4672
|
],
|
|
4336
4673
|
scripts: {
|
|
4337
|
-
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4674
|
+
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4338
4675
|
"build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
|
|
4339
4676
|
typecheck: "tsc --noEmit",
|
|
4340
4677
|
test: "bun test",
|
|
@@ -4369,11 +4706,13 @@ var package_default = {
|
|
|
4369
4706
|
bun: ">=1.0.0"
|
|
4370
4707
|
},
|
|
4371
4708
|
dependencies: {
|
|
4709
|
+
"@hasna/contracts": "^0.4.1",
|
|
4372
4710
|
"@hasna/events": "^0.1.9",
|
|
4373
4711
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
4374
4712
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
4375
4713
|
ai: "6.0.204",
|
|
4376
4714
|
commander: "^13.1.0",
|
|
4715
|
+
pg: "^8.13.1",
|
|
4377
4716
|
zod: "4.4.3"
|
|
4378
4717
|
},
|
|
4379
4718
|
optionalDependencies: {
|
|
@@ -4381,6 +4720,7 @@ var package_default = {
|
|
|
4381
4720
|
},
|
|
4382
4721
|
devDependencies: {
|
|
4383
4722
|
"@types/bun": "latest",
|
|
4723
|
+
"@types/pg": "^8.11.10",
|
|
4384
4724
|
typescript: "^5.7.3"
|
|
4385
4725
|
},
|
|
4386
4726
|
publishConfig: {
|
|
@@ -4459,6 +4799,50 @@ function sourceOfTruthForMode(mode) {
|
|
|
4459
4799
|
return "self_hosted_control_plane";
|
|
4460
4800
|
return "cloud_control_plane";
|
|
4461
4801
|
}
|
|
4802
|
+
var ROUTE_ADMISSION_GATES = [
|
|
4803
|
+
"max_dispatch",
|
|
4804
|
+
"max_active",
|
|
4805
|
+
"max_active_per_project",
|
|
4806
|
+
"max_active_per_project_group",
|
|
4807
|
+
"max_active_scope",
|
|
4808
|
+
"max_per_profile"
|
|
4809
|
+
];
|
|
4810
|
+
function remoteSchedulerBackendForMode(mode, config) {
|
|
4811
|
+
if (mode === "local")
|
|
4812
|
+
return "none";
|
|
4813
|
+
if (mode === "cloud")
|
|
4814
|
+
return config.cloudApiUrl ? "hosted_control_plane_contract" : "unconfigured";
|
|
4815
|
+
if (config.databaseUrlPresent)
|
|
4816
|
+
return "postgres_contract";
|
|
4817
|
+
if (config.apiUrl)
|
|
4818
|
+
return "api_control_plane_contract";
|
|
4819
|
+
return "unconfigured";
|
|
4820
|
+
}
|
|
4821
|
+
function schedulerStateForMode(args) {
|
|
4822
|
+
const nonLocal = args.deploymentMode !== "local";
|
|
4823
|
+
return {
|
|
4824
|
+
authority: args.sourceOfTruth,
|
|
4825
|
+
localStore: {
|
|
4826
|
+
backend: "sqlite",
|
|
4827
|
+
role: args.localRole,
|
|
4828
|
+
runArtifacts: "local_files",
|
|
4829
|
+
routeAdmissionState: "workflow_work_items"
|
|
4830
|
+
},
|
|
4831
|
+
remoteStore: {
|
|
4832
|
+
backend: remoteSchedulerBackendForMode(args.deploymentMode, args.config),
|
|
4833
|
+
configured: nonLocal && args.controlPlaneConfigured,
|
|
4834
|
+
applySupported: false,
|
|
4835
|
+
objectArtifacts: nonLocal ? "object_store_contract" : "none",
|
|
4836
|
+
mutatesAws: false
|
|
4837
|
+
},
|
|
4838
|
+
routeAdmission: {
|
|
4839
|
+
stateStore: nonLocal ? "control_plane_contract" : "local_sqlite",
|
|
4840
|
+
activeStatuses: ["admitted", "running"],
|
|
4841
|
+
gates: ROUTE_ADMISSION_GATES,
|
|
4842
|
+
dryRunEvaluatesLiveCounts: false
|
|
4843
|
+
}
|
|
4844
|
+
};
|
|
4845
|
+
}
|
|
4462
4846
|
function displayControlPlaneUrl(value) {
|
|
4463
4847
|
if (!value)
|
|
4464
4848
|
return;
|
|
@@ -4484,6 +4868,9 @@ function buildDeploymentStatus(opts = {}) {
|
|
|
4484
4868
|
if (deploymentMode === "self_hosted" && !controlPlaneConfigured) {
|
|
4485
4869
|
warnings.push("self_hosted mode needs LOOPS_API_URL or LOOPS_DATABASE_URL before it can become authoritative");
|
|
4486
4870
|
}
|
|
4871
|
+
if (deploymentMode === "self_hosted" && config.databaseUrlPresent && !config.apiUrl) {
|
|
4872
|
+
warnings.push("LOOPS_DATABASE_URL selects the self_hosted storage contract; loops-runner still needs LOOPS_API_URL to claim remote work");
|
|
4873
|
+
}
|
|
4487
4874
|
if (deploymentMode === "cloud" && config.apiUrl && !config.cloudApiUrl) {
|
|
4488
4875
|
warnings.push("LOOPS_API_URL selects self_hosted; cloud mode uses LOOPS_CLOUD_API_URL");
|
|
4489
4876
|
}
|
|
@@ -4517,6 +4904,13 @@ function buildDeploymentStatus(opts = {}) {
|
|
|
4517
4904
|
required: deploymentMode !== "local",
|
|
4518
4905
|
role: deploymentMode === "local" ? "daemon" : "control_plane_worker"
|
|
4519
4906
|
},
|
|
4907
|
+
schedulerState: schedulerStateForMode({
|
|
4908
|
+
deploymentMode,
|
|
4909
|
+
sourceOfTruth: sourceOfTruthForMode(deploymentMode),
|
|
4910
|
+
localRole: deploymentMode === "local" ? "authoritative" : "cache_and_spool",
|
|
4911
|
+
controlPlaneConfigured,
|
|
4912
|
+
config
|
|
4913
|
+
}),
|
|
4520
4914
|
warnings
|
|
4521
4915
|
};
|
|
4522
4916
|
}
|
|
@@ -4529,142 +4923,27 @@ function deploymentStatusLine(status) {
|
|
|
4529
4923
|
`source=${status.deploymentModeSource}`,
|
|
4530
4924
|
`truth=${status.sourceOfTruth}`,
|
|
4531
4925
|
`local=${status.localStore.role}`,
|
|
4926
|
+
`scheduler=${status.schedulerState.routeAdmission.stateStore}`,
|
|
4532
4927
|
`control_plane=${configured}`
|
|
4533
4928
|
].join(" ");
|
|
4534
4929
|
}
|
|
4535
4930
|
|
|
4536
4931
|
// src/cli/index.ts
|
|
4537
4932
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
4538
|
-
import { existsSync as existsSync10, readdirSync as readdirSync3, readFileSync as readFileSync10, rmSync as rmSync7, statSync as statSync2, writeFileSync as
|
|
4539
|
-
import { join as
|
|
4933
|
+
import { existsSync as existsSync10, readdirSync as readdirSync3, readFileSync as readFileSync10, rmSync as rmSync7, statSync as statSync2, writeFileSync as writeFileSync7 } from "fs";
|
|
4934
|
+
import { join as join12 } from "path";
|
|
4540
4935
|
import { Database as Database3 } from "bun:sqlite";
|
|
4541
4936
|
import { Command } from "commander";
|
|
4542
4937
|
|
|
4543
|
-
// src/lib/format.ts
|
|
4544
|
-
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
4545
|
-
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
4546
|
-
function redact(value, visible = 0) {
|
|
4547
|
-
if (!value)
|
|
4548
|
-
return value;
|
|
4549
|
-
if (value.length <= visible)
|
|
4550
|
-
return value;
|
|
4551
|
-
if (visible <= 0)
|
|
4552
|
-
return `[redacted ${value.length} chars]`;
|
|
4553
|
-
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
4554
|
-
}
|
|
4555
|
-
function truncateTextOutput(value) {
|
|
4556
|
-
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
4557
|
-
return value;
|
|
4558
|
-
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
4559
|
-
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
4560
|
-
}
|
|
4561
|
-
function redactSensitivePayload(value, key) {
|
|
4562
|
-
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
4563
|
-
if (typeof value === "string")
|
|
4564
|
-
return redact(value);
|
|
4565
|
-
if (value === undefined || value === null)
|
|
4566
|
-
return value;
|
|
4567
|
-
return "[redacted]";
|
|
4568
|
-
}
|
|
4569
|
-
if (Array.isArray(value))
|
|
4570
|
-
return value.map((item) => redactSensitivePayload(item));
|
|
4571
|
-
if (value && typeof value === "object") {
|
|
4572
|
-
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
4573
|
-
}
|
|
4574
|
-
return value;
|
|
4575
|
-
}
|
|
4576
|
-
function textOutputBlocks(value, opts = {}) {
|
|
4577
|
-
const indent = opts.indent ?? "";
|
|
4578
|
-
const nested = `${indent} `;
|
|
4579
|
-
const blocks = [];
|
|
4580
|
-
for (const [label, output] of [
|
|
4581
|
-
["stdout", value.stdout],
|
|
4582
|
-
["stderr", value.stderr]
|
|
4583
|
-
]) {
|
|
4584
|
-
if (!output)
|
|
4585
|
-
continue;
|
|
4586
|
-
blocks.push(`${indent}${label}:`);
|
|
4587
|
-
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
4588
|
-
blocks.push(`${nested}${line}`);
|
|
4589
|
-
}
|
|
4590
|
-
}
|
|
4591
|
-
return blocks;
|
|
4592
|
-
}
|
|
4593
|
-
function publicLoop(loop) {
|
|
4594
|
-
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
4595
|
-
return {
|
|
4596
|
-
...loop,
|
|
4597
|
-
target
|
|
4598
|
-
};
|
|
4599
|
-
}
|
|
4600
|
-
function publicRun(run, showOutput = false, opts = {}) {
|
|
4601
|
-
return {
|
|
4602
|
-
...run,
|
|
4603
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
4604
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
4605
|
-
error: opts.redactError ? redact(run.error) : run.error
|
|
4606
|
-
};
|
|
4607
|
-
}
|
|
4608
|
-
function publicExecutorResult(result, showOutput = false) {
|
|
4609
|
-
return {
|
|
4610
|
-
...result,
|
|
4611
|
-
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
4612
|
-
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
4613
|
-
error: redact(result.error)
|
|
4614
|
-
};
|
|
4615
|
-
}
|
|
4616
|
-
function publicWorkflow(workflow) {
|
|
4617
|
-
return {
|
|
4618
|
-
...workflow,
|
|
4619
|
-
steps: workflow.steps.map((step) => ({
|
|
4620
|
-
...step,
|
|
4621
|
-
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
4622
|
-
}))
|
|
4623
|
-
};
|
|
4624
|
-
}
|
|
4625
|
-
function publicWorkflowRun(run) {
|
|
4626
|
-
return { ...run, error: redact(run.error) };
|
|
4627
|
-
}
|
|
4628
|
-
function publicWorkflowInvocation(invocation) {
|
|
4629
|
-
return redactSensitivePayload(invocation);
|
|
4630
|
-
}
|
|
4631
|
-
function publicWorkflowWorkItem(item) {
|
|
4632
|
-
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
4633
|
-
}
|
|
4634
|
-
function publicWorkflowStepRun(run, showOutput = false) {
|
|
4635
|
-
return {
|
|
4636
|
-
...run,
|
|
4637
|
-
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
4638
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
4639
|
-
error: redact(run.error)
|
|
4640
|
-
};
|
|
4641
|
-
}
|
|
4642
|
-
function publicWorkflowEvent(event) {
|
|
4643
|
-
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
4644
|
-
}
|
|
4645
|
-
function publicGoal(goal) {
|
|
4646
|
-
return {
|
|
4647
|
-
...goal,
|
|
4648
|
-
objective: redact(goal.objective, 120)
|
|
4649
|
-
};
|
|
4650
|
-
}
|
|
4651
|
-
function publicGoalRun(run) {
|
|
4652
|
-
return {
|
|
4653
|
-
...run,
|
|
4654
|
-
evidence: redactSensitivePayload(run.evidence),
|
|
4655
|
-
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
4656
|
-
};
|
|
4657
|
-
}
|
|
4658
|
-
|
|
4659
4938
|
// src/lib/executor.ts
|
|
4660
|
-
import { spawn as spawn2, spawnSync as
|
|
4939
|
+
import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
|
|
4661
4940
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
4662
4941
|
import { once } from "events";
|
|
4663
4942
|
import { lstatSync, mkdirSync as mkdirSync4, realpathSync } from "fs";
|
|
4664
4943
|
import { dirname as dirname3, resolve as resolve2 } from "path";
|
|
4665
4944
|
|
|
4666
4945
|
// src/lib/accounts.ts
|
|
4667
|
-
import { spawnSync as
|
|
4946
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
4668
4947
|
import { existsSync as existsSync2 } from "fs";
|
|
4669
4948
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4670
4949
|
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
@@ -4769,7 +5048,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4769
5048
|
if (!account)
|
|
4770
5049
|
return {};
|
|
4771
5050
|
const tool = requiredAccountTool(account, toolHint);
|
|
4772
|
-
const result =
|
|
5051
|
+
const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
|
|
4773
5052
|
encoding: "utf8",
|
|
4774
5053
|
env,
|
|
4775
5054
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -4785,8 +5064,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4785
5064
|
|
|
4786
5065
|
// src/lib/env.ts
|
|
4787
5066
|
import { accessSync, constants } from "fs";
|
|
4788
|
-
import { homedir as
|
|
4789
|
-
import { delimiter, join as
|
|
5067
|
+
import { homedir as homedir3 } from "os";
|
|
5068
|
+
import { delimiter, join as join5 } from "path";
|
|
4790
5069
|
function compactPathParts(parts) {
|
|
4791
5070
|
const seen = new Set;
|
|
4792
5071
|
const result = [];
|
|
@@ -4800,16 +5079,16 @@ function compactPathParts(parts) {
|
|
|
4800
5079
|
return result;
|
|
4801
5080
|
}
|
|
4802
5081
|
function commonExecutableDirs(env = process.env) {
|
|
4803
|
-
const home = env.HOME ||
|
|
5082
|
+
const home = env.HOME || homedir3();
|
|
4804
5083
|
return compactPathParts([
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
env.BUN_INSTALL ?
|
|
5084
|
+
join5(home, ".local", "bin"),
|
|
5085
|
+
join5(home, ".bun", "bin"),
|
|
5086
|
+
join5(home, ".cargo", "bin"),
|
|
5087
|
+
join5(home, ".npm-global", "bin"),
|
|
5088
|
+
join5(home, "bin"),
|
|
5089
|
+
env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
|
|
4811
5090
|
env.PNPM_HOME,
|
|
4812
|
-
env.NPM_CONFIG_PREFIX ?
|
|
5091
|
+
env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
4813
5092
|
"/opt/homebrew/bin",
|
|
4814
5093
|
"/usr/local/bin",
|
|
4815
5094
|
"/usr/bin",
|
|
@@ -4833,7 +5112,7 @@ function executableExists(command, env = process.env) {
|
|
|
4833
5112
|
if (command.includes("/"))
|
|
4834
5113
|
return isExecutable(command);
|
|
4835
5114
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
4836
|
-
if (dir && isExecutable(
|
|
5115
|
+
if (dir && isExecutable(join5(dir, command)))
|
|
4837
5116
|
return true;
|
|
4838
5117
|
}
|
|
4839
5118
|
return false;
|
|
@@ -4974,6 +5253,32 @@ function timeoutResult(startedAt, error, fields = {}) {
|
|
|
4974
5253
|
function successResult(startedAt, fields = {}) {
|
|
4975
5254
|
return buildResult("succeeded", startedAt, fields);
|
|
4976
5255
|
}
|
|
5256
|
+
function codewithJsonlHasTerminalSuccess(stdout) {
|
|
5257
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
5258
|
+
const trimmed = line.trim();
|
|
5259
|
+
if (!trimmed || !trimmed.startsWith("{"))
|
|
5260
|
+
continue;
|
|
5261
|
+
let event;
|
|
5262
|
+
try {
|
|
5263
|
+
event = JSON.parse(trimmed);
|
|
5264
|
+
} catch {
|
|
5265
|
+
continue;
|
|
5266
|
+
}
|
|
5267
|
+
if (!event || typeof event !== "object")
|
|
5268
|
+
continue;
|
|
5269
|
+
const record = event;
|
|
5270
|
+
if (record.type === "task_complete")
|
|
5271
|
+
return true;
|
|
5272
|
+
const payload = record.payload;
|
|
5273
|
+
if (payload && typeof payload === "object" && payload.type === "task_complete") {
|
|
5274
|
+
return true;
|
|
5275
|
+
}
|
|
5276
|
+
}
|
|
5277
|
+
return false;
|
|
5278
|
+
}
|
|
5279
|
+
function codewithJsonlReconciledSuccess(spec, fields) {
|
|
5280
|
+
return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
|
|
5281
|
+
}
|
|
4977
5282
|
function notifySpawn(pid, opts) {
|
|
4978
5283
|
if (!pid)
|
|
4979
5284
|
return;
|
|
@@ -4989,10 +5294,48 @@ function shellQuote(value) {
|
|
|
4989
5294
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
4990
5295
|
}
|
|
4991
5296
|
function codewithProfileCandidateFromLine(line) {
|
|
4992
|
-
const
|
|
4993
|
-
if (
|
|
4994
|
-
return
|
|
4995
|
-
|
|
5297
|
+
const trimmed = line.trim();
|
|
5298
|
+
if (!trimmed || trimmed === "No auth profiles saved.")
|
|
5299
|
+
return;
|
|
5300
|
+
const cols = trimmed.split(/\s+/);
|
|
5301
|
+
const candidate = cols[0] === "*" ? cols[1] : cols[0];
|
|
5302
|
+
if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
|
|
5303
|
+
return;
|
|
5304
|
+
if (candidate === '"name":') {
|
|
5305
|
+
const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
|
|
5306
|
+
if (jsonName)
|
|
5307
|
+
return jsonName;
|
|
5308
|
+
}
|
|
5309
|
+
return candidate;
|
|
5310
|
+
}
|
|
5311
|
+
function codewithProfileCandidatesFromJson(value) {
|
|
5312
|
+
const candidates = [];
|
|
5313
|
+
const root = value && typeof value === "object" ? value : undefined;
|
|
5314
|
+
const addProfile = (entry) => {
|
|
5315
|
+
if (!entry || typeof entry !== "object")
|
|
5316
|
+
return;
|
|
5317
|
+
const name = entry.name;
|
|
5318
|
+
if (typeof name === "string" && name)
|
|
5319
|
+
candidates.push(name);
|
|
5320
|
+
};
|
|
5321
|
+
const data = root?.data;
|
|
5322
|
+
if (Array.isArray(data))
|
|
5323
|
+
data.forEach(addProfile);
|
|
5324
|
+
const profiles = root?.profiles;
|
|
5325
|
+
if (Array.isArray(profiles))
|
|
5326
|
+
profiles.forEach(addProfile);
|
|
5327
|
+
return candidates;
|
|
5328
|
+
}
|
|
5329
|
+
function codewithProfileCandidatesFromOutput(output) {
|
|
5330
|
+
const trimmed = output.trim();
|
|
5331
|
+
const jsonStart = trimmed.indexOf("{");
|
|
5332
|
+
const jsonEnd = trimmed.lastIndexOf("}");
|
|
5333
|
+
if (jsonStart >= 0 && jsonEnd >= jsonStart) {
|
|
5334
|
+
try {
|
|
5335
|
+
return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
|
|
5336
|
+
} catch {}
|
|
5337
|
+
}
|
|
5338
|
+
return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
|
|
4996
5339
|
}
|
|
4997
5340
|
function codewithProfileForError(profile) {
|
|
4998
5341
|
return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
|
|
@@ -5085,6 +5428,7 @@ function commandSpec(target, opts) {
|
|
|
5085
5428
|
preflightAnyOf: invocation.preflightAnyOf,
|
|
5086
5429
|
stdin: invocation.stdin,
|
|
5087
5430
|
allowlist: agentTarget.allowlist,
|
|
5431
|
+
agentProvider: agentTarget.provider,
|
|
5088
5432
|
worktree: agentTarget.worktree
|
|
5089
5433
|
};
|
|
5090
5434
|
}
|
|
@@ -5244,7 +5588,7 @@ function remotePreflightScript(spec, metadata) {
|
|
|
5244
5588
|
}
|
|
5245
5589
|
if (spec.nativeAuthProfile?.provider === "codewith") {
|
|
5246
5590
|
const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
|
|
5247
|
-
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE",
|
|
5591
|
+
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
|
|
5248
5592
|
}
|
|
5249
5593
|
return lines.join(`
|
|
5250
5594
|
`);
|
|
@@ -5262,6 +5606,12 @@ function transportEnv(opts) {
|
|
|
5262
5606
|
return env;
|
|
5263
5607
|
}
|
|
5264
5608
|
function assertCodewithProfileListed(profile, result) {
|
|
5609
|
+
const profiles = codewithProfileSetFromResult(result);
|
|
5610
|
+
if (!profiles.has(profile)) {
|
|
5611
|
+
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5612
|
+
}
|
|
5613
|
+
}
|
|
5614
|
+
function codewithProfileSetFromResult(result) {
|
|
5265
5615
|
if (result.error) {
|
|
5266
5616
|
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
5267
5617
|
}
|
|
@@ -5269,32 +5619,42 @@ function assertCodewithProfileListed(profile, result) {
|
|
|
5269
5619
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5270
5620
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
5271
5621
|
}
|
|
5272
|
-
|
|
5273
|
-
if (!profiles.has(profile)) {
|
|
5274
|
-
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5275
|
-
}
|
|
5622
|
+
return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
|
|
5276
5623
|
}
|
|
5277
5624
|
function preflightNativeAuthProfileSync(spec, env) {
|
|
5278
5625
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5279
5626
|
return;
|
|
5280
|
-
const
|
|
5281
|
-
|
|
5282
|
-
|
|
5283
|
-
|
|
5284
|
-
|
|
5285
|
-
|
|
5286
|
-
|
|
5287
|
-
|
|
5288
|
-
|
|
5289
|
-
|
|
5290
|
-
|
|
5291
|
-
|
|
5627
|
+
const runProfileList = (args) => {
|
|
5628
|
+
const result = spawnSync4(spec.command, args, {
|
|
5629
|
+
encoding: "utf8",
|
|
5630
|
+
env,
|
|
5631
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
5632
|
+
timeout: 15000
|
|
5633
|
+
});
|
|
5634
|
+
return {
|
|
5635
|
+
status: result.status,
|
|
5636
|
+
stdout: result.stdout ?? "",
|
|
5637
|
+
stderr: result.stderr ?? "",
|
|
5638
|
+
error: result.error?.message
|
|
5639
|
+
};
|
|
5640
|
+
};
|
|
5641
|
+
const jsonResult = runProfileList(["profile", "list", "--json"]);
|
|
5642
|
+
try {
|
|
5643
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
5644
|
+
return;
|
|
5645
|
+
} catch {}
|
|
5646
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
|
|
5292
5647
|
}
|
|
5293
5648
|
async function preflightNativeAuthProfile(spec, env) {
|
|
5294
5649
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5295
5650
|
return;
|
|
5296
|
-
const
|
|
5297
|
-
|
|
5651
|
+
const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
|
|
5652
|
+
try {
|
|
5653
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
5654
|
+
return;
|
|
5655
|
+
} catch {}
|
|
5656
|
+
const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
5657
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
|
|
5298
5658
|
}
|
|
5299
5659
|
function resolvedDirEquals(left, right) {
|
|
5300
5660
|
try {
|
|
@@ -5397,7 +5757,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
|
5397
5757
|
}
|
|
5398
5758
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
5399
5759
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
5400
|
-
const result =
|
|
5760
|
+
const result = spawnSync4(plan.command, plan.args, {
|
|
5401
5761
|
encoding: "utf8",
|
|
5402
5762
|
env: transportEnv(opts),
|
|
5403
5763
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -5496,6 +5856,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
|
5496
5856
|
if (timedOut || idleTimedOut) {
|
|
5497
5857
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5498
5858
|
}
|
|
5859
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
5860
|
+
return successResult(startedAt, fields);
|
|
5861
|
+
}
|
|
5499
5862
|
if (error || exitCode !== 0) {
|
|
5500
5863
|
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5501
5864
|
}
|
|
@@ -5631,6 +5994,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
5631
5994
|
if (timedOut || idleTimedOut) {
|
|
5632
5995
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5633
5996
|
}
|
|
5997
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
5998
|
+
return successResult(startedAt, fields);
|
|
5999
|
+
}
|
|
5634
6000
|
if (error || exitCode !== 0) {
|
|
5635
6001
|
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5636
6002
|
}
|
|
@@ -6576,12 +6942,18 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
6576
6942
|
|
|
6577
6943
|
// src/lib/health.ts
|
|
6578
6944
|
import { createHash as createHash2 } from "crypto";
|
|
6579
|
-
import { existsSync as existsSync3, readFileSync as
|
|
6945
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
6946
|
+
import { join as join6 } from "path";
|
|
6580
6947
|
var EVIDENCE_CHARS = 2000;
|
|
6581
6948
|
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6949
|
+
var DEFAULT_SCAN_LIMIT = 200;
|
|
6950
|
+
var DEFAULT_SCAN_STATUSES = ["active", "paused"];
|
|
6951
|
+
var DEFAULT_MAX_FINDINGS = 100;
|
|
6952
|
+
var MIN_STALE_RUNNING_MS = 10 * 60000;
|
|
6582
6953
|
var CLASSIFICATIONS = [
|
|
6583
6954
|
"rate_limit",
|
|
6584
6955
|
"auth",
|
|
6956
|
+
"provider_unavailable",
|
|
6585
6957
|
"model_not_found",
|
|
6586
6958
|
"context_length",
|
|
6587
6959
|
"schema_response_format",
|
|
@@ -6590,10 +6962,12 @@ var CLASSIFICATIONS = [
|
|
|
6590
6962
|
"route_functional",
|
|
6591
6963
|
"timeout",
|
|
6592
6964
|
"sigsegv",
|
|
6965
|
+
"restart_interrupted",
|
|
6593
6966
|
"skipped_previous_active",
|
|
6594
6967
|
"circuit_breaker",
|
|
6595
6968
|
"unknown"
|
|
6596
6969
|
];
|
|
6970
|
+
var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
|
|
6597
6971
|
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
6598
6972
|
if (!value)
|
|
6599
6973
|
return;
|
|
@@ -6607,7 +6981,7 @@ function redactedEvidence(value) {
|
|
|
6607
6981
|
}
|
|
6608
6982
|
function searchableText(run) {
|
|
6609
6983
|
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
6610
|
-
`)
|
|
6984
|
+
`);
|
|
6611
6985
|
}
|
|
6612
6986
|
function isRecord(value) {
|
|
6613
6987
|
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
@@ -6632,13 +7006,36 @@ function stableFingerprint(parts) {
|
|
|
6632
7006
|
return createHash2("sha256").update(parts.join(`
|
|
6633
7007
|
`)).digest("hex").slice(0, 16);
|
|
6634
7008
|
}
|
|
6635
|
-
function
|
|
7009
|
+
function stableScanFingerprint(parts) {
|
|
7010
|
+
return `openloops:health-scan:${stableFingerprint(parts)}`;
|
|
7011
|
+
}
|
|
7012
|
+
function safeHost(value) {
|
|
7013
|
+
if (!value)
|
|
7014
|
+
return;
|
|
7015
|
+
let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
|
|
7016
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
7017
|
+
return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
|
|
7018
|
+
}
|
|
7019
|
+
function isCursorHost(host) {
|
|
7020
|
+
return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
|
|
7021
|
+
}
|
|
7022
|
+
function providerUnavailableSummary(rawText) {
|
|
7023
|
+
const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
|
|
7024
|
+
if (dns) {
|
|
7025
|
+
const host = safeHost(dns[2]);
|
|
7026
|
+
if (isCursorHost(host))
|
|
7027
|
+
return `provider DNS lookup failed: ${dns[1]} ${host}`;
|
|
7028
|
+
}
|
|
7029
|
+
return;
|
|
7030
|
+
}
|
|
7031
|
+
function stableFailureFingerprint(run, classification, summary) {
|
|
7032
|
+
const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
|
|
6636
7033
|
return stableFingerprint([
|
|
6637
7034
|
run.loopId,
|
|
6638
7035
|
classification,
|
|
6639
7036
|
String(run.status),
|
|
6640
7037
|
String(run.exitCode ?? ""),
|
|
6641
|
-
|
|
7038
|
+
evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6642
7039
|
]);
|
|
6643
7040
|
}
|
|
6644
7041
|
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
@@ -6656,13 +7053,279 @@ function healthRun(run) {
|
|
|
6656
7053
|
stderr: redactedEvidence(run.stderr)
|
|
6657
7054
|
};
|
|
6658
7055
|
}
|
|
7056
|
+
function compactText(value, limit = 500) {
|
|
7057
|
+
const text = redact(bounded(value, limit));
|
|
7058
|
+
return text?.replace(/\s+/g, " ").trim();
|
|
7059
|
+
}
|
|
7060
|
+
function publicDoctorCheck(check) {
|
|
7061
|
+
return {
|
|
7062
|
+
id: check.id,
|
|
7063
|
+
status: check.status,
|
|
7064
|
+
message: redact(bounded(check.message, 500)) ?? "",
|
|
7065
|
+
detail: redact(bounded(check.detail, 800))
|
|
7066
|
+
};
|
|
7067
|
+
}
|
|
7068
|
+
function publicDoctorReport(report) {
|
|
7069
|
+
return {
|
|
7070
|
+
ok: report.ok,
|
|
7071
|
+
checks: report.checks.map(publicDoctorCheck)
|
|
7072
|
+
};
|
|
7073
|
+
}
|
|
7074
|
+
function includedStatusSet(statuses) {
|
|
7075
|
+
const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
|
|
7076
|
+
return new Set(values);
|
|
7077
|
+
}
|
|
7078
|
+
function statusCounts(loops) {
|
|
7079
|
+
return {
|
|
7080
|
+
loops: loops.length,
|
|
7081
|
+
active: loops.filter((loop) => loop.status === "active").length,
|
|
7082
|
+
paused: loops.filter((loop) => loop.status === "paused").length,
|
|
7083
|
+
stopped: loops.filter((loop) => loop.status === "stopped").length,
|
|
7084
|
+
expired: loops.filter((loop) => loop.status === "expired").length
|
|
7085
|
+
};
|
|
7086
|
+
}
|
|
7087
|
+
function compareLoopsForScan(left, right) {
|
|
7088
|
+
const statusOrder = left.status.localeCompare(right.status);
|
|
7089
|
+
if (statusOrder !== 0)
|
|
7090
|
+
return statusOrder;
|
|
7091
|
+
return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
|
|
7092
|
+
}
|
|
7093
|
+
function scanLoops(store, statuses, opts) {
|
|
7094
|
+
const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
|
|
7095
|
+
return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
|
|
7096
|
+
}
|
|
7097
|
+
function healthReportForLoops(store, loops, generatedAt) {
|
|
7098
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
7099
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
7100
|
+
for (const expectation of expectations) {
|
|
7101
|
+
if (expectation.failure)
|
|
7102
|
+
classifications[expectation.failure.classification] += 1;
|
|
7103
|
+
}
|
|
7104
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
7105
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
7106
|
+
return {
|
|
7107
|
+
ok: unhealthy === 0,
|
|
7108
|
+
generatedAt,
|
|
7109
|
+
summary: {
|
|
7110
|
+
loops: expectations.length,
|
|
7111
|
+
healthy: expectations.length - unhealthy,
|
|
7112
|
+
unhealthy,
|
|
7113
|
+
warnings
|
|
7114
|
+
},
|
|
7115
|
+
classifications,
|
|
7116
|
+
expectations
|
|
7117
|
+
};
|
|
7118
|
+
}
|
|
7119
|
+
function ageMs(run, now) {
|
|
7120
|
+
const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
|
|
7121
|
+
const time = Date.parse(stamp);
|
|
7122
|
+
return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
|
|
7123
|
+
}
|
|
7124
|
+
function shortLoop(loop) {
|
|
7125
|
+
return {
|
|
7126
|
+
id: loop.id,
|
|
7127
|
+
name: loop.name,
|
|
7128
|
+
status: loop.status,
|
|
7129
|
+
nextRunAt: loop.nextRunAt,
|
|
7130
|
+
leaseMs: loop.leaseMs
|
|
7131
|
+
};
|
|
7132
|
+
}
|
|
7133
|
+
function priorityForSeverity(severity) {
|
|
7134
|
+
if (severity === "critical")
|
|
7135
|
+
return "critical";
|
|
7136
|
+
if (severity === "high")
|
|
7137
|
+
return "high";
|
|
7138
|
+
if (severity === "low")
|
|
7139
|
+
return "low";
|
|
7140
|
+
return "medium";
|
|
7141
|
+
}
|
|
7142
|
+
function recommendedFindingTask(finding, route) {
|
|
7143
|
+
const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
|
|
7144
|
+
if (finding.classification)
|
|
7145
|
+
tags.push(finding.classification);
|
|
7146
|
+
const description = [
|
|
7147
|
+
`OpenLoops health scan found a ${finding.kind} issue.`,
|
|
7148
|
+
finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
|
|
7149
|
+
finding.run ? `Run: ${finding.run.id}` : undefined,
|
|
7150
|
+
finding.classification ? `Classification: ${finding.classification}` : undefined,
|
|
7151
|
+
finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
|
|
7152
|
+
finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
|
|
7153
|
+
`Fingerprint: ${finding.fingerprint}`,
|
|
7154
|
+
`Severity: ${finding.severity}`,
|
|
7155
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
7156
|
+
route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
7157
|
+
route?.provider ? `Provider: ${route.provider}` : undefined,
|
|
7158
|
+
"",
|
|
7159
|
+
finding.message,
|
|
7160
|
+
finding.run?.error ? `Error:
|
|
7161
|
+
${finding.run.error}` : undefined,
|
|
7162
|
+
finding.run?.stderr ? `Stderr:
|
|
7163
|
+
${finding.run.stderr}` : undefined
|
|
7164
|
+
].filter(Boolean).join(`
|
|
7165
|
+
|
|
7166
|
+
`);
|
|
7167
|
+
return {
|
|
7168
|
+
title: finding.title,
|
|
7169
|
+
description,
|
|
7170
|
+
priority: priorityForSeverity(finding.severity),
|
|
7171
|
+
tags,
|
|
7172
|
+
dedupeKey: finding.fingerprint,
|
|
7173
|
+
search: { query: finding.fingerprint },
|
|
7174
|
+
compatibilityFallback: {
|
|
7175
|
+
search: ["todos", "search", finding.fingerprint, "--json"],
|
|
7176
|
+
add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
|
|
7177
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
7178
|
+
},
|
|
7179
|
+
futureNativeUpsert: {
|
|
7180
|
+
command: "todos task upsert",
|
|
7181
|
+
fields: {
|
|
7182
|
+
title: finding.title,
|
|
7183
|
+
description,
|
|
7184
|
+
priority: priorityForSeverity(finding.severity),
|
|
7185
|
+
tags,
|
|
7186
|
+
dedupeKey: finding.fingerprint,
|
|
7187
|
+
routeSource: route?.source ?? "openloops",
|
|
7188
|
+
routeKind: route?.kind ?? "health_scan",
|
|
7189
|
+
routeLoopId: finding.loop?.id ?? "",
|
|
7190
|
+
routeLoopName: finding.loop?.name ?? ""
|
|
7191
|
+
}
|
|
7192
|
+
}
|
|
7193
|
+
};
|
|
7194
|
+
}
|
|
7195
|
+
function daemonFinding(daemon) {
|
|
7196
|
+
if (daemon.running && !daemon.stale)
|
|
7197
|
+
return;
|
|
7198
|
+
const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
|
|
7199
|
+
const severity = "critical";
|
|
7200
|
+
const finding = {
|
|
7201
|
+
kind: "daemon",
|
|
7202
|
+
severity,
|
|
7203
|
+
fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
|
|
7204
|
+
title: "OpenLoops daemon health issue",
|
|
7205
|
+
message: reason
|
|
7206
|
+
};
|
|
7207
|
+
return {
|
|
7208
|
+
...finding,
|
|
7209
|
+
recommendedTask: recommendedFindingTask(finding, undefined)
|
|
7210
|
+
};
|
|
7211
|
+
}
|
|
7212
|
+
function doctorSeverity(check) {
|
|
7213
|
+
if (check.status === "fail" && check.id === "data-dir")
|
|
7214
|
+
return "critical";
|
|
7215
|
+
if (check.status === "fail")
|
|
7216
|
+
return "high";
|
|
7217
|
+
return "medium";
|
|
7218
|
+
}
|
|
7219
|
+
function doctorFinding(check, loop, route) {
|
|
7220
|
+
if (check.status === "ok" || check.id === "loop-runs")
|
|
7221
|
+
return;
|
|
7222
|
+
const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
|
|
7223
|
+
const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
|
|
7224
|
+
const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
|
|
7225
|
+
const finding = {
|
|
7226
|
+
kind,
|
|
7227
|
+
severity,
|
|
7228
|
+
fingerprint,
|
|
7229
|
+
title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
|
|
7230
|
+
message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
|
|
7231
|
+
loop: loop ? shortLoop(loop) : undefined,
|
|
7232
|
+
route,
|
|
7233
|
+
doctorCheck: publicDoctorCheck(check)
|
|
7234
|
+
};
|
|
7235
|
+
return {
|
|
7236
|
+
...finding,
|
|
7237
|
+
recommendedTask: recommendedFindingTask(finding, route)
|
|
7238
|
+
};
|
|
7239
|
+
}
|
|
7240
|
+
function latestRunFinding(expectation) {
|
|
7241
|
+
if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
|
|
7242
|
+
return;
|
|
7243
|
+
const failure = expectation.failure;
|
|
7244
|
+
if (!failure)
|
|
7245
|
+
return;
|
|
7246
|
+
const severity = expectation.loop.status === "active" ? "high" : "medium";
|
|
7247
|
+
return {
|
|
7248
|
+
kind: "latest-run",
|
|
7249
|
+
severity,
|
|
7250
|
+
fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
|
|
7251
|
+
title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
|
|
7252
|
+
message: expectation.check.message,
|
|
7253
|
+
loop: expectation.loop,
|
|
7254
|
+
run: expectation.latestRun,
|
|
7255
|
+
route: expectation.route,
|
|
7256
|
+
classification: failure.classification,
|
|
7257
|
+
doctorCheck: undefined,
|
|
7258
|
+
recommendedTask: expectation.recommendedTask
|
|
7259
|
+
};
|
|
7260
|
+
}
|
|
7261
|
+
function staleRunningFinding(loop, expectation, now, staleRunningMs) {
|
|
7262
|
+
const run = expectation.latestRun;
|
|
7263
|
+
if (loop.status !== "active" || run?.status !== "running")
|
|
7264
|
+
return;
|
|
7265
|
+
const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
|
|
7266
|
+
const age = ageMs(run, now);
|
|
7267
|
+
if (age <= threshold)
|
|
7268
|
+
return;
|
|
7269
|
+
const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
|
|
7270
|
+
const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
|
|
7271
|
+
const finding = {
|
|
7272
|
+
kind: "stale-running",
|
|
7273
|
+
severity: "critical",
|
|
7274
|
+
fingerprint,
|
|
7275
|
+
title: `OpenLoops stale running run - ${loop.name}`,
|
|
7276
|
+
message,
|
|
7277
|
+
loop: shortLoop(loop),
|
|
7278
|
+
run,
|
|
7279
|
+
route: expectation.route,
|
|
7280
|
+
ageMs: age,
|
|
7281
|
+
staleThresholdMs: threshold
|
|
7282
|
+
};
|
|
7283
|
+
return {
|
|
7284
|
+
...finding,
|
|
7285
|
+
recommendedTask: recommendedFindingTask(finding, expectation.route)
|
|
7286
|
+
};
|
|
7287
|
+
}
|
|
7288
|
+
function scanStatus(findings) {
|
|
7289
|
+
if (findings.some((finding) => finding.severity === "critical"))
|
|
7290
|
+
return "critical";
|
|
7291
|
+
if (findings.length > 0)
|
|
7292
|
+
return "degraded";
|
|
7293
|
+
return "ok";
|
|
7294
|
+
}
|
|
7295
|
+
function timestampDir(root, generatedAt) {
|
|
7296
|
+
const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
|
|
7297
|
+
return join6(root, stamp);
|
|
7298
|
+
}
|
|
7299
|
+
function healthScanMarkdown(scan) {
|
|
7300
|
+
return [
|
|
7301
|
+
"# OpenLoops Health Scan",
|
|
7302
|
+
"",
|
|
7303
|
+
`- status: ${scan.status}`,
|
|
7304
|
+
`- generated_at: ${scan.generatedAt}`,
|
|
7305
|
+
`- included_statuses: ${scan.includedStatuses.join(",")}`,
|
|
7306
|
+
`- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
|
|
7307
|
+
`- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
|
|
7308
|
+
scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
|
|
7309
|
+
scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
|
|
7310
|
+
scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
|
|
7311
|
+
"",
|
|
7312
|
+
"## Findings",
|
|
7313
|
+
scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
|
|
7314
|
+
`) : "None."
|
|
7315
|
+
].join(`
|
|
7316
|
+
`);
|
|
7317
|
+
}
|
|
6659
7318
|
function classifyRunFailure(run) {
|
|
6660
7319
|
if (run.status === "succeeded" || run.status === "running")
|
|
6661
7320
|
return;
|
|
6662
|
-
const
|
|
7321
|
+
const rawText = searchableText(run);
|
|
7322
|
+
const text = rawText.toLowerCase();
|
|
6663
7323
|
let classification = "unknown";
|
|
7324
|
+
let summary;
|
|
6664
7325
|
if (run.status === "timed_out")
|
|
6665
7326
|
classification = "timeout";
|
|
7327
|
+
else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
|
|
7328
|
+
classification = "restart_interrupted";
|
|
6666
7329
|
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
6667
7330
|
classification = "circuit_breaker";
|
|
6668
7331
|
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
@@ -6671,8 +7334,10 @@ function classifyRunFailure(run) {
|
|
|
6671
7334
|
classification = "preflight";
|
|
6672
7335
|
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
6673
7336
|
classification = "rate_limit";
|
|
6674
|
-
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
7337
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
|
|
6675
7338
|
classification = "auth";
|
|
7339
|
+
else if (summary = providerUnavailableSummary(rawText))
|
|
7340
|
+
classification = "provider_unavailable";
|
|
6676
7341
|
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
6677
7342
|
classification = "model_not_found";
|
|
6678
7343
|
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
@@ -6685,8 +7350,9 @@ function classifyRunFailure(run) {
|
|
|
6685
7350
|
classification = "sigsegv";
|
|
6686
7351
|
return {
|
|
6687
7352
|
classification,
|
|
6688
|
-
fingerprint: stableFailureFingerprint(run, classification),
|
|
7353
|
+
fingerprint: stableFailureFingerprint(run, classification, summary),
|
|
6689
7354
|
evidence: {
|
|
7355
|
+
summary,
|
|
6690
7356
|
error: redactedEvidence(run.error),
|
|
6691
7357
|
stdout: redactedEvidence(run.stdout),
|
|
6692
7358
|
stderr: redactedEvidence(run.stderr),
|
|
@@ -6731,7 +7397,7 @@ function routeEvidenceReport(run) {
|
|
|
6731
7397
|
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
6732
7398
|
if (evidencePath && existsSync3(evidencePath)) {
|
|
6733
7399
|
try {
|
|
6734
|
-
return parseJsonObject(
|
|
7400
|
+
return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
|
|
6735
7401
|
} catch {
|
|
6736
7402
|
return stdoutReport;
|
|
6737
7403
|
}
|
|
@@ -6863,6 +7529,12 @@ function targetRoute(loop) {
|
|
|
6863
7529
|
loopName: loop.name
|
|
6864
7530
|
};
|
|
6865
7531
|
}
|
|
7532
|
+
function expectationLoop(loop) {
|
|
7533
|
+
return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
|
|
7534
|
+
}
|
|
7535
|
+
function hasPendingRetry(loop, run) {
|
|
7536
|
+
return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
|
|
7537
|
+
}
|
|
6866
7538
|
function recommendedTask(loop, run, failure, route) {
|
|
6867
7539
|
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
6868
7540
|
const description = [
|
|
@@ -6874,6 +7546,7 @@ function recommendedTask(loop, run, failure, route) {
|
|
|
6874
7546
|
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6875
7547
|
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6876
7548
|
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
7549
|
+
failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
|
|
6877
7550
|
failure.evidence.error ? `Error:
|
|
6878
7551
|
${failure.evidence.error}` : undefined,
|
|
6879
7552
|
failure.evidence.stderr ? `Stderr:
|
|
@@ -6883,7 +7556,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6883
7556
|
`);
|
|
6884
7557
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6885
7558
|
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6886
|
-
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
7559
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
|
|
6887
7560
|
return {
|
|
6888
7561
|
title,
|
|
6889
7562
|
description,
|
|
@@ -6917,7 +7590,7 @@ function expectationForLoop(store, loop) {
|
|
|
6917
7590
|
const route = targetRoute(loop);
|
|
6918
7591
|
if (!latestRun) {
|
|
6919
7592
|
return {
|
|
6920
|
-
loop:
|
|
7593
|
+
loop: expectationLoop(loop),
|
|
6921
7594
|
ok: true,
|
|
6922
7595
|
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
6923
7596
|
route
|
|
@@ -6926,7 +7599,7 @@ function expectationForLoop(store, loop) {
|
|
|
6926
7599
|
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
6927
7600
|
if (routeFailure) {
|
|
6928
7601
|
return {
|
|
6929
|
-
loop:
|
|
7602
|
+
loop: expectationLoop(loop),
|
|
6930
7603
|
ok: false,
|
|
6931
7604
|
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
6932
7605
|
latestRun: healthRun(latestRun),
|
|
@@ -6937,7 +7610,7 @@ function expectationForLoop(store, loop) {
|
|
|
6937
7610
|
}
|
|
6938
7611
|
if (latestRun.status === "succeeded") {
|
|
6939
7612
|
return {
|
|
6940
|
-
loop:
|
|
7613
|
+
loop: expectationLoop(loop),
|
|
6941
7614
|
ok: true,
|
|
6942
7615
|
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
6943
7616
|
latestRun: healthRun(latestRun),
|
|
@@ -6948,7 +7621,7 @@ function expectationForLoop(store, loop) {
|
|
|
6948
7621
|
if (failure?.classification === "circuit_breaker") {
|
|
6949
7622
|
if (loop.status !== "paused") {
|
|
6950
7623
|
return {
|
|
6951
|
-
loop:
|
|
7624
|
+
loop: expectationLoop(loop),
|
|
6952
7625
|
ok: true,
|
|
6953
7626
|
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
6954
7627
|
latestRun: healthRun(latestRun),
|
|
@@ -6956,7 +7629,7 @@ function expectationForLoop(store, loop) {
|
|
|
6956
7629
|
};
|
|
6957
7630
|
}
|
|
6958
7631
|
return {
|
|
6959
|
-
loop:
|
|
7632
|
+
loop: expectationLoop(loop),
|
|
6960
7633
|
ok: false,
|
|
6961
7634
|
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
6962
7635
|
latestRun: healthRun(latestRun),
|
|
@@ -6965,8 +7638,33 @@ function expectationForLoop(store, loop) {
|
|
|
6965
7638
|
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
6966
7639
|
};
|
|
6967
7640
|
}
|
|
7641
|
+
if (failure?.classification === "restart_interrupted") {
|
|
7642
|
+
return {
|
|
7643
|
+
loop: expectationLoop(loop),
|
|
7644
|
+
ok: true,
|
|
7645
|
+
check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
|
|
7646
|
+
latestRun: healthRun(latestRun),
|
|
7647
|
+
failure,
|
|
7648
|
+
route
|
|
7649
|
+
};
|
|
7650
|
+
}
|
|
7651
|
+
if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
|
|
7652
|
+
const message = [
|
|
7653
|
+
"provider unavailable/network failure; retry is scheduled",
|
|
7654
|
+
loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
|
|
7655
|
+
failure.evidence.summary
|
|
7656
|
+
].filter(Boolean).join("; ");
|
|
7657
|
+
return {
|
|
7658
|
+
loop: expectationLoop(loop),
|
|
7659
|
+
ok: true,
|
|
7660
|
+
check: { id: "latest-run-succeeded", status: "warn", message },
|
|
7661
|
+
latestRun: healthRun(latestRun),
|
|
7662
|
+
failure,
|
|
7663
|
+
route
|
|
7664
|
+
};
|
|
7665
|
+
}
|
|
6968
7666
|
return {
|
|
6969
|
-
loop:
|
|
7667
|
+
loop: expectationLoop(loop),
|
|
6970
7668
|
ok: false,
|
|
6971
7669
|
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
6972
7670
|
latestRun: healthRun(latestRun),
|
|
@@ -6998,6 +7696,89 @@ function buildHealthReport(store, opts = {}) {
|
|
|
6998
7696
|
expectations
|
|
6999
7697
|
};
|
|
7000
7698
|
}
|
|
7699
|
+
function buildHealthScan(store, opts = {}) {
|
|
7700
|
+
const generatedAt = (opts.now ?? new Date).toISOString();
|
|
7701
|
+
const now = opts.now ?? new Date(generatedAt);
|
|
7702
|
+
const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
|
|
7703
|
+
const loops = scanLoops(store, includeStatuses, opts);
|
|
7704
|
+
const health = healthReportForLoops(store, loops, generatedAt);
|
|
7705
|
+
const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
|
|
7706
|
+
const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
|
|
7707
|
+
const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
|
|
7708
|
+
const allFindings = [];
|
|
7709
|
+
const pushFinding = (finding) => {
|
|
7710
|
+
if (!finding)
|
|
7711
|
+
return;
|
|
7712
|
+
allFindings.push(finding);
|
|
7713
|
+
};
|
|
7714
|
+
if (opts.daemon)
|
|
7715
|
+
pushFinding(daemonFinding(opts.daemon));
|
|
7716
|
+
if (opts.latestRun !== false) {
|
|
7717
|
+
for (const loop of loops) {
|
|
7718
|
+
const expectation = expectationsByLoopId.get(loop.id);
|
|
7719
|
+
if (!expectation)
|
|
7720
|
+
continue;
|
|
7721
|
+
pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
|
|
7722
|
+
pushFinding(latestRunFinding(expectation));
|
|
7723
|
+
}
|
|
7724
|
+
}
|
|
7725
|
+
const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
|
|
7726
|
+
if (doctor) {
|
|
7727
|
+
for (const check of doctor.checks) {
|
|
7728
|
+
const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
|
|
7729
|
+
const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
|
|
7730
|
+
const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
|
|
7731
|
+
pushFinding(doctorFinding(check, loop, route));
|
|
7732
|
+
}
|
|
7733
|
+
}
|
|
7734
|
+
const findings = allFindings.slice(0, maxFindings);
|
|
7735
|
+
const status = scanStatus(allFindings);
|
|
7736
|
+
const baseCounts = statusCounts(loops);
|
|
7737
|
+
return {
|
|
7738
|
+
ok: status === "ok",
|
|
7739
|
+
status,
|
|
7740
|
+
generatedAt,
|
|
7741
|
+
includedStatuses: includeStatuses,
|
|
7742
|
+
counts: {
|
|
7743
|
+
...baseCounts,
|
|
7744
|
+
latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
|
|
7745
|
+
staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
|
|
7746
|
+
daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
|
|
7747
|
+
doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
|
|
7748
|
+
preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
|
|
7749
|
+
findings: allFindings.length,
|
|
7750
|
+
reportedFindings: findings.length,
|
|
7751
|
+
truncatedFindings: Math.max(0, allFindings.length - findings.length)
|
|
7752
|
+
},
|
|
7753
|
+
daemon: opts.daemon ? {
|
|
7754
|
+
running: opts.daemon.running,
|
|
7755
|
+
stale: opts.daemon.stale,
|
|
7756
|
+
pid: opts.daemon.pid,
|
|
7757
|
+
host: opts.daemon.host,
|
|
7758
|
+
loops: opts.daemon.loops,
|
|
7759
|
+
runs: opts.daemon.runs,
|
|
7760
|
+
logPath: opts.daemon.logPath
|
|
7761
|
+
} : undefined,
|
|
7762
|
+
doctor,
|
|
7763
|
+
health,
|
|
7764
|
+
selfHeals: opts.selfHeals ?? [],
|
|
7765
|
+
findings
|
|
7766
|
+
};
|
|
7767
|
+
}
|
|
7768
|
+
function writeHealthScanReports(scan, opts = {}) {
|
|
7769
|
+
const root = opts.reportDir ?? join6(dataDir(), "reports", "health-scan");
|
|
7770
|
+
const dir = timestampDir(root, scan.generatedAt);
|
|
7771
|
+
mkdirSync5(dir, { recursive: true, mode: 448 });
|
|
7772
|
+
const json = join6(dir, "summary.json");
|
|
7773
|
+
const markdown = join6(dir, "report.md");
|
|
7774
|
+
const withReports = {
|
|
7775
|
+
...scan,
|
|
7776
|
+
reports: { dir, json, markdown }
|
|
7777
|
+
};
|
|
7778
|
+
writeFileSync2(json, JSON.stringify(withReports, null, 2), { mode: 384 });
|
|
7779
|
+
writeFileSync2(markdown, healthScanMarkdown(withReports), { mode: 384 });
|
|
7780
|
+
return withReports;
|
|
7781
|
+
}
|
|
7001
7782
|
|
|
7002
7783
|
// src/lib/scheduler.ts
|
|
7003
7784
|
function loopLane(loop) {
|
|
@@ -7084,7 +7865,7 @@ var MAX_RETRY_EXPONENT = 20;
|
|
|
7084
7865
|
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
7085
7866
|
const attempt = Math.max(1, run.attempt);
|
|
7086
7867
|
const failure = classifyRunFailure(run);
|
|
7087
|
-
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
7868
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
|
|
7088
7869
|
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
7089
7870
|
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
7090
7871
|
const jitter = 0.5 + random();
|
|
@@ -7211,20 +7992,21 @@ async function executeClaimedRun(deps) {
|
|
|
7211
7992
|
daemonLeaseId: deps.daemonLeaseId,
|
|
7212
7993
|
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
7213
7994
|
})))(deps.loop, deps.run);
|
|
7995
|
+
const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
|
|
7214
7996
|
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
7215
7997
|
return deps.store.finalizeRun(deps.run.id, {
|
|
7216
|
-
status:
|
|
7217
|
-
finishedAt:
|
|
7218
|
-
durationMs:
|
|
7219
|
-
stdout:
|
|
7220
|
-
stderr:
|
|
7221
|
-
exitCode:
|
|
7222
|
-
error:
|
|
7223
|
-
pid:
|
|
7998
|
+
status: finalResult.status,
|
|
7999
|
+
finishedAt: finalResult.finishedAt,
|
|
8000
|
+
durationMs: finalResult.durationMs,
|
|
8001
|
+
stdout: finalResult.stdout,
|
|
8002
|
+
stderr: finalResult.stderr,
|
|
8003
|
+
exitCode: finalResult.exitCode,
|
|
8004
|
+
error: finalResult.error,
|
|
8005
|
+
pid: finalResult.pid
|
|
7224
8006
|
}, {
|
|
7225
8007
|
claimedBy: deps.runnerId,
|
|
7226
8008
|
daemonLeaseId: deps.daemonLeaseId,
|
|
7227
|
-
now: deps.now?.() ?? new Date(
|
|
8009
|
+
now: deps.now?.() ?? new Date(finalResult.finishedAt)
|
|
7228
8010
|
});
|
|
7229
8011
|
} catch (err) {
|
|
7230
8012
|
deps.onError?.(deps.loop, err);
|
|
@@ -7459,10 +8241,10 @@ async function tick(deps) {
|
|
|
7459
8241
|
}
|
|
7460
8242
|
|
|
7461
8243
|
// src/daemon/control.ts
|
|
7462
|
-
import { existsSync as existsSync4, mkdirSync as
|
|
7463
|
-
import { spawnSync as
|
|
8244
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, rmSync as rmSync4, writeFileSync as writeFileSync3 } from "fs";
|
|
8245
|
+
import { spawnSync as spawnSync5 } from "child_process";
|
|
7464
8246
|
import { hostname } from "os";
|
|
7465
|
-
import { dirname as dirname4, join as
|
|
8247
|
+
import { dirname as dirname4, join as join7 } from "path";
|
|
7466
8248
|
|
|
7467
8249
|
// src/daemon/loop.ts
|
|
7468
8250
|
function realSleep(ms) {
|
|
@@ -7492,7 +8274,7 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
7492
8274
|
return;
|
|
7493
8275
|
let raw;
|
|
7494
8276
|
try {
|
|
7495
|
-
raw =
|
|
8277
|
+
raw = readFileSync5(path, "utf8").trim();
|
|
7496
8278
|
} catch {
|
|
7497
8279
|
return;
|
|
7498
8280
|
}
|
|
@@ -7514,11 +8296,11 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
7514
8296
|
return Number.isInteger(pid) && pid > 0 ? { pid } : undefined;
|
|
7515
8297
|
}
|
|
7516
8298
|
function writePid(pid = process.pid, path = pidFilePath()) {
|
|
7517
|
-
|
|
7518
|
-
|
|
8299
|
+
mkdirSync6(dirname4(path), { recursive: true, mode: 448 });
|
|
8300
|
+
writeFileSync3(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
7519
8301
|
}
|
|
7520
8302
|
function removePid(path = pidFilePath()) {
|
|
7521
|
-
|
|
8303
|
+
rmSync4(path, { force: true });
|
|
7522
8304
|
}
|
|
7523
8305
|
function isAlive(pid, startedAt) {
|
|
7524
8306
|
try {
|
|
@@ -7548,7 +8330,7 @@ function ownProcessGroupId() {
|
|
|
7548
8330
|
return pgrp;
|
|
7549
8331
|
}
|
|
7550
8332
|
try {
|
|
7551
|
-
const run =
|
|
8333
|
+
const run = spawnSync5("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
7552
8334
|
const pgid = Number(run.stdout.trim());
|
|
7553
8335
|
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
7554
8336
|
return pgid;
|
|
@@ -7644,7 +8426,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7644
8426
|
if (!state.running || !state.pid)
|
|
7645
8427
|
return { wasRunning: false, stopped: false, forced: false };
|
|
7646
8428
|
const sleep = opts.sleep ?? realSleep;
|
|
7647
|
-
const store = new Store(
|
|
8429
|
+
const store = new Store(join7(dirname4(path), "loops.db"));
|
|
7648
8430
|
try {
|
|
7649
8431
|
const lease = store.getDaemonLease();
|
|
7650
8432
|
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
@@ -7682,7 +8464,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7682
8464
|
}
|
|
7683
8465
|
|
|
7684
8466
|
// src/daemon/daemon.ts
|
|
7685
|
-
import { copyFileSync, existsSync as existsSync5, openSync, renameSync as renameSync2, rmSync as
|
|
8467
|
+
import { copyFileSync, existsSync as existsSync5, openSync as openSync2, renameSync as renameSync2, rmSync as rmSync5, statSync, truncateSync } from "fs";
|
|
7686
8468
|
import { hostname as hostname2 } from "os";
|
|
7687
8469
|
import { spawn as spawn3 } from "child_process";
|
|
7688
8470
|
function intervalFromEnv() {
|
|
@@ -7724,7 +8506,7 @@ function rotateDaemonLog(path = daemonLogPath(), maxBytes = DAEMON_LOG_MAX_BYTES
|
|
|
7724
8506
|
if (size < maxBytes)
|
|
7725
8507
|
return false;
|
|
7726
8508
|
try {
|
|
7727
|
-
|
|
8509
|
+
rmSync5(`${path}.${keep}`, { force: true });
|
|
7728
8510
|
for (let i = keep - 1;i >= 1; i--) {
|
|
7729
8511
|
if (existsSync5(`${path}.${i}`))
|
|
7730
8512
|
renameSync2(`${path}.${i}`, `${path}.${i + 1}`);
|
|
@@ -7773,6 +8555,7 @@ async function runDaemon(opts = {}) {
|
|
|
7773
8555
|
let runAbort = new AbortController;
|
|
7774
8556
|
const activeRuns = new Map;
|
|
7775
8557
|
const activeByLane = { command: 0, agent: 0 };
|
|
8558
|
+
const isControlledStopInterruption = (result) => stopFlag && !leaseLost && result.status === "failed" && result.error?.includes("terminated by SIGTERM") === true;
|
|
7776
8559
|
const requestStop = (message) => {
|
|
7777
8560
|
stopFlag = true;
|
|
7778
8561
|
if (!runAbort.signal.aborted)
|
|
@@ -7856,6 +8639,16 @@ async function runDaemon(opts = {}) {
|
|
|
7856
8639
|
store.recordRunProcess(run.id, info, { daemonLeaseId: leaseId });
|
|
7857
8640
|
}
|
|
7858
8641
|
})),
|
|
8642
|
+
finalizeResult: (result) => {
|
|
8643
|
+
if (!isControlledStopInterruption(result))
|
|
8644
|
+
return result;
|
|
8645
|
+
return {
|
|
8646
|
+
...result,
|
|
8647
|
+
status: "skipped",
|
|
8648
|
+
exitCode: undefined,
|
|
8649
|
+
error: `${RESTART_INTERRUPTED_RUN_PREFIX}: child process terminated by SIGTERM during daemon stop/restart`
|
|
8650
|
+
};
|
|
8651
|
+
},
|
|
7859
8652
|
onError: (loop, err) => log(`loop ${loop.id} failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
7860
8653
|
});
|
|
7861
8654
|
ensureLease();
|
|
@@ -7949,7 +8742,7 @@ async function startDaemon(opts) {
|
|
|
7949
8742
|
return { started: false, alreadyRunning: true, pid: state.pid };
|
|
7950
8743
|
if (state.stale)
|
|
7951
8744
|
removePid();
|
|
7952
|
-
const out =
|
|
8745
|
+
const out = openSync2(daemonLogPath(), "a");
|
|
7953
8746
|
const child = spawn3(opts.execPath ?? process.execPath, [opts.cliEntry, ...opts.args ?? ["daemon", "run"]], {
|
|
7954
8747
|
detached: true,
|
|
7955
8748
|
stdio: ["ignore", out, out]
|
|
@@ -7967,8 +8760,8 @@ async function startDaemon(opts) {
|
|
|
7967
8760
|
}
|
|
7968
8761
|
|
|
7969
8762
|
// src/daemon/install.ts
|
|
7970
|
-
import { chmodSync as chmodSync2, mkdirSync as
|
|
7971
|
-
import { spawnSync as
|
|
8763
|
+
import { chmodSync as chmodSync2, mkdirSync as mkdirSync7, writeFileSync as writeFileSync4 } from "fs";
|
|
8764
|
+
import { spawnSync as spawnSync6 } from "child_process";
|
|
7972
8765
|
import { dirname as dirname5 } from "path";
|
|
7973
8766
|
function systemdEscapeExecPart(part) {
|
|
7974
8767
|
const escaped = part.replaceAll("%", "%%");
|
|
@@ -8000,9 +8793,9 @@ function installStartup(cliEntry, execPath = process.execPath, args = ["daemon",
|
|
|
8000
8793
|
const dataDirPath = ensureDataDir();
|
|
8001
8794
|
if (platform === "linux") {
|
|
8002
8795
|
const path = systemdServicePath();
|
|
8003
|
-
|
|
8796
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 448 });
|
|
8004
8797
|
const execStart = [execPath, cliEntry, ...args].map(systemdEscapeExecPart).join(" ");
|
|
8005
|
-
|
|
8798
|
+
writeFileSync4(path, `[Unit]
|
|
8006
8799
|
Description=Hasna OpenLoops daemon
|
|
8007
8800
|
After=basic.target
|
|
8008
8801
|
|
|
@@ -8030,8 +8823,8 @@ WantedBy=default.target
|
|
|
8030
8823
|
}
|
|
8031
8824
|
if (platform === "darwin") {
|
|
8032
8825
|
const path = launchdPlistPath();
|
|
8033
|
-
|
|
8034
|
-
|
|
8826
|
+
mkdirSync7(dirname5(path), { recursive: true, mode: 448 });
|
|
8827
|
+
writeFileSync4(path, `<?xml version="1.0" encoding="UTF-8"?>
|
|
8035
8828
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
8036
8829
|
<plist version="1.0">
|
|
8037
8830
|
<dict>
|
|
@@ -8068,7 +8861,7 @@ ${args.map((arg) => ` <string>${xmlEscape(arg)}</string>`).join(`
|
|
|
8068
8861
|
function enableStartup(result) {
|
|
8069
8862
|
const commands = result.platform === "linux" ? ["systemctl --user daemon-reload", "systemctl --user enable --now loops-daemon.service"] : result.platform === "darwin" ? launchctlCommands(result.path) : [];
|
|
8070
8863
|
return commands.map((command) => {
|
|
8071
|
-
const run =
|
|
8864
|
+
const run = spawnSync6("sh", ["-c", command], {
|
|
8072
8865
|
encoding: "utf8",
|
|
8073
8866
|
stdio: ["ignore", "pipe", "pipe"]
|
|
8074
8867
|
});
|
|
@@ -8082,7 +8875,7 @@ function enableStartup(result) {
|
|
|
8082
8875
|
}
|
|
8083
8876
|
|
|
8084
8877
|
// src/lib/doctor.ts
|
|
8085
|
-
import { spawnSync as
|
|
8878
|
+
import { spawnSync as spawnSync7 } from "child_process";
|
|
8086
8879
|
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
8087
8880
|
var PROVIDER_COMMANDS = [
|
|
8088
8881
|
"claude",
|
|
@@ -8093,11 +8886,11 @@ var PROVIDER_COMMANDS = [
|
|
|
8093
8886
|
"codex"
|
|
8094
8887
|
];
|
|
8095
8888
|
function hasCommand(command) {
|
|
8096
|
-
const result =
|
|
8889
|
+
const result = spawnSync7("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
8097
8890
|
return (result.status ?? 1) === 0;
|
|
8098
8891
|
}
|
|
8099
8892
|
function commandVersion(command) {
|
|
8100
|
-
const result =
|
|
8893
|
+
const result = spawnSync7(command, ["--version"], {
|
|
8101
8894
|
encoding: "utf8",
|
|
8102
8895
|
stdio: ["ignore", "pipe", "pipe"]
|
|
8103
8896
|
});
|
|
@@ -8146,7 +8939,30 @@ function runDoctor(store) {
|
|
|
8146
8939
|
const status = daemonStatus(store);
|
|
8147
8940
|
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
8148
8941
|
const failedRuns = store.countRuns("failed");
|
|
8942
|
+
const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
|
|
8149
8943
|
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
8944
|
+
if (restartInterruptedRuns > 0) {
|
|
8945
|
+
checks.push({
|
|
8946
|
+
id: "loop-runs:restart-interrupted",
|
|
8947
|
+
status: "warn",
|
|
8948
|
+
message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
|
|
8949
|
+
});
|
|
8950
|
+
}
|
|
8951
|
+
const deployment = buildDeploymentStatus();
|
|
8952
|
+
const schedulerState = deployment.schedulerState;
|
|
8953
|
+
checks.push({
|
|
8954
|
+
id: "scheduler-state",
|
|
8955
|
+
status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
|
|
8956
|
+
message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
|
|
8957
|
+
detail: [
|
|
8958
|
+
`route_state=${schedulerState.routeAdmission.stateStore}`,
|
|
8959
|
+
`active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
|
|
8960
|
+
`gates=${schedulerState.routeAdmission.gates.join(",")}`,
|
|
8961
|
+
`artifacts=${schedulerState.localStore.runArtifacts}`,
|
|
8962
|
+
`remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
|
|
8963
|
+
`remote_apply=${String(schedulerState.remoteStore.applySupported)}`
|
|
8964
|
+
].join(" ")
|
|
8965
|
+
});
|
|
8150
8966
|
for (const loop of store.listLoops({ status: "active" })) {
|
|
8151
8967
|
try {
|
|
8152
8968
|
if (loop.target.type === "workflow") {
|
|
@@ -8177,6 +8993,365 @@ function runDoctor(store) {
|
|
|
8177
8993
|
};
|
|
8178
8994
|
}
|
|
8179
8995
|
|
|
8996
|
+
// src/cli/ui.ts
|
|
8997
|
+
var CLEAR_SCREEN = "\x1B[2J\x1B[H";
|
|
8998
|
+
var ENTER_ALT_SCREEN = "\x1B[?1049h";
|
|
8999
|
+
var EXIT_ALT_SCREEN = "\x1B[?1049l";
|
|
9000
|
+
var HIDE_CURSOR = "\x1B[?25l";
|
|
9001
|
+
var SHOW_CURSOR = "\x1B[?25h";
|
|
9002
|
+
var RESET = "\x1B[0m";
|
|
9003
|
+
var BOLD = "\x1B[1m";
|
|
9004
|
+
var DIM = "\x1B[2m";
|
|
9005
|
+
var CYAN = "\x1B[36m";
|
|
9006
|
+
var GREEN = "\x1B[32m";
|
|
9007
|
+
var YELLOW = "\x1B[33m";
|
|
9008
|
+
var RED = "\x1B[31m";
|
|
9009
|
+
var MAGENTA = "\x1B[35m";
|
|
9010
|
+
var DEFAULT_REFRESH_MS = 2000;
|
|
9011
|
+
var MIN_REFRESH_MS = 500;
|
|
9012
|
+
var MAX_ACTIVE_LOOPS = 1e4;
|
|
9013
|
+
var MAX_RUNNING_RUNS = 1e5;
|
|
9014
|
+
function buildLoopUiSnapshot(store, opts = {}) {
|
|
9015
|
+
const now = opts.now ?? new Date;
|
|
9016
|
+
const activeLoops = store.listLoops({ status: "active", limit: opts.limit ?? MAX_ACTIVE_LOOPS });
|
|
9017
|
+
const runningRuns = store.listRuns({ status: "running", limit: MAX_RUNNING_RUNS });
|
|
9018
|
+
const runningByLoop = new Map;
|
|
9019
|
+
for (const run of runningRuns) {
|
|
9020
|
+
runningByLoop.set(run.loopId, (runningByLoop.get(run.loopId) ?? 0) + 1);
|
|
9021
|
+
}
|
|
9022
|
+
return {
|
|
9023
|
+
rows: activeLoops.map((loop) => {
|
|
9024
|
+
const latest = store.listRuns({ loopId: loop.id, limit: 1 })[0];
|
|
9025
|
+
return {
|
|
9026
|
+
id: loop.id,
|
|
9027
|
+
name: loop.name,
|
|
9028
|
+
status: loop.status,
|
|
9029
|
+
cadence: scheduleLabel(loop.schedule),
|
|
9030
|
+
nextRun: nextRunLabel(loop.nextRunAt, now),
|
|
9031
|
+
lastRunOutcome: runOutcomeLabel(latest),
|
|
9032
|
+
provider: providerLabel(loop.target),
|
|
9033
|
+
activeRuns: runningByLoop.get(loop.id) ?? 0
|
|
9034
|
+
};
|
|
9035
|
+
}),
|
|
9036
|
+
stats: {
|
|
9037
|
+
activeLoops: store.countLoops("active"),
|
|
9038
|
+
pausedLoops: store.countLoops("paused"),
|
|
9039
|
+
stoppedLoops: store.countLoops("stopped"),
|
|
9040
|
+
runningRuns: store.countRuns("running"),
|
|
9041
|
+
failedRuns: store.countRuns("failed"),
|
|
9042
|
+
updatedAt: now.toISOString()
|
|
9043
|
+
}
|
|
9044
|
+
};
|
|
9045
|
+
}
|
|
9046
|
+
function renderLoopUiFrame(snapshot, opts = {}) {
|
|
9047
|
+
const columns = Math.max(40, opts.columns ?? 100);
|
|
9048
|
+
const rows = opts.rows ?? snapshot.rows.length + 8;
|
|
9049
|
+
const refreshMs = opts.refreshMs ?? DEFAULT_REFRESH_MS;
|
|
9050
|
+
const color = opts.color ?? false;
|
|
9051
|
+
const compact2 = columns < 88;
|
|
9052
|
+
const widths = tableWidths(columns, compact2);
|
|
9053
|
+
const maxBodyRows = Math.max(0, rows - 7);
|
|
9054
|
+
const shownRows = snapshot.rows.slice(0, maxBodyRows);
|
|
9055
|
+
const hidden = snapshot.rows.length - shownRows.length;
|
|
9056
|
+
const updated = timeOnly(snapshot.stats.updatedAt);
|
|
9057
|
+
const statLine = [
|
|
9058
|
+
`active loops ${snapshot.stats.activeLoops}`,
|
|
9059
|
+
`running runs ${snapshot.stats.runningRuns}`,
|
|
9060
|
+
`failed runs ${snapshot.stats.failedRuns}`,
|
|
9061
|
+
`paused ${snapshot.stats.pausedLoops}`,
|
|
9062
|
+
`refresh ${durationLabel(refreshMs) || `${refreshMs}ms`}`,
|
|
9063
|
+
`updated ${updated}`
|
|
9064
|
+
].join(" | ");
|
|
9065
|
+
const lines = [
|
|
9066
|
+
paint(`OpenLoops`, `${BOLD}${CYAN}`, color) + paint(" live loops", DIM, color),
|
|
9067
|
+
paint(fitLine(statLine, columns), DIM, color),
|
|
9068
|
+
"",
|
|
9069
|
+
paint(tableHeader(widths, compact2), `${BOLD}${CYAN}`, color),
|
|
9070
|
+
paint("-".repeat(Math.min(columns, tableWidth(widths))), DIM, color)
|
|
9071
|
+
];
|
|
9072
|
+
for (const row of shownRows) {
|
|
9073
|
+
lines.push(tableRow(row, widths, color));
|
|
9074
|
+
}
|
|
9075
|
+
if (snapshot.rows.length === 0) {
|
|
9076
|
+
lines.push(paint("No active loops.", DIM, color));
|
|
9077
|
+
} else if (hidden > 0) {
|
|
9078
|
+
lines.push(paint(`showing ${shownRows.length} of ${snapshot.rows.length} active loops`, DIM, color));
|
|
9079
|
+
}
|
|
9080
|
+
lines.push("");
|
|
9081
|
+
lines.push(paint("q quit | Ctrl-C exit", DIM, color));
|
|
9082
|
+
return `${lines.map((line) => fitLine(line, columns, color)).join(`
|
|
9083
|
+
`)}
|
|
9084
|
+
`;
|
|
9085
|
+
}
|
|
9086
|
+
async function runLoopsUiApp(opts = {}) {
|
|
9087
|
+
const refreshMs = Math.max(MIN_REFRESH_MS, opts.refreshMs ?? DEFAULT_REFRESH_MS);
|
|
9088
|
+
const input = opts.input ?? process.stdin;
|
|
9089
|
+
const output = opts.output ?? process.stdout;
|
|
9090
|
+
const storeFactory = opts.storeFactory ?? (() => new Store);
|
|
9091
|
+
const rawInput = input;
|
|
9092
|
+
const signalExitCodes = { SIGINT: 130, SIGTERM: 143 };
|
|
9093
|
+
const signalHandlers = [];
|
|
9094
|
+
let interval;
|
|
9095
|
+
let closed = false;
|
|
9096
|
+
let cleaned = false;
|
|
9097
|
+
let rawModeEnabled = false;
|
|
9098
|
+
let terminalEntered = false;
|
|
9099
|
+
let store;
|
|
9100
|
+
let stopApp;
|
|
9101
|
+
let renderHandler;
|
|
9102
|
+
const render = () => {
|
|
9103
|
+
if (closed || !store)
|
|
9104
|
+
return;
|
|
9105
|
+
const snapshot = buildLoopUiSnapshot(store);
|
|
9106
|
+
output.write(`${CLEAR_SCREEN}${renderLoopUiFrame(snapshot, {
|
|
9107
|
+
columns: output.columns,
|
|
9108
|
+
rows: output.rows,
|
|
9109
|
+
refreshMs,
|
|
9110
|
+
color: true
|
|
9111
|
+
})}`);
|
|
9112
|
+
};
|
|
9113
|
+
const onData = (chunk) => {
|
|
9114
|
+
const value = chunk.toString("utf8");
|
|
9115
|
+
if (value.toLowerCase().includes("q") || value.includes("\x03") || value.includes("\x1B"))
|
|
9116
|
+
stopApp?.();
|
|
9117
|
+
};
|
|
9118
|
+
const cleanup = () => {
|
|
9119
|
+
if (cleaned)
|
|
9120
|
+
return;
|
|
9121
|
+
cleaned = true;
|
|
9122
|
+
if (interval)
|
|
9123
|
+
clearInterval(interval);
|
|
9124
|
+
input.off("data", onData);
|
|
9125
|
+
if (renderHandler)
|
|
9126
|
+
output.off("resize", renderHandler);
|
|
9127
|
+
for (const [signal, handler] of signalHandlers)
|
|
9128
|
+
process.removeListener(signal, handler);
|
|
9129
|
+
if (rawModeEnabled)
|
|
9130
|
+
rawInput.setRawMode?.(false);
|
|
9131
|
+
if (terminalEntered) {
|
|
9132
|
+
try {
|
|
9133
|
+
output.write(`${SHOW_CURSOR}${CLEAR_SCREEN}${EXIT_ALT_SCREEN}`);
|
|
9134
|
+
} catch {}
|
|
9135
|
+
}
|
|
9136
|
+
try {
|
|
9137
|
+
store?.close();
|
|
9138
|
+
} catch {}
|
|
9139
|
+
};
|
|
9140
|
+
try {
|
|
9141
|
+
store = storeFactory();
|
|
9142
|
+
output.write(`${ENTER_ALT_SCREEN}${HIDE_CURSOR}${CLEAR_SCREEN}`);
|
|
9143
|
+
terminalEntered = true;
|
|
9144
|
+
await new Promise((resolve3, reject) => {
|
|
9145
|
+
const fail = (error) => {
|
|
9146
|
+
if (closed)
|
|
9147
|
+
return;
|
|
9148
|
+
closed = true;
|
|
9149
|
+
cleanup();
|
|
9150
|
+
reject(error);
|
|
9151
|
+
};
|
|
9152
|
+
const stop = () => {
|
|
9153
|
+
if (closed)
|
|
9154
|
+
return;
|
|
9155
|
+
closed = true;
|
|
9156
|
+
cleanup();
|
|
9157
|
+
resolve3();
|
|
9158
|
+
};
|
|
9159
|
+
const safeRender = () => {
|
|
9160
|
+
try {
|
|
9161
|
+
render();
|
|
9162
|
+
} catch (error) {
|
|
9163
|
+
fail(error);
|
|
9164
|
+
}
|
|
9165
|
+
};
|
|
9166
|
+
stopApp = stop;
|
|
9167
|
+
renderHandler = safeRender;
|
|
9168
|
+
for (const signal of ["SIGINT", "SIGTERM"]) {
|
|
9169
|
+
const handler = () => {
|
|
9170
|
+
process.exitCode = signalExitCodes[signal] ?? 1;
|
|
9171
|
+
stop();
|
|
9172
|
+
};
|
|
9173
|
+
signalHandlers.push([signal, handler]);
|
|
9174
|
+
process.once(signal, handler);
|
|
9175
|
+
}
|
|
9176
|
+
if (typeof rawInput.setRawMode === "function") {
|
|
9177
|
+
rawInput.setRawMode(true);
|
|
9178
|
+
rawModeEnabled = true;
|
|
9179
|
+
}
|
|
9180
|
+
input.resume();
|
|
9181
|
+
input.on("data", onData);
|
|
9182
|
+
output.on("resize", safeRender);
|
|
9183
|
+
safeRender();
|
|
9184
|
+
if (!closed)
|
|
9185
|
+
interval = setInterval(safeRender, refreshMs);
|
|
9186
|
+
});
|
|
9187
|
+
} finally {
|
|
9188
|
+
cleanup();
|
|
9189
|
+
}
|
|
9190
|
+
}
|
|
9191
|
+
function tableWidths(columns, compact2) {
|
|
9192
|
+
const widths = {
|
|
9193
|
+
name: 20,
|
|
9194
|
+
status: compact2 ? 6 : 7,
|
|
9195
|
+
cadence: compact2 ? 10 : 14,
|
|
9196
|
+
nextRun: compact2 ? 8 : 10,
|
|
9197
|
+
lastRunOutcome: compact2 ? 8 : 11,
|
|
9198
|
+
provider: compact2 ? 7 : 10,
|
|
9199
|
+
activeRuns: compact2 ? 3 : 11
|
|
9200
|
+
};
|
|
9201
|
+
const fixed = widths.status + widths.cadence + widths.nextRun + widths.lastRunOutcome + widths.provider + widths.activeRuns + 12;
|
|
9202
|
+
widths.name = Math.max(6, columns - fixed);
|
|
9203
|
+
return widths;
|
|
9204
|
+
}
|
|
9205
|
+
function tableWidth(widths) {
|
|
9206
|
+
return widths.name + widths.status + widths.cadence + widths.nextRun + widths.lastRunOutcome + widths.provider + widths.activeRuns + 12;
|
|
9207
|
+
}
|
|
9208
|
+
function tableHeader(widths, compact2) {
|
|
9209
|
+
return [
|
|
9210
|
+
cell("NAME", widths.name),
|
|
9211
|
+
cell("STATUS", widths.status),
|
|
9212
|
+
cell("CADENCE", widths.cadence),
|
|
9213
|
+
cell("NEXT-RUN", widths.nextRun),
|
|
9214
|
+
cell("LAST-RUN", widths.lastRunOutcome),
|
|
9215
|
+
cell("PROVIDER", widths.provider),
|
|
9216
|
+
cell(compact2 ? "RUN" : "ACTIVE-RUNS", widths.activeRuns, "right")
|
|
9217
|
+
].join(" ");
|
|
9218
|
+
}
|
|
9219
|
+
function tableRow(row, widths, color) {
|
|
9220
|
+
return [
|
|
9221
|
+
cell(row.name, widths.name),
|
|
9222
|
+
paint(cell(row.status, widths.status), statusColor(row.status), color),
|
|
9223
|
+
cell(row.cadence, widths.cadence),
|
|
9224
|
+
cell(row.nextRun, widths.nextRun),
|
|
9225
|
+
paint(cell(row.lastRunOutcome, widths.lastRunOutcome), outcomeColor(row.lastRunOutcome), color),
|
|
9226
|
+
cell(row.provider, widths.provider),
|
|
9227
|
+
cell(String(row.activeRuns), widths.activeRuns, "right")
|
|
9228
|
+
].join(" ");
|
|
9229
|
+
}
|
|
9230
|
+
function cell(value, width, align = "left") {
|
|
9231
|
+
const clipped = clip(value, width);
|
|
9232
|
+
return align === "right" ? clipped.padStart(width) : clipped.padEnd(width);
|
|
9233
|
+
}
|
|
9234
|
+
function clip(value, width) {
|
|
9235
|
+
if (width <= 0)
|
|
9236
|
+
return "";
|
|
9237
|
+
if (value.length <= width)
|
|
9238
|
+
return value;
|
|
9239
|
+
if (width === 1)
|
|
9240
|
+
return "~";
|
|
9241
|
+
return `${value.slice(0, width - 1)}~`;
|
|
9242
|
+
}
|
|
9243
|
+
function fitLine(value, columns, hasAnsi = false) {
|
|
9244
|
+
if (!hasAnsi)
|
|
9245
|
+
return clip(value, columns).trimEnd();
|
|
9246
|
+
let visible = 0;
|
|
9247
|
+
let index = 0;
|
|
9248
|
+
let out = "";
|
|
9249
|
+
while (index < value.length) {
|
|
9250
|
+
const char = value[index];
|
|
9251
|
+
if (char === "\x1B" && value[index + 1] === "[") {
|
|
9252
|
+
let end = index + 2;
|
|
9253
|
+
while (end < value.length && !/[A-Za-z]/.test(value[end]))
|
|
9254
|
+
end += 1;
|
|
9255
|
+
out += value.slice(index, Math.min(end + 1, value.length));
|
|
9256
|
+
index = end + 1;
|
|
9257
|
+
continue;
|
|
9258
|
+
}
|
|
9259
|
+
if (visible < columns) {
|
|
9260
|
+
out += char;
|
|
9261
|
+
visible += 1;
|
|
9262
|
+
}
|
|
9263
|
+
index += 1;
|
|
9264
|
+
}
|
|
9265
|
+
return out;
|
|
9266
|
+
}
|
|
9267
|
+
function paint(value, code, enabled) {
|
|
9268
|
+
return enabled ? `${code}${value}${RESET}` : value;
|
|
9269
|
+
}
|
|
9270
|
+
function statusColor(status) {
|
|
9271
|
+
if (status === "active")
|
|
9272
|
+
return GREEN;
|
|
9273
|
+
if (status === "paused")
|
|
9274
|
+
return YELLOW;
|
|
9275
|
+
if (status === "expired")
|
|
9276
|
+
return MAGENTA;
|
|
9277
|
+
return DIM;
|
|
9278
|
+
}
|
|
9279
|
+
function outcomeColor(outcome) {
|
|
9280
|
+
if (outcome.startsWith("succeeded"))
|
|
9281
|
+
return GREEN;
|
|
9282
|
+
if (outcome.startsWith("running"))
|
|
9283
|
+
return CYAN;
|
|
9284
|
+
if (outcome.startsWith("failed") || outcome.startsWith("timed_out") || outcome.startsWith("abandoned"))
|
|
9285
|
+
return RED;
|
|
9286
|
+
if (outcome.startsWith("skipped"))
|
|
9287
|
+
return YELLOW;
|
|
9288
|
+
return DIM;
|
|
9289
|
+
}
|
|
9290
|
+
function runOutcomeLabel(run) {
|
|
9291
|
+
if (!run)
|
|
9292
|
+
return "-";
|
|
9293
|
+
if (run.status === "failed" && run.exitCode !== undefined)
|
|
9294
|
+
return `failed(${run.exitCode})`;
|
|
9295
|
+
return run.status;
|
|
9296
|
+
}
|
|
9297
|
+
function providerLabel(target) {
|
|
9298
|
+
if (target.type === "command")
|
|
9299
|
+
return "command";
|
|
9300
|
+
if (target.type === "agent")
|
|
9301
|
+
return target.provider;
|
|
9302
|
+
return "workflow";
|
|
9303
|
+
}
|
|
9304
|
+
function scheduleLabel(schedule) {
|
|
9305
|
+
if (schedule.type === "once")
|
|
9306
|
+
return "once";
|
|
9307
|
+
if (schedule.type === "interval")
|
|
9308
|
+
return `every:${durationLabel(schedule.everyMs) || `${schedule.everyMs}ms`}`;
|
|
9309
|
+
if (schedule.type === "cron")
|
|
9310
|
+
return `cron:${schedule.expression}`;
|
|
9311
|
+
return schedule.minIntervalMs ? `dynamic:${durationLabel(schedule.minIntervalMs) || `${schedule.minIntervalMs}ms`}` : "dynamic";
|
|
9312
|
+
}
|
|
9313
|
+
function nextRunLabel(nextRunAt, now) {
|
|
9314
|
+
if (!nextRunAt)
|
|
9315
|
+
return "-";
|
|
9316
|
+
const date = new Date(nextRunAt);
|
|
9317
|
+
if (Number.isNaN(date.getTime()))
|
|
9318
|
+
return "invalid";
|
|
9319
|
+
return relativeTime(date.getTime() - now.getTime());
|
|
9320
|
+
}
|
|
9321
|
+
function relativeTime(deltaMs) {
|
|
9322
|
+
const past = deltaMs < 0;
|
|
9323
|
+
const abs = Math.abs(deltaMs);
|
|
9324
|
+
const value = durationLabel(abs);
|
|
9325
|
+
if (!value || value === "0ms")
|
|
9326
|
+
return "now";
|
|
9327
|
+
return past ? `${value} ago` : `in ${value}`;
|
|
9328
|
+
}
|
|
9329
|
+
function durationLabel(ms) {
|
|
9330
|
+
if (ms === undefined || !Number.isFinite(ms))
|
|
9331
|
+
return "";
|
|
9332
|
+
if (ms < 1000)
|
|
9333
|
+
return `${Math.max(0, Math.round(ms))}ms`;
|
|
9334
|
+
const units = [
|
|
9335
|
+
[7 * 24 * 60 * 60 * 1000, "w"],
|
|
9336
|
+
[24 * 60 * 60 * 1000, "d"],
|
|
9337
|
+
[60 * 60 * 1000, "h"],
|
|
9338
|
+
[60 * 1000, "m"],
|
|
9339
|
+
[1000, "s"]
|
|
9340
|
+
];
|
|
9341
|
+
for (const [unitMs, label] of units) {
|
|
9342
|
+
const value = ms / unitMs;
|
|
9343
|
+
if (value >= 1)
|
|
9344
|
+
return `${Math.round(value)}${label}`;
|
|
9345
|
+
}
|
|
9346
|
+
return `${ms}ms`;
|
|
9347
|
+
}
|
|
9348
|
+
function timeOnly(value) {
|
|
9349
|
+
const date = new Date(value);
|
|
9350
|
+
if (Number.isNaN(date.getTime()))
|
|
9351
|
+
return value;
|
|
9352
|
+
return date.toISOString().slice(11, 19);
|
|
9353
|
+
}
|
|
9354
|
+
|
|
8180
9355
|
// src/lib/migration.ts
|
|
8181
9356
|
import { createHash as createHash3 } from "crypto";
|
|
8182
9357
|
import { hostname as hostname3 } from "os";
|
|
@@ -8688,7 +9863,7 @@ function selfHostedControlPlaneSummary(env = process.env) {
|
|
|
8688
9863
|
|
|
8689
9864
|
// src/lib/hygiene.ts
|
|
8690
9865
|
import { basename as basename3 } from "path";
|
|
8691
|
-
import { homedir as
|
|
9866
|
+
import { homedir as homedir4 } from "os";
|
|
8692
9867
|
var PROVIDER_TOKENS = new Set([
|
|
8693
9868
|
"codewith",
|
|
8694
9869
|
"claude",
|
|
@@ -8704,7 +9879,7 @@ var REPO_GENERIC_TOKENS = new Set(["repo", "repoops"]);
|
|
|
8704
9879
|
var CADENCE_SUFFIX_TOKENS = new Set(["hourly", "daily", "weekly", "monthly"]);
|
|
8705
9880
|
var CADENCE_SUFFIX_PATTERN = /^(?:every-?)?\d+(?:s|m|h|d|w)$/;
|
|
8706
9881
|
function userHome() {
|
|
8707
|
-
return process.env.HOME ||
|
|
9882
|
+
return process.env.HOME || homedir4();
|
|
8708
9883
|
}
|
|
8709
9884
|
function slugify(value) {
|
|
8710
9885
|
return value.normalize("NFKD").replace(/[^\w\s.-]/g, "-").replace(/[_\s.:/]+/g, "-").replace(/[^a-zA-Z0-9-]+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").toLowerCase();
|
|
@@ -8942,14 +10117,14 @@ function buildScriptInventoryReport(store, opts = {}) {
|
|
|
8942
10117
|
// src/lib/templates.ts
|
|
8943
10118
|
import { execFileSync } from "child_process";
|
|
8944
10119
|
import { existsSync as existsSync7 } from "fs";
|
|
8945
|
-
import { homedir as
|
|
8946
|
-
import { basename as basename4, isAbsolute as isAbsolute2, join as
|
|
10120
|
+
import { homedir as homedir5 } from "os";
|
|
10121
|
+
import { basename as basename4, isAbsolute as isAbsolute2, join as join9, relative, resolve as resolve4 } from "path";
|
|
8947
10122
|
|
|
8948
10123
|
// src/lib/template-kit.ts
|
|
8949
10124
|
var TODOS_TASK_WORKER_VERIFIER_TEMPLATE_ID = "todos-task-worker-verifier";
|
|
8950
10125
|
var EVENT_WORKER_VERIFIER_TEMPLATE_ID = "event-worker-verifier";
|
|
8951
10126
|
var BOUNDED_AGENT_WORKER_VERIFIER_TEMPLATE_ID = "bounded-agent-worker-verifier";
|
|
8952
|
-
var
|
|
10127
|
+
var TASK_LIFECYCLE_TEMPLATE_ID2 = "task-lifecycle";
|
|
8953
10128
|
var PR_REVIEW_TEMPLATE_ID = "pr-review";
|
|
8954
10129
|
var SCHEDULED_AUDIT_TEMPLATE_ID = "scheduled-audit";
|
|
8955
10130
|
var KNOWLEDGE_REFRESH_TEMPLATE_ID = "knowledge-refresh";
|
|
@@ -9056,7 +10231,7 @@ var BUILTIN_TEMPLATE_SUMMARIES = [
|
|
|
9056
10231
|
]
|
|
9057
10232
|
},
|
|
9058
10233
|
{
|
|
9059
|
-
id:
|
|
10234
|
+
id: TASK_LIFECYCLE_TEMPLATE_ID2,
|
|
9060
10235
|
name: "Task Lifecycle",
|
|
9061
10236
|
description: "Run the standard task-created lifecycle: triage/dedupe, plan, worker execution, independent verification, and todos closure/follow-up evidence.",
|
|
9062
10237
|
kind: "workflow",
|
|
@@ -9476,6 +10651,50 @@ var PR_HANDOFF_SCRIPT = [
|
|
|
9476
10651
|
"console.log(`PR handoff complete: ${finalPrUrl}`);"
|
|
9477
10652
|
].join(`
|
|
9478
10653
|
`);
|
|
10654
|
+
var PR_HANDOFF_NO_ARTIFACT_SCRIPT = [
|
|
10655
|
+
"const { spawnSync } = await import('node:child_process');",
|
|
10656
|
+
"const artifactPath = process.env.OPENLOOPS_PR_HANDOFF_ARTIFACT || '';",
|
|
10657
|
+
"const taskId = process.env.OPENLOOPS_PR_HANDOFF_TASK_ID || '';",
|
|
10658
|
+
"const todosProject = process.env.OPENLOOPS_PR_HANDOFF_TODOS_PROJECT || '';",
|
|
10659
|
+
"const worktree = process.env.OPENLOOPS_PR_HANDOFF_WORKTREE || process.cwd();",
|
|
10660
|
+
"const expectedBranch = process.env.OPENLOOPS_PR_HANDOFF_EXPECTED_BRANCH || '';",
|
|
10661
|
+
"const todosBin = process.env.OPENLOOPS_PR_HANDOFF_TODOS_BIN || 'todos';",
|
|
10662
|
+
"const gitBin = process.env.OPENLOOPS_PR_HANDOFF_GIT_BIN || 'git';",
|
|
10663
|
+
"const ghBin = process.env.OPENLOOPS_PR_HANDOFF_GH_BIN || 'gh';",
|
|
10664
|
+
"process.stdout.write(`no PR handoff artifact at ${artifactPath}\\n`);",
|
|
10665
|
+
"const run = (command, args, options = {}) => {",
|
|
10666
|
+
" try { return spawnSync(command, args, { encoding: 'utf8', ...options }); }",
|
|
10667
|
+
" catch (error) { return { status: 1, stdout: '', stderr: String((error && error.message) || error) }; }",
|
|
10668
|
+
"};",
|
|
10669
|
+
"const todosArgs = (...args) => todosProject ? ['--project', todosProject, ...args] : args;",
|
|
10670
|
+
"const comment = (text) => {",
|
|
10671
|
+
" const result = run(todosBin, todosArgs('comment', taskId, text));",
|
|
10672
|
+
" if (result.status !== 0) console.error(`failed to comment original task: ${result.stderr || result.stdout || result.status}`);",
|
|
10673
|
+
"};",
|
|
10674
|
+
"const main = () => {",
|
|
10675
|
+
" let branch = expectedBranch;",
|
|
10676
|
+
" if (!branch) {",
|
|
10677
|
+
" const shown = run(gitBin, ['-C', worktree, 'branch', '--show-current']);",
|
|
10678
|
+
" branch = String((shown.status === 0 ? shown.stdout : '') || '').trim();",
|
|
10679
|
+
" }",
|
|
10680
|
+
" if (!branch) { console.log('pr-handoff: no artifact and no resolvable branch; nothing to hand off'); return; }",
|
|
10681
|
+
" const listed = run(ghBin, ['pr', 'list', '--head', branch, '--state', 'open', '--json', 'url,number,headRefName,headRefOid'], { cwd: worktree });",
|
|
10682
|
+
" if (listed.status !== 0) { console.log(`pr-handoff: no artifact; PR lookup failed for branch ${branch}: ${String(listed.stderr || listed.stdout || listed.status).slice(0, 300)}`); return; }",
|
|
10683
|
+
" let prs = [];",
|
|
10684
|
+
" try { prs = JSON.parse(String(listed.stdout || '[]')); } catch { prs = []; }",
|
|
10685
|
+
" const pr = Array.isArray(prs) ? prs.find((entry) => entry && entry.headRefName === branch && typeof entry.url === 'string' && entry.url) : undefined;",
|
|
10686
|
+
" if (!pr) { console.log(`pr-handoff: no artifact and no open PR for branch ${branch}; worker completed without opening a PR`); return; }",
|
|
10687
|
+
" let commit = String(pr.headRefOid || '').trim();",
|
|
10688
|
+
" if (!commit) {",
|
|
10689
|
+
" const head = run(gitBin, ['-C', worktree, 'rev-parse', 'HEAD']);",
|
|
10690
|
+
" commit = String((head.status === 0 ? head.stdout : '') || '').trim();",
|
|
10691
|
+
" }",
|
|
10692
|
+
" comment(`openloops:pr-handoff=done task=${taskId} pr=${pr.url} commit=${commit || 'unknown'} branch=${branch}`);",
|
|
10693
|
+
" console.log(`PR handoff complete (worker-opened PR): ${pr.url}`);",
|
|
10694
|
+
"};",
|
|
10695
|
+
"try { main(); } catch (error) { console.error(`pr-handoff no-artifact detection error (ignored): ${String((error && error.message) || error)}`); }"
|
|
10696
|
+
].join(`
|
|
10697
|
+
`);
|
|
9479
10698
|
function prHandoffCommand(opts) {
|
|
9480
10699
|
return [
|
|
9481
10700
|
"set -euo pipefail",
|
|
@@ -9486,12 +10705,14 @@ function prHandoffCommand(opts) {
|
|
|
9486
10705
|
`export OPENLOOPS_PR_HANDOFF_WORKTREE_ROOT=${shellQuote3(opts.worktreeRoot)}`,
|
|
9487
10706
|
`export OPENLOOPS_PR_HANDOFF_EXPECTED_BRANCH=${shellQuote3(opts.expectedBranch)}`,
|
|
9488
10707
|
'if [ ! -s "$OPENLOOPS_PR_HANDOFF_ARTIFACT" ]; then',
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
"
|
|
10708
|
+
"bun - <<'OPENLOOPS_PR_HANDOFF_NOARTIFACT'",
|
|
10709
|
+
PR_HANDOFF_NO_ARTIFACT_SCRIPT,
|
|
10710
|
+
"OPENLOOPS_PR_HANDOFF_NOARTIFACT",
|
|
10711
|
+
"else",
|
|
9492
10712
|
"bun - <<'BUN'",
|
|
9493
10713
|
PR_HANDOFF_SCRIPT,
|
|
9494
|
-
"BUN"
|
|
10714
|
+
"BUN",
|
|
10715
|
+
"fi"
|
|
9495
10716
|
].join(`
|
|
9496
10717
|
`);
|
|
9497
10718
|
}
|
|
@@ -9569,8 +10790,8 @@ function assignPoolAuthProfiles(input) {
|
|
|
9569
10790
|
return { profiles, deferred: false, minLoad: Number.isFinite(minLoad) ? minLoad : 0 };
|
|
9570
10791
|
}
|
|
9571
10792
|
// src/lib/templates-custom.ts
|
|
9572
|
-
import { existsSync as existsSync6, lstatSync as lstatSync2, mkdirSync as
|
|
9573
|
-
import { join as
|
|
10793
|
+
import { existsSync as existsSync6, lstatSync as lstatSync2, mkdirSync as mkdirSync8, readdirSync, readFileSync as readFileSync6, writeFileSync as writeFileSync5 } from "fs";
|
|
10794
|
+
import { join as join8, resolve as resolve3 } from "path";
|
|
9574
10795
|
var CUSTOM_TEMPLATE_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}$/;
|
|
9575
10796
|
var CUSTOM_TEMPLATE_VARIABLE_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
9576
10797
|
var CUSTOM_TEMPLATE_VARIABLE_TYPES = new Set(["string", "number", "boolean", "json", "string[]"]);
|
|
@@ -9597,11 +10818,11 @@ function assertTemplateKind(value, label) {
|
|
|
9597
10818
|
return kind;
|
|
9598
10819
|
}
|
|
9599
10820
|
function customLoopTemplatesDir() {
|
|
9600
|
-
return
|
|
10821
|
+
return join8(dataDir(), "templates");
|
|
9601
10822
|
}
|
|
9602
10823
|
function ensureCustomLoopTemplatesDir() {
|
|
9603
10824
|
const dir = customLoopTemplatesDir();
|
|
9604
|
-
|
|
10825
|
+
mkdirSync8(dir, { recursive: true, mode: 448 });
|
|
9605
10826
|
return dir;
|
|
9606
10827
|
}
|
|
9607
10828
|
function validateCustomTemplateId(id, label) {
|
|
@@ -9732,7 +10953,7 @@ function customTemplateSummary(definition, sourcePath) {
|
|
|
9732
10953
|
function readCustomTemplateFile(file) {
|
|
9733
10954
|
let parsed;
|
|
9734
10955
|
try {
|
|
9735
|
-
parsed = JSON.parse(
|
|
10956
|
+
parsed = JSON.parse(readFileSync6(file, "utf8"));
|
|
9736
10957
|
} catch (error) {
|
|
9737
10958
|
const message = error instanceof Error ? error.message : String(error);
|
|
9738
10959
|
throw new Error(`failed to read custom template ${file}: ${message}`);
|
|
@@ -9760,7 +10981,7 @@ function loadCustomLoopTemplatesRaw() {
|
|
|
9760
10981
|
if (!existsSync6(dir))
|
|
9761
10982
|
return [];
|
|
9762
10983
|
return readdirSync(dir, { withFileTypes: true }).filter((entry) => entry.name.endsWith(".json")).sort((left, right) => left.name.localeCompare(right.name)).map((entry) => {
|
|
9763
|
-
const file =
|
|
10984
|
+
const file = join8(dir, entry.name);
|
|
9764
10985
|
if (entry.isSymbolicLink())
|
|
9765
10986
|
throw new Error(`refusing symlinked custom template file: ${file}`);
|
|
9766
10987
|
if (!entry.isFile())
|
|
@@ -9880,7 +11101,7 @@ function importCustomLoopTemplate(file, reservedKeys, opts = {}) {
|
|
|
9880
11101
|
const source = resolve3(file);
|
|
9881
11102
|
const entry = readCustomTemplateFile(source);
|
|
9882
11103
|
const dir = ensureCustomLoopTemplatesDir();
|
|
9883
|
-
const destination =
|
|
11104
|
+
const destination = join8(dir, `${entry.definition.id}.json`);
|
|
9884
11105
|
const replaced = existsSync6(destination);
|
|
9885
11106
|
const existing = loadCustomLoopTemplatesRaw().filter((template) => resolve3(template.path) !== resolve3(destination));
|
|
9886
11107
|
assertNoTemplateCollisions([...existing, { ...entry, path: destination, summary: customTemplateSummary(entry.definition, destination) }], reservedKeys);
|
|
@@ -9891,7 +11112,7 @@ function importCustomLoopTemplate(file, reservedKeys, opts = {}) {
|
|
|
9891
11112
|
if (!opts.replace)
|
|
9892
11113
|
throw new Error(`custom template already exists: ${entry.definition.id}; use --replace to overwrite it`);
|
|
9893
11114
|
}
|
|
9894
|
-
|
|
11115
|
+
writeFileSync5(destination, `${JSON.stringify(entry.definition, null, 2)}
|
|
9895
11116
|
`, { mode: 384 });
|
|
9896
11117
|
const imported = readCustomTemplateFile(destination);
|
|
9897
11118
|
return { template: structuredClone(imported.summary), path: destination, replaced };
|
|
@@ -10009,10 +11230,10 @@ function normalizeWorktreeMode(mode) {
|
|
|
10009
11230
|
}
|
|
10010
11231
|
function defaultWorktreeRoot(root) {
|
|
10011
11232
|
if (root?.trim()) {
|
|
10012
|
-
const expanded = root.trim().replace(/^~(?=$|\/)/,
|
|
11233
|
+
const expanded = root.trim().replace(/^~(?=$|\/)/, homedir5());
|
|
10013
11234
|
return isAbsolute2(expanded) ? expanded : resolve4(expanded);
|
|
10014
11235
|
}
|
|
10015
|
-
return
|
|
11236
|
+
return join9(homedir5(), ".hasna", "loops", "worktrees");
|
|
10016
11237
|
}
|
|
10017
11238
|
function gitRootFor(path) {
|
|
10018
11239
|
if (!existsSync7(path))
|
|
@@ -10069,9 +11290,9 @@ function worktreePlan(input, seed) {
|
|
|
10069
11290
|
const root = defaultWorktreeRoot(input.worktreeRoot);
|
|
10070
11291
|
const repoSlug = slugSegment(basename4(repoRoot), "repo");
|
|
10071
11292
|
const seedSlug = `${slugSegment(seed, "run").slice(0, 48)}-${stableHex(`${repoRoot}:${seed}`)}`;
|
|
10072
|
-
const worktreePath =
|
|
11293
|
+
const worktreePath = join9(root, repoSlug, seedSlug);
|
|
10073
11294
|
const relativeCwd = relative(repoRoot, originalCwd);
|
|
10074
|
-
const cwd = relativeCwd && !relativeCwd.startsWith("..") && !isAbsolute2(relativeCwd) ?
|
|
11295
|
+
const cwd = relativeCwd && !relativeCwd.startsWith("..") && !isAbsolute2(relativeCwd) ? join9(worktreePath, relativeCwd) : worktreePath;
|
|
10075
11296
|
const branchPrefix = (input.worktreeBranchPrefix?.trim() || "openloops").replace(/^\/+|\/+$/g, "") || "openloops";
|
|
10076
11297
|
const branch = `${branchPrefix}/${repoSlug}/${seedSlug}`;
|
|
10077
11298
|
return {
|
|
@@ -10190,7 +11411,7 @@ function lifecycleGateStep(opts) {
|
|
|
10190
11411
|
});
|
|
10191
11412
|
}
|
|
10192
11413
|
function prHandoffArtifactPath(plan, taskId) {
|
|
10193
|
-
return
|
|
11414
|
+
return join9(plan.cwd, ".openloops", "pr-handoff", `${slugSegment(taskId, "task")}.json`);
|
|
10194
11415
|
}
|
|
10195
11416
|
function prHandoffStep(input, plan, todosProjectPath) {
|
|
10196
11417
|
return commandStep({
|
|
@@ -10707,7 +11928,7 @@ var BOUNDED_LIFECYCLE_TEMPLATES = {
|
|
|
10707
11928
|
};
|
|
10708
11929
|
function renderLifecycleBoundedTemplate(id, values) {
|
|
10709
11930
|
const base = agentTemplateInput(values);
|
|
10710
|
-
if (id ===
|
|
11931
|
+
if (id === TASK_LIFECYCLE_TEMPLATE_ID2) {
|
|
10711
11932
|
const taskId = values.taskId ?? "";
|
|
10712
11933
|
if (!taskId.trim())
|
|
10713
11934
|
throw new Error("taskId is required");
|
|
@@ -10823,8 +12044,8 @@ function renderLoopTemplate(id, values, opts = {}) {
|
|
|
10823
12044
|
|
|
10824
12045
|
// src/lib/backup.ts
|
|
10825
12046
|
import { Database as Database2 } from "bun:sqlite";
|
|
10826
|
-
import { chmodSync as chmodSync3, existsSync as existsSync8, mkdirSync as
|
|
10827
|
-
import { dirname as dirname6, join as
|
|
12047
|
+
import { chmodSync as chmodSync3, existsSync as existsSync8, mkdirSync as mkdirSync9, readdirSync as readdirSync2, rmSync as rmSync6 } from "fs";
|
|
12048
|
+
import { dirname as dirname6, join as join10 } from "path";
|
|
10828
12049
|
var DEFAULT_KEEP = 3;
|
|
10829
12050
|
var DEBOUNCE_MS = 60 * 60 * 1000;
|
|
10830
12051
|
function reasonSlug(reason) {
|
|
@@ -10855,10 +12076,10 @@ function backupDatabase(opts) {
|
|
|
10855
12076
|
if (!existsSync8(file)) {
|
|
10856
12077
|
return { skipped: true, skipReason: `database file not found: ${file}`, prunedPaths: [] };
|
|
10857
12078
|
}
|
|
10858
|
-
const dir = opts.backupsDir ??
|
|
12079
|
+
const dir = opts.backupsDir ?? join10(dirname6(file), "backups");
|
|
10859
12080
|
const slug = reasonSlug(opts.reason);
|
|
10860
12081
|
const now = opts.now ?? new Date;
|
|
10861
|
-
|
|
12082
|
+
mkdirSync9(dir, { recursive: true, mode: 448 });
|
|
10862
12083
|
const existing = listBackups(dir, slug);
|
|
10863
12084
|
const newest = existing[0];
|
|
10864
12085
|
if (newest && now.getTime() - newest.timeMs < DEBOUNCE_MS) {
|
|
@@ -10868,12 +12089,12 @@ function backupDatabase(opts) {
|
|
|
10868
12089
|
prunedPaths: []
|
|
10869
12090
|
};
|
|
10870
12091
|
}
|
|
10871
|
-
const target =
|
|
12092
|
+
const target = join10(dir, `loops-${slug}-${backupStamp(now)}.db`);
|
|
10872
12093
|
const db = new Database2(file, { readonly: true });
|
|
10873
12094
|
try {
|
|
10874
12095
|
db.query("VACUUM INTO ?").run(target);
|
|
10875
12096
|
} catch (error) {
|
|
10876
|
-
|
|
12097
|
+
rmSync6(target, { force: true });
|
|
10877
12098
|
throw error;
|
|
10878
12099
|
} finally {
|
|
10879
12100
|
db.close();
|
|
@@ -10881,8 +12102,8 @@ function backupDatabase(opts) {
|
|
|
10881
12102
|
chmodSync3(target, 384);
|
|
10882
12103
|
const prunedPaths = [];
|
|
10883
12104
|
for (const entry of listBackups(dir, slug).slice(keep)) {
|
|
10884
|
-
const path =
|
|
10885
|
-
|
|
12105
|
+
const path = join10(dir, entry.name);
|
|
12106
|
+
rmSync6(path, { force: true });
|
|
10886
12107
|
prunedPaths.push(path);
|
|
10887
12108
|
}
|
|
10888
12109
|
return { path: target, skipped: false, prunedPaths };
|
|
@@ -10987,6 +12208,14 @@ function taskEventField(data, keys) {
|
|
|
10987
12208
|
if (direct)
|
|
10988
12209
|
return direct;
|
|
10989
12210
|
}
|
|
12211
|
+
const payloadTask = payload.task;
|
|
12212
|
+
if (payloadTask && typeof payloadTask === "object" && !Array.isArray(payloadTask)) {
|
|
12213
|
+
for (const key of keys) {
|
|
12214
|
+
const direct = stringField(payloadTask[key]);
|
|
12215
|
+
if (direct)
|
|
12216
|
+
return direct;
|
|
12217
|
+
}
|
|
12218
|
+
}
|
|
10990
12219
|
}
|
|
10991
12220
|
return;
|
|
10992
12221
|
}
|
|
@@ -11199,17 +12428,17 @@ function routeFieldList(records, fields) {
|
|
|
11199
12428
|
}
|
|
11200
12429
|
// src/lib/route/cursors.ts
|
|
11201
12430
|
import { randomUUID } from "crypto";
|
|
11202
|
-
import { existsSync as existsSync9, mkdirSync as
|
|
11203
|
-
import { join as
|
|
12431
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync10, readFileSync as readFileSync7, writeFileSync as writeFileSync6 } from "fs";
|
|
12432
|
+
import { join as join11 } from "path";
|
|
11204
12433
|
function routeCursorsPath() {
|
|
11205
|
-
return
|
|
12434
|
+
return join11(dataDir(), "route-cursors.json");
|
|
11206
12435
|
}
|
|
11207
12436
|
function readRouteCursors() {
|
|
11208
12437
|
const path = routeCursorsPath();
|
|
11209
12438
|
if (!existsSync9(path))
|
|
11210
12439
|
return {};
|
|
11211
12440
|
try {
|
|
11212
|
-
const parsed = JSON.parse(
|
|
12441
|
+
const parsed = JSON.parse(readFileSync7(path, "utf8"));
|
|
11213
12442
|
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
11214
12443
|
} catch {
|
|
11215
12444
|
return {};
|
|
@@ -11220,15 +12449,15 @@ function writeRouteCursor(key, lastFingerprint) {
|
|
|
11220
12449
|
return;
|
|
11221
12450
|
const cursors = readRouteCursors();
|
|
11222
12451
|
cursors[key] = { lastFingerprint, updatedAt: new Date().toISOString() };
|
|
11223
|
-
|
|
12452
|
+
writeFileSync6(routeCursorsPath(), JSON.stringify(cursors, null, 2), { mode: 384 });
|
|
11224
12453
|
}
|
|
11225
12454
|
function writeRouteEvidence(kind, value, evidenceDir) {
|
|
11226
12455
|
if (!evidenceDir)
|
|
11227
12456
|
return;
|
|
11228
|
-
|
|
12457
|
+
mkdirSync10(evidenceDir, { recursive: true, mode: 448 });
|
|
11229
12458
|
const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\./g, "");
|
|
11230
|
-
const evidencePath =
|
|
11231
|
-
|
|
12459
|
+
const evidencePath = join11(evidenceDir, `${kind}-${stamp}-${randomUUID().slice(0, 8)}.json`);
|
|
12460
|
+
writeFileSync6(evidencePath, JSON.stringify(value, null, 2), { mode: 384, flag: "wx" });
|
|
11232
12461
|
return evidencePath;
|
|
11233
12462
|
}
|
|
11234
12463
|
function selectRouteItems(items, maxActions, cursorKey, fingerprintOf) {
|
|
@@ -11262,7 +12491,7 @@ function routeCursorKey(kind, parts, opts) {
|
|
|
11262
12491
|
return `${kind}:${stableHash([...parts, ...routeMode])}`;
|
|
11263
12492
|
}
|
|
11264
12493
|
// src/lib/route/gates.ts
|
|
11265
|
-
import { readFileSync as
|
|
12494
|
+
import { readFileSync as readFileSync8 } from "fs";
|
|
11266
12495
|
import { dirname as dirname7, resolve as resolve5 } from "path";
|
|
11267
12496
|
class GateError extends CodedError {
|
|
11268
12497
|
gate;
|
|
@@ -11298,7 +12527,7 @@ function normalizeWorkflowForStorage(body, context) {
|
|
|
11298
12527
|
function workflowBodyFromFile(file, fallbackName, context) {
|
|
11299
12528
|
try {
|
|
11300
12529
|
const resolved = resolve5(file);
|
|
11301
|
-
return workflowBodyFromJson(JSON.parse(
|
|
12530
|
+
return workflowBodyFromJson(JSON.parse(readFileSync8(resolved, "utf8")), fallbackName, { baseDir: dirname7(resolved) });
|
|
11302
12531
|
} catch (error) {
|
|
11303
12532
|
gateFailure("validation", error, context);
|
|
11304
12533
|
}
|
|
@@ -11487,7 +12716,7 @@ function permissionModeFromOpts(opts, provider) {
|
|
|
11487
12716
|
return mode;
|
|
11488
12717
|
}
|
|
11489
12718
|
// src/lib/route/pr-review.ts
|
|
11490
|
-
import { spawnSync as
|
|
12719
|
+
import { spawnSync as spawnSync8 } from "child_process";
|
|
11491
12720
|
var PR_AUTHOR_FIELDS = [
|
|
11492
12721
|
"github_author",
|
|
11493
12722
|
"githubAuthor",
|
|
@@ -11671,13 +12900,13 @@ function prFingerprintFromTask(data, metadata) {
|
|
|
11671
12900
|
return ref ? prFingerprint(ref) : undefined;
|
|
11672
12901
|
}
|
|
11673
12902
|
function ghAuthorResolver(ref) {
|
|
11674
|
-
const result =
|
|
12903
|
+
const result = spawnSync8("gh", ["pr", "view", String(ref.number), "--repo", `${ref.owner}/${ref.repo}`, "--json", "author", "-q", ".author.login"], { encoding: "utf8", timeout: 20000 });
|
|
11675
12904
|
if (result.error || result.status !== 0)
|
|
11676
12905
|
return;
|
|
11677
12906
|
return githubLogin((result.stdout ?? "").trim());
|
|
11678
12907
|
}
|
|
11679
12908
|
function ghStateResolver(ref) {
|
|
11680
|
-
const result =
|
|
12909
|
+
const result = spawnSync8("gh", ["pr", "view", String(ref.number), "--repo", `${ref.owner}/${ref.repo}`, "--json", "state,mergeStateStatus"], { encoding: "utf8", timeout: 20000 });
|
|
11681
12910
|
if (result.error || result.status !== 0)
|
|
11682
12911
|
return;
|
|
11683
12912
|
try {
|
|
@@ -11806,7 +13035,7 @@ function prReviewRoutingDecision(data, metadata, opts, resolveAuthor = ghAuthorR
|
|
|
11806
13035
|
}
|
|
11807
13036
|
// src/lib/route/throttle.ts
|
|
11808
13037
|
import { realpathSync as realpathSync2 } from "fs";
|
|
11809
|
-
import { spawnSync as
|
|
13038
|
+
import { spawnSync as spawnSync9 } from "child_process";
|
|
11810
13039
|
import { resolve as resolve6 } from "path";
|
|
11811
13040
|
function routeThrottleLimitsFromOpts(opts) {
|
|
11812
13041
|
return {
|
|
@@ -11828,7 +13057,7 @@ function normalizeRoutePath(value) {
|
|
|
11828
13057
|
} catch {
|
|
11829
13058
|
return canonical;
|
|
11830
13059
|
}
|
|
11831
|
-
const gitRoot =
|
|
13060
|
+
const gitRoot = spawnSync9("git", ["-C", canonical, "rev-parse", "--show-toplevel"], { encoding: "utf8" });
|
|
11832
13061
|
if (gitRoot.status === 0 && gitRoot.stdout.trim()) {
|
|
11833
13062
|
try {
|
|
11834
13063
|
return realpathSync2(gitRoot.stdout.trim());
|
|
@@ -11879,7 +13108,7 @@ function routeThrottleDryRunPreview(args) {
|
|
|
11879
13108
|
};
|
|
11880
13109
|
}
|
|
11881
13110
|
function isExistingGitProjectPath(path) {
|
|
11882
|
-
const result =
|
|
13111
|
+
const result = spawnSync9("git", ["-C", path, "rev-parse", "--is-inside-work-tree"], { encoding: "utf8" });
|
|
11883
13112
|
return result.status === 0;
|
|
11884
13113
|
}
|
|
11885
13114
|
function validateRequiredRouteWorktreeProjectPath(opts, projectPath) {
|
|
@@ -11890,7 +13119,7 @@ function validateRequiredRouteWorktreeProjectPath(opts, projectPath) {
|
|
|
11890
13119
|
}
|
|
11891
13120
|
}
|
|
11892
13121
|
// src/lib/route/route-event.ts
|
|
11893
|
-
import { readFileSync as
|
|
13122
|
+
import { readFileSync as readFileSync9 } from "fs";
|
|
11894
13123
|
import { resolve as resolve7 } from "path";
|
|
11895
13124
|
function generatedRouteSandboxPreflight(workflow) {
|
|
11896
13125
|
const checks = [];
|
|
@@ -11946,7 +13175,7 @@ function routeWorkflowForStorage(store, workflowBody) {
|
|
|
11946
13175
|
}
|
|
11947
13176
|
var TODOS_TASK_ROUTE_TEMPLATE_IDS = new Set([
|
|
11948
13177
|
TODOS_TASK_WORKER_VERIFIER_TEMPLATE_ID,
|
|
11949
|
-
|
|
13178
|
+
TASK_LIFECYCLE_TEMPLATE_ID2
|
|
11950
13179
|
]);
|
|
11951
13180
|
var UNCLEARED_ROUTE_WORK_ITEM_STATUSES = new Set([
|
|
11952
13181
|
"admitted",
|
|
@@ -11987,6 +13216,36 @@ function reactivateStaleTodosTaskWorkItem(store, routeKey, item, now = Date.now(
|
|
|
11987
13216
|
reason: `re-admitted from ${item.status}: todos task still actionable after prior run (attempt ${item.attempts + 1}/${MAX_TODOS_TASK_ROUTE_REDISPATCHES})`
|
|
11988
13217
|
});
|
|
11989
13218
|
}
|
|
13219
|
+
function findRouteWorkItemByKeys(store, routeKey, idempotencyKeys) {
|
|
13220
|
+
const [primaryKey, ...aliasKeys] = idempotencyKeys;
|
|
13221
|
+
if (primaryKey) {
|
|
13222
|
+
const existingItem = store.findWorkflowWorkItem(routeKey, primaryKey);
|
|
13223
|
+
if (existingItem && isUnclearedRouteWorkItem(existingItem))
|
|
13224
|
+
return existingItem;
|
|
13225
|
+
}
|
|
13226
|
+
for (const key of aliasKeys) {
|
|
13227
|
+
const existingItem = store.findWorkflowWorkItem(routeKey, key);
|
|
13228
|
+
if (existingItem && (isUnclearedRouteWorkItem(existingItem) || existingItem.status === "queued" || existingItem.status === "deferred"))
|
|
13229
|
+
return existingItem;
|
|
13230
|
+
}
|
|
13231
|
+
return;
|
|
13232
|
+
}
|
|
13233
|
+
function isPrBacklogTask(data, metadata) {
|
|
13234
|
+
const explicitFingerprint = taskEventField(data, [
|
|
13235
|
+
"pr_fingerprint",
|
|
13236
|
+
"prFingerprint",
|
|
13237
|
+
"github_pr",
|
|
13238
|
+
"githubPr",
|
|
13239
|
+
"github_pr_fingerprint",
|
|
13240
|
+
"githubPrFingerprint",
|
|
13241
|
+
"pull_request_fingerprint",
|
|
13242
|
+
"pullRequestFingerprint"
|
|
13243
|
+
]);
|
|
13244
|
+
if (explicitFingerprint)
|
|
13245
|
+
return true;
|
|
13246
|
+
const tags = new Set(taskEventTags(taskEventRecords(data, metadata)).map((tag) => tag.toLowerCase()));
|
|
13247
|
+
return tags.has("github-pr") && tags.has("pr-merge-queue");
|
|
13248
|
+
}
|
|
11990
13249
|
function todosTaskRouteTemplateId(opts) {
|
|
11991
13250
|
const id = (opts.template ?? TODOS_TASK_WORKER_VERIFIER_TEMPLATE_ID).trim();
|
|
11992
13251
|
if (!TODOS_TASK_ROUTE_TEMPLATE_IDS.has(id)) {
|
|
@@ -11995,7 +13254,7 @@ function todosTaskRouteTemplateId(opts) {
|
|
|
11995
13254
|
return id;
|
|
11996
13255
|
}
|
|
11997
13256
|
async function readEventEnvelopeInput(opts = {}) {
|
|
11998
|
-
const raw = opts.eventJson ?? (opts.eventFile ?
|
|
13257
|
+
const raw = opts.eventJson ?? (opts.eventFile ? readFileSync9(opts.eventFile, "utf8") : process.env.HASNA_EVENT_JSON || await Bun.stdin.text());
|
|
11999
13258
|
const event = JSON.parse(raw);
|
|
12000
13259
|
if (!event || typeof event !== "object" || Array.isArray(event))
|
|
12001
13260
|
throw new ValidationError("event JSON must be an object");
|
|
@@ -12054,6 +13313,7 @@ function dedupedRoutePrint(plan, outcome) {
|
|
|
12054
13313
|
function routeEvent(plan) {
|
|
12055
13314
|
const { event, opts, idempotencyKey, workflowBody } = plan;
|
|
12056
13315
|
const sandboxPreflight = generatedRouteSandboxPreflight(workflowBody);
|
|
13316
|
+
const dedupeKeys = [idempotencyKey, ...plan.dedupeAliases ?? []];
|
|
12057
13317
|
const workItemInput = {
|
|
12058
13318
|
routeKey: plan.routeKey,
|
|
12059
13319
|
idempotencyKey,
|
|
@@ -12106,8 +13366,8 @@ function routeEvent(plan) {
|
|
|
12106
13366
|
let poolAssignment;
|
|
12107
13367
|
const outcome = store.writeTransaction(() => {
|
|
12108
13368
|
const invocation = store.createWorkflowInvocation(plan.invocationInput);
|
|
12109
|
-
const existingItem = store
|
|
12110
|
-
if (existingItem
|
|
13369
|
+
const existingItem = findRouteWorkItemByKeys(store, plan.routeKey, dedupeKeys);
|
|
13370
|
+
if (existingItem) {
|
|
12111
13371
|
if (!reactivateStaleTodosTaskWorkItem(store, plan.routeKey, existingItem)) {
|
|
12112
13372
|
const existingLoop = existingItem.loopId ? store.getLoop(existingItem.loopId) : undefined;
|
|
12113
13373
|
const existingWorkflow = existingItem.workflowId ? store.getWorkflow(existingItem.workflowId) : undefined;
|
|
@@ -12270,8 +13530,10 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12270
13530
|
const projectGroup = routeProjectGroup(opts.projectGroup, data, metadata);
|
|
12271
13531
|
const throttleLimits = routeThrottleLimitsFromOpts(opts);
|
|
12272
13532
|
const sourceProjectIdempotencyPrefix = sourceTodosProjectPath ? normalizeRoutePath(sourceTodosProjectPath) ?? resolve7(sourceTodosProjectPath) : undefined;
|
|
12273
|
-
const prFingerprint2 = prFingerprintFromTask(data, metadata);
|
|
13533
|
+
const prFingerprint2 = isPrBacklogTask(data, metadata) ? prFingerprintFromTask(data, metadata) : undefined;
|
|
12274
13534
|
const idempotencyKey = prFingerprint2 ? `todos-task:pr:${prFingerprint2}` : sourceProjectIdempotencyPrefix ? `todos-task:${sourceProjectIdempotencyPrefix}:${taskId}` : `todos-task:${taskId}`;
|
|
13535
|
+
const legacyTaskIdempotencyKey = `todos-task:${taskId}`;
|
|
13536
|
+
const dedupeAliases = legacyTaskIdempotencyKey === idempotencyKey ? [] : [legacyTaskIdempotencyKey];
|
|
12275
13537
|
const idempotencySuffix = stableSuffix(idempotencyKey);
|
|
12276
13538
|
const namePrefix = opts.namePrefix ?? "event:todos-task";
|
|
12277
13539
|
const workflowName = `${namePrefix}:${taskId.slice(0, 8)}:${idempotencySuffix}:workflow`;
|
|
@@ -12279,8 +13541,8 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12279
13541
|
if (!opts.dryRun) {
|
|
12280
13542
|
const store = new Store;
|
|
12281
13543
|
try {
|
|
12282
|
-
const existingItem = store
|
|
12283
|
-
if (existingItem
|
|
13544
|
+
const existingItem = findRouteWorkItemByKeys(store, "todos-task", [idempotencyKey, ...dedupeAliases]);
|
|
13545
|
+
if (existingItem) {
|
|
12284
13546
|
if (!reactivateStaleTodosTaskWorkItem(store, "todos-task", existingItem)) {
|
|
12285
13547
|
const existingLoop = existingItem.loopId ? store.getLoop(existingItem.loopId) : undefined;
|
|
12286
13548
|
const existingWorkflow = existingItem.workflowId ? store.getWorkflow(existingItem.workflowId) : undefined;
|
|
@@ -12347,7 +13609,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12347
13609
|
worktreeMode: opts.worktreeMode ?? "auto",
|
|
12348
13610
|
worktreeRoot: opts.worktreeRoot,
|
|
12349
13611
|
worktreeBranchPrefix: opts.worktreeBranchPrefix ?? "openloops",
|
|
12350
|
-
prHandoff: templateId ===
|
|
13612
|
+
prHandoff: templateId === TASK_LIFECYCLE_TEMPLATE_ID2 ? Boolean(opts.prHandoff) : false,
|
|
12351
13613
|
prReviewRouting: prReviewRouting.required ? prReviewRouting : undefined,
|
|
12352
13614
|
eventId: event.id,
|
|
12353
13615
|
eventType: event.type,
|
|
@@ -12358,7 +13620,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12358
13620
|
type: "todos-task-event-workflow",
|
|
12359
13621
|
event: event.id
|
|
12360
13622
|
};
|
|
12361
|
-
let workflowBody = templateId ===
|
|
13623
|
+
let workflowBody = templateId === TASK_LIFECYCLE_TEMPLATE_ID2 ? renderTaskLifecycleWorkflow(workflowInput) : renderTodosTaskWorkerVerifierWorkflow(workflowInput);
|
|
12362
13624
|
workflowBody.name = workflowName;
|
|
12363
13625
|
workflowBody.description = `Task-triggered ${templateId} workflow for ${taskTitle ?? taskId} from ${event.source}/${event.type}; ` + `idempotency=${idempotencyKey}; event=${event.id}; project=${projectPath}; projectGroup=${projectGroup ?? "-"}`;
|
|
12364
13626
|
workflowBody = normalizeWorkflowForStorage(workflowBody, workflowContext);
|
|
@@ -12384,7 +13646,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12384
13646
|
worktreePolicy: opts.worktreeMode ?? "auto",
|
|
12385
13647
|
permissions: permissionMode,
|
|
12386
13648
|
manualBreakGlass: Boolean(opts.manualBreakGlass),
|
|
12387
|
-
prHandoff: templateId ===
|
|
13649
|
+
prHandoff: templateId === TASK_LIFECYCLE_TEMPLATE_ID2 ? Boolean(opts.prHandoff) : false,
|
|
12388
13650
|
accountPolicy: providerRouting.authProfilePool?.length || providerRouting.accountPool?.length ? "pool" : hasExplicitRoleAccount ? "role-explicit" : "single",
|
|
12389
13651
|
providerRouting: providerRoutingPublic(providerRouting),
|
|
12390
13652
|
prReviewRouting: prReviewRouting.required ? prReviewRouting : undefined,
|
|
@@ -12400,6 +13662,7 @@ function routeTodosTaskEvent(event, opts) {
|
|
|
12400
13662
|
event,
|
|
12401
13663
|
opts,
|
|
12402
13664
|
idempotencyKey,
|
|
13665
|
+
dedupeAliases,
|
|
12403
13666
|
workflowBody,
|
|
12404
13667
|
workflowContext,
|
|
12405
13668
|
invocationInput,
|
|
@@ -12541,80 +13804,6 @@ function routeEventByKind(kind, event, opts) {
|
|
|
12541
13804
|
}
|
|
12542
13805
|
// src/lib/route/drain.ts
|
|
12543
13806
|
import { resolve as resolve8 } from "path";
|
|
12544
|
-
|
|
12545
|
-
// src/lib/route/todos-cli.ts
|
|
12546
|
-
import { closeSync, mkdtempSync as mkdtempSync2, openSync as openSync2, readFileSync as readFileSync9, rmSync as rmSync6 } from "fs";
|
|
12547
|
-
import { spawnSync as spawnSync9 } from "child_process";
|
|
12548
|
-
import { join as join10 } from "path";
|
|
12549
|
-
import { homedir as homedir5, tmpdir as tmpdir2 } from "os";
|
|
12550
|
-
function defaultLoopsProject() {
|
|
12551
|
-
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir5()}/.hasna/loops`;
|
|
12552
|
-
}
|
|
12553
|
-
function runLocalCommand(command, args, opts = {}) {
|
|
12554
|
-
const result = spawnSync9(command, args, {
|
|
12555
|
-
input: opts.input,
|
|
12556
|
-
encoding: "utf8",
|
|
12557
|
-
timeout: opts.timeoutMs ?? 30000,
|
|
12558
|
-
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
12559
|
-
env: process.env
|
|
12560
|
-
});
|
|
12561
|
-
return {
|
|
12562
|
-
ok: result.status === 0,
|
|
12563
|
-
status: result.status,
|
|
12564
|
-
stdout: result.stdout || "",
|
|
12565
|
-
stderr: result.stderr || "",
|
|
12566
|
-
error: result.error ? String(result.error.message || result.error) : ""
|
|
12567
|
-
};
|
|
12568
|
-
}
|
|
12569
|
-
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
12570
|
-
const tempDir = mkdtempSync2(join10(tmpdir2(), "loops-command-output-"));
|
|
12571
|
-
const stdoutPath = join10(tempDir, "stdout");
|
|
12572
|
-
const stdoutFd = openSync2(stdoutPath, "w");
|
|
12573
|
-
let result;
|
|
12574
|
-
try {
|
|
12575
|
-
result = spawnSync9(command, args, {
|
|
12576
|
-
input: opts.input,
|
|
12577
|
-
encoding: "utf8",
|
|
12578
|
-
timeout: opts.timeoutMs ?? 30000,
|
|
12579
|
-
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
12580
|
-
env: process.env,
|
|
12581
|
-
stdio: ["pipe", stdoutFd, "pipe"]
|
|
12582
|
-
});
|
|
12583
|
-
} finally {
|
|
12584
|
-
closeSync(stdoutFd);
|
|
12585
|
-
}
|
|
12586
|
-
try {
|
|
12587
|
-
return {
|
|
12588
|
-
ok: result.status === 0,
|
|
12589
|
-
status: result.status,
|
|
12590
|
-
stdout: readFileSync9(stdoutPath, "utf8"),
|
|
12591
|
-
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
12592
|
-
error: result.error ? String(result.error.message || result.error) : ""
|
|
12593
|
-
};
|
|
12594
|
-
} finally {
|
|
12595
|
-
rmSync6(tempDir, { recursive: true, force: true });
|
|
12596
|
-
}
|
|
12597
|
-
}
|
|
12598
|
-
function ensureTodosTaskList(project, slug, name, description) {
|
|
12599
|
-
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
12600
|
-
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
12601
|
-
if (!list.ok)
|
|
12602
|
-
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
12603
|
-
const values = JSON.parse(list.stdout || "[]");
|
|
12604
|
-
const found = values.find((entry) => entry.slug === slug);
|
|
12605
|
-
if (!found)
|
|
12606
|
-
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
12607
|
-
return found.id;
|
|
12608
|
-
}
|
|
12609
|
-
function todosMutationSummary(result) {
|
|
12610
|
-
return {
|
|
12611
|
-
ok: result.ok,
|
|
12612
|
-
status: result.status,
|
|
12613
|
-
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
12614
|
-
};
|
|
12615
|
-
}
|
|
12616
|
-
|
|
12617
|
-
// src/lib/route/drain.ts
|
|
12618
13807
|
function taskField(task, keys) {
|
|
12619
13808
|
for (const key of keys) {
|
|
12620
13809
|
const value = stringField(task[key]);
|
|
@@ -13517,6 +14706,24 @@ function print(value, human) {
|
|
|
13517
14706
|
else
|
|
13518
14707
|
console.log(human);
|
|
13519
14708
|
}
|
|
14709
|
+
var LOOP_STATUS_VALUES = ["active", "paused", "stopped", "expired"];
|
|
14710
|
+
function parseLoopStatuses(value, label = "--include") {
|
|
14711
|
+
const raw = splitList(value) ?? ["active", "paused"];
|
|
14712
|
+
const expanded = raw.flatMap((entry) => entry === "all" ? LOOP_STATUS_VALUES : [entry]);
|
|
14713
|
+
const invalid = expanded.filter((entry) => !LOOP_STATUS_VALUES.includes(entry));
|
|
14714
|
+
if (invalid.length > 0)
|
|
14715
|
+
throw new ValidationError(`${label} has invalid loop status: ${invalid.join(", ")}`);
|
|
14716
|
+
return [...new Set(expanded)];
|
|
14717
|
+
}
|
|
14718
|
+
function compactHealthScanOutput(scan) {
|
|
14719
|
+
if (!scan || typeof scan !== "object" || !("health" in scan))
|
|
14720
|
+
return scan;
|
|
14721
|
+
const healthValue = scan.health;
|
|
14722
|
+
if (!healthValue || typeof healthValue !== "object")
|
|
14723
|
+
return scan;
|
|
14724
|
+
const { expectations: _expectations, ...health } = healthValue;
|
|
14725
|
+
return { ...scan, health };
|
|
14726
|
+
}
|
|
13520
14727
|
function reportCliError(error) {
|
|
13521
14728
|
const message = error instanceof Error ? error.message : String(error);
|
|
13522
14729
|
process.exitCode = 1;
|
|
@@ -13607,6 +14814,15 @@ function workflowWithAgentTimeouts(workflow, timeoutMs, opts = {}) {
|
|
|
13607
14814
|
agentStepIds
|
|
13608
14815
|
};
|
|
13609
14816
|
}
|
|
14817
|
+
function agentLoopTargetWithTimeout(loop, timeoutMs) {
|
|
14818
|
+
if (loop.target.type !== "agent")
|
|
14819
|
+
throw new Error(`loop is not an agent loop: ${loop.name || loop.id}`);
|
|
14820
|
+
const target = { ...loop.target, timeoutMs };
|
|
14821
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
14822
|
+
delete target.idleTimeoutMs;
|
|
14823
|
+
const changed = loop.target.timeoutMs !== timeoutMs || timeoutMs === null && loop.target.idleTimeoutMs !== undefined;
|
|
14824
|
+
return { changed, target };
|
|
14825
|
+
}
|
|
13610
14826
|
function workflowTimeoutMigrationName(workflow, timeoutMs) {
|
|
13611
14827
|
const policy = timeoutMs === null ? "agent-timeout-unlimited" : `agent-timeout-${timeoutMs}ms`;
|
|
13612
14828
|
const suffix = randomUUID2().replace(/-/g, "").slice(0, 8);
|
|
@@ -13685,7 +14901,7 @@ function parsePolicy(opts) {
|
|
|
13685
14901
|
leaseMs: positiveDuration(opts.lease, "--lease")
|
|
13686
14902
|
};
|
|
13687
14903
|
}
|
|
13688
|
-
function
|
|
14904
|
+
function durationLabel2(ms) {
|
|
13689
14905
|
if (!ms || !Number.isFinite(ms))
|
|
13690
14906
|
return "";
|
|
13691
14907
|
const units = [
|
|
@@ -13701,14 +14917,14 @@ function durationLabel(ms) {
|
|
|
13701
14917
|
}
|
|
13702
14918
|
return `${ms}ms`;
|
|
13703
14919
|
}
|
|
13704
|
-
function
|
|
14920
|
+
function scheduleLabel2(schedule) {
|
|
13705
14921
|
if (schedule.type === "once")
|
|
13706
14922
|
return `once:${schedule.at}`;
|
|
13707
14923
|
if (schedule.type === "interval")
|
|
13708
|
-
return `every:${
|
|
14924
|
+
return `every:${durationLabel2(schedule.everyMs) || `${schedule.everyMs}ms`}`;
|
|
13709
14925
|
if (schedule.type === "cron")
|
|
13710
14926
|
return `cron:${schedule.expression}`;
|
|
13711
|
-
return schedule.minIntervalMs ? `dynamic:min-${
|
|
14927
|
+
return schedule.minIntervalMs ? `dynamic:min-${durationLabel2(schedule.minIntervalMs) || `${schedule.minIntervalMs}ms`}` : "dynamic";
|
|
13712
14928
|
}
|
|
13713
14929
|
function targetLabel(target) {
|
|
13714
14930
|
if (target.type === "command")
|
|
@@ -13720,7 +14936,7 @@ function targetLabel(target) {
|
|
|
13720
14936
|
function defaultLoopDescription(name, schedule, target) {
|
|
13721
14937
|
return [
|
|
13722
14938
|
`Why: keep ${name} running as an OpenLoops scheduled automation.`,
|
|
13723
|
-
`How: ${targetLabel(target)} on cadence ${
|
|
14939
|
+
`How: ${targetLabel(target)} on cadence ${scheduleLabel2(schedule)}.`,
|
|
13724
14940
|
"Outcome: record each run, status, retries, and evidence in OpenLoops for operator review."
|
|
13725
14941
|
].join(" ");
|
|
13726
14942
|
}
|
|
@@ -13950,7 +15166,7 @@ program.command("export").description("export a local OpenLoops migration bundle
|
|
|
13950
15166
|
throw new ValidationError("export is not no-loss because redactions/blockers are present; rerun with --allow-redacted to write a redacted bundle");
|
|
13951
15167
|
}
|
|
13952
15168
|
if (!opts.dryRun)
|
|
13953
|
-
|
|
15169
|
+
writeFileSync7(opts.file, `${JSON.stringify(bundle, null, 2)}
|
|
13954
15170
|
`, { mode: 384 });
|
|
13955
15171
|
const output = {
|
|
13956
15172
|
ok: true,
|
|
@@ -14248,6 +15464,7 @@ addScheduleOptions(addTodosDrainOptions(routes.command("schedule <kind> <name>")
|
|
|
14248
15464
|
})).action(runAction((kind, name, opts) => {
|
|
14249
15465
|
if (kind !== "todos-task")
|
|
14250
15466
|
throw new ValidationError("route schedule currently supports kind todos-task");
|
|
15467
|
+
todosTaskRouteTemplateId(opts);
|
|
14251
15468
|
const store = new Store;
|
|
14252
15469
|
try {
|
|
14253
15470
|
const target = {
|
|
@@ -14487,7 +15704,7 @@ workflows.command("recover <runId>").description("reset interrupted running work
|
|
|
14487
15704
|
store.close();
|
|
14488
15705
|
}
|
|
14489
15706
|
}));
|
|
14490
|
-
workflows.command("migrate-agent-timeouts").description("
|
|
15707
|
+
workflows.command("migrate-agent-timeouts").description("migrate workflow loops, or a direct agent loop selected with --loop, to a new agent timeout policy").option("--loop <idOrName>", "migrate only one loop; required for direct agent loops").option("--timeout <duration>", "agent timeout policy; use none/unlimited for no timeout", "none").option("--apply", "create new workflow specs or update direct agent targets for eligible loops").option("--archive-old", "archive old workflow specs after retargeting when no active loops still reference them").action(runAction((opts) => {
|
|
14491
15708
|
const store = new Store;
|
|
14492
15709
|
try {
|
|
14493
15710
|
const timeoutMs = timeoutDuration(opts.timeout, "--timeout") ?? null;
|
|
@@ -14498,8 +15715,45 @@ workflows.command("migrate-agent-timeouts").description("append-only migrate act
|
|
|
14498
15715
|
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "loop is archived" });
|
|
14499
15716
|
continue;
|
|
14500
15717
|
}
|
|
15718
|
+
if (loop.target.type === "agent") {
|
|
15719
|
+
const migration2 = agentLoopTargetWithTimeout(loop, timeoutMs);
|
|
15720
|
+
if (!migration2.changed) {
|
|
15721
|
+
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "agent timeout policy already matches" });
|
|
15722
|
+
continue;
|
|
15723
|
+
}
|
|
15724
|
+
if (store.hasRunningRun(loop.id)) {
|
|
15725
|
+
rows.push({ loop: publicLoop(loop), status: "blocked", reason: "loop has a running run; retry after it finishes" });
|
|
15726
|
+
continue;
|
|
15727
|
+
}
|
|
15728
|
+
if (!opts.apply) {
|
|
15729
|
+
rows.push({
|
|
15730
|
+
loop: publicLoop(loop),
|
|
15731
|
+
status: "would_update",
|
|
15732
|
+
target: { ...migration2.target, prompt: redact(migration2.target.prompt) },
|
|
15733
|
+
timeoutMs
|
|
15734
|
+
});
|
|
15735
|
+
continue;
|
|
15736
|
+
}
|
|
15737
|
+
try {
|
|
15738
|
+
const updated = store.updateAgentLoopTimeout(loop.id, timeoutMs);
|
|
15739
|
+
rows.push({
|
|
15740
|
+
loop: publicLoop(updated),
|
|
15741
|
+
previousLoop: publicLoop(loop),
|
|
15742
|
+
status: "updated",
|
|
15743
|
+
timeoutMs
|
|
15744
|
+
});
|
|
15745
|
+
} catch (error) {
|
|
15746
|
+
rows.push({
|
|
15747
|
+
loop: publicLoop(loop),
|
|
15748
|
+
status: "blocked",
|
|
15749
|
+
reason: migrationErrorReason(error),
|
|
15750
|
+
timeoutMs
|
|
15751
|
+
});
|
|
15752
|
+
}
|
|
15753
|
+
continue;
|
|
15754
|
+
}
|
|
14501
15755
|
if (loop.target.type !== "workflow") {
|
|
14502
|
-
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "loop is not
|
|
15756
|
+
rows.push({ loop: publicLoop(loop), status: "skipped", reason: "loop is not an agent or workflow loop" });
|
|
14503
15757
|
continue;
|
|
14504
15758
|
}
|
|
14505
15759
|
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
@@ -14558,11 +15812,13 @@ workflows.command("migrate-agent-timeouts").description("append-only migrate act
|
|
|
14558
15812
|
timeoutMs,
|
|
14559
15813
|
total: rows.length,
|
|
14560
15814
|
migrated: rows.filter((row) => row.status === "migrated").length,
|
|
15815
|
+
updated: rows.filter((row) => row.status === "updated").length,
|
|
14561
15816
|
wouldMigrate: rows.filter((row) => row.status === "would_migrate").length,
|
|
15817
|
+
wouldUpdate: rows.filter((row) => row.status === "would_update").length,
|
|
14562
15818
|
blocked: rows.filter((row) => row.status === "blocked").length,
|
|
14563
15819
|
skipped: rows.filter((row) => row.status === "skipped").length
|
|
14564
15820
|
};
|
|
14565
|
-
print({ summary, rows }, opts.apply ? `migrated=${summary.migrated} blocked=${summary.blocked} skipped=${summary.skipped}` : `would_migrate=${summary.wouldMigrate} blocked=${summary.blocked} skipped=${summary.skipped}`);
|
|
15821
|
+
print({ summary, rows }, opts.apply ? `migrated=${summary.migrated} updated=${summary.updated} blocked=${summary.blocked} skipped=${summary.skipped}` : `would_migrate=${summary.wouldMigrate} would_update=${summary.wouldUpdate} blocked=${summary.blocked} skipped=${summary.skipped}`);
|
|
14566
15822
|
} finally {
|
|
14567
15823
|
store.close();
|
|
14568
15824
|
}
|
|
@@ -14676,13 +15932,23 @@ program.command("list").alias("ls").description("list loops with schedule and ne
|
|
|
14676
15932
|
for (const loop of loops) {
|
|
14677
15933
|
const machine = loop.machine ? ` machine=${loop.machine.id}` : "";
|
|
14678
15934
|
const archive = loop.archivedAt ? ` archived=${loop.archivedAt} from=${loop.archivedFromStatus ?? "-"}` : "";
|
|
14679
|
-
console.log(`${loop.id} ${loop.status.padEnd(7)} cadence=${
|
|
15935
|
+
console.log(`${loop.id} ${loop.status.padEnd(7)} cadence=${scheduleLabel2(loop.schedule)} next=${loop.nextRunAt ?? "-"} ${loop.name}${machine}${archive}`);
|
|
14680
15936
|
}
|
|
14681
15937
|
}
|
|
14682
15938
|
} finally {
|
|
14683
15939
|
store.close();
|
|
14684
15940
|
}
|
|
14685
15941
|
}));
|
|
15942
|
+
program.command("ui").description("open a live table of active loops").option("--refresh <duration>", "refresh interval", "2s").action(runAction(async (opts) => {
|
|
15943
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
|
15944
|
+
console.error("OpenLoops UI requires a TTY terminal.");
|
|
15945
|
+
console.error("Use `loops list`, `loops runs`, or `loops daemon status` non-interactively.");
|
|
15946
|
+
process.exitCode = 1;
|
|
15947
|
+
return;
|
|
15948
|
+
}
|
|
15949
|
+
const refreshMs = Math.max(500, parseDuration(opts.refresh ?? "2s"));
|
|
15950
|
+
await runLoopsUiApp({ refreshMs });
|
|
15951
|
+
}));
|
|
14686
15952
|
program.command("show <idOrName>").description("show one loop by id or name").action(runAction((idOrName) => {
|
|
14687
15953
|
const store = new Store;
|
|
14688
15954
|
try {
|
|
@@ -14737,8 +16003,9 @@ var health = program.command("health").description("summarize loop health and la
|
|
|
14737
16003
|
console.log(JSON.stringify(report, null, 2));
|
|
14738
16004
|
else {
|
|
14739
16005
|
console.log(`loops=${report.summary.loops} healthy=${report.summary.healthy} unhealthy=${report.summary.unhealthy} warnings=${report.summary.warnings}`);
|
|
14740
|
-
for (const expectation of report.expectations.filter((entry) => !entry.ok)) {
|
|
14741
|
-
|
|
16006
|
+
for (const expectation of report.expectations.filter((entry) => !entry.ok || entry.check.status === "warn")) {
|
|
16007
|
+
const status = expectation.ok ? "warn" : "fail";
|
|
16008
|
+
console.log(`${status} ${expectation.loop.name} ${expectation.failure?.classification ?? "unknown"} ${expectation.failure?.fingerprint ?? "-"}`);
|
|
14742
16009
|
}
|
|
14743
16010
|
}
|
|
14744
16011
|
if (!report.ok)
|
|
@@ -14747,6 +16014,122 @@ var health = program.command("health").description("summarize loop health and la
|
|
|
14747
16014
|
store.close();
|
|
14748
16015
|
}
|
|
14749
16016
|
}));
|
|
16017
|
+
health.command("scan").description("scan OpenLoops health, write bounded reports, and optionally upsert deduped todos findings").option("--include <statuses>", "comma-separated loop statuses to inventory: active,paused,stopped,expired,all", "active,paused").option("--limit <n>", "maximum loops to inspect", "200").option("--max-findings <n>", "maximum findings to include in output", "100").option("--latest-run", "include latest-run and stale-running checks").option("--no-latest-run", "skip latest-run and stale-running checks").option("--stale-running-after <duration>", "minimum age before a running latest run is stale; loop lease and 10m still apply").option("--doctor", "include doctor/preflight checks").option("--daemon", "include daemon status").option("--start-daemon", "safe self-heal: start the daemon if it is not running").option("--report-dir <path>", "write summary.json and report.md under this reports root").option("--evidence-dir <path>", "alias for --report-dir and todo evidence output").option("--upsert-todos", "upsert deduped todos tasks for scan findings").option("--project <path>", "todos project path for --upsert-todos", defaultLoopsProject()).option("--task-list <slug>", "todos task-list slug for --upsert-todos", "loop-error-self-heal").option("--max-actions <n>", "maximum todos tasks to upsert", "5").option("--dry-run", "with --upsert-todos, print intended task upserts without mutating todos").option("--auto-route", "with --upsert-todos, opt routed tasks into task-created headless worker/verifier automation").option("--route-project-path <path>", "fallback project path for --auto-route when the finding has no cwd").option("-j, --json", "print JSON for this command").action(runAction(async (opts) => {
|
|
16018
|
+
const store = new Store;
|
|
16019
|
+
try {
|
|
16020
|
+
const includeStatuses = parseLoopStatuses(opts.include, "--include");
|
|
16021
|
+
let daemon = opts.daemon || opts.startDaemon ? daemonStatus(store) : undefined;
|
|
16022
|
+
const selfHeals = [];
|
|
16023
|
+
if (opts.startDaemon) {
|
|
16024
|
+
if (daemon?.running) {
|
|
16025
|
+
selfHeals.push({
|
|
16026
|
+
kind: "daemon-start",
|
|
16027
|
+
attempted: false,
|
|
16028
|
+
ok: true,
|
|
16029
|
+
reason: "daemon already running"
|
|
16030
|
+
});
|
|
16031
|
+
} else {
|
|
16032
|
+
const result = await startDaemon({ cliEntry: process.argv[1] ?? "loops" });
|
|
16033
|
+
const ok = Boolean(result.started || result.alreadyRunning);
|
|
16034
|
+
selfHeals.push({
|
|
16035
|
+
kind: "daemon-start",
|
|
16036
|
+
attempted: true,
|
|
16037
|
+
ok,
|
|
16038
|
+
reason: daemon?.stale ? "daemon pid file was stale" : "daemon was not running",
|
|
16039
|
+
result
|
|
16040
|
+
});
|
|
16041
|
+
daemon = daemonStatus(store);
|
|
16042
|
+
}
|
|
16043
|
+
}
|
|
16044
|
+
let scan = writeHealthScanReports(buildHealthScan(store, {
|
|
16045
|
+
includeStatuses,
|
|
16046
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 200,
|
|
16047
|
+
maxFindings: nonNegativeInteger(opts.maxFindings, "--max-findings") ?? 100,
|
|
16048
|
+
latestRun: opts.latestRun !== false,
|
|
16049
|
+
staleRunningMs: opts.staleRunningAfter ? positiveDuration(opts.staleRunningAfter, "--stale-running-after") : undefined,
|
|
16050
|
+
doctor: opts.doctor ? runDoctor(store) : undefined,
|
|
16051
|
+
daemon,
|
|
16052
|
+
selfHeals
|
|
16053
|
+
}), { reportDir: opts.reportDir ?? opts.evidenceDir });
|
|
16054
|
+
if (opts.upsertTodos) {
|
|
16055
|
+
const tasks = scan.findings.filter((finding) => finding.recommendedTask).map((finding) => {
|
|
16056
|
+
const task = finding.recommendedTask;
|
|
16057
|
+
const description = [
|
|
16058
|
+
task.description,
|
|
16059
|
+
scan.reports ? `Report: ${scan.reports.markdown}` : undefined
|
|
16060
|
+
].filter(Boolean).join(`
|
|
16061
|
+
|
|
16062
|
+
`);
|
|
16063
|
+
return {
|
|
16064
|
+
title: task.title,
|
|
16065
|
+
description,
|
|
16066
|
+
priority: task.priority,
|
|
16067
|
+
tags: task.tags,
|
|
16068
|
+
fingerprint: task.dedupeKey,
|
|
16069
|
+
extra: {
|
|
16070
|
+
kind: finding.kind,
|
|
16071
|
+
severity: finding.severity,
|
|
16072
|
+
classification: finding.classification
|
|
16073
|
+
},
|
|
16074
|
+
metadata: {
|
|
16075
|
+
source: "openloops.health.scan",
|
|
16076
|
+
kind: finding.kind,
|
|
16077
|
+
severity: finding.severity,
|
|
16078
|
+
loop_id: finding.loop?.id,
|
|
16079
|
+
loop_name: finding.loop?.name,
|
|
16080
|
+
loop_status: finding.loop?.status,
|
|
16081
|
+
run_id: finding.run?.id,
|
|
16082
|
+
classification: finding.classification,
|
|
16083
|
+
fingerprint: task.dedupeKey,
|
|
16084
|
+
cwd: finding.route?.cwd,
|
|
16085
|
+
provider: finding.route?.provider,
|
|
16086
|
+
report_dir: scan.reports?.dir,
|
|
16087
|
+
no_tmux_dispatch: true
|
|
16088
|
+
}
|
|
16089
|
+
};
|
|
16090
|
+
});
|
|
16091
|
+
const result = upsertRouteTasks({
|
|
16092
|
+
project: opts.project,
|
|
16093
|
+
taskList: {
|
|
16094
|
+
slug: opts.taskList,
|
|
16095
|
+
name: "Loop Error Self Heal",
|
|
16096
|
+
description: "Deduped OpenLoops health scan findings for daemon, doctor, preflight, latest-run, and stale-running issues."
|
|
16097
|
+
},
|
|
16098
|
+
cursorKey: routeCursorKey("health", ["scan", opts.project, opts.taskList, includeStatuses.join(","), opts.limit, Boolean(opts.doctor), Boolean(opts.daemon)], { autoRoute: Boolean(opts.autoRoute), routeProjectPath: opts.routeProjectPath }),
|
|
16099
|
+
maxActions: positiveInteger(opts.maxActions, "--max-actions") ?? 5,
|
|
16100
|
+
dryRun: Boolean(opts.dryRun),
|
|
16101
|
+
autoRoute: Boolean(opts.autoRoute),
|
|
16102
|
+
routeProjectPath: opts.routeProjectPath,
|
|
16103
|
+
source: "openloops.health.scan",
|
|
16104
|
+
evidence: { kind: "health-scan-route-tasks", dir: opts.evidenceDir ?? opts.reportDir },
|
|
16105
|
+
summary: {
|
|
16106
|
+
status: scan.status,
|
|
16107
|
+
inspected: scan.counts.loops,
|
|
16108
|
+
findings: scan.counts.findings,
|
|
16109
|
+
reportDir: scan.reports?.dir
|
|
16110
|
+
},
|
|
16111
|
+
tasks
|
|
16112
|
+
});
|
|
16113
|
+
scan = { ...scan, todos: result.output };
|
|
16114
|
+
if (!result.ok)
|
|
16115
|
+
process.exitCode = 1;
|
|
16116
|
+
}
|
|
16117
|
+
const jsonMode = isJson() || Boolean(opts.json);
|
|
16118
|
+
if (jsonMode)
|
|
16119
|
+
console.log(JSON.stringify(compactHealthScanOutput(scan), null, 2));
|
|
16120
|
+
else {
|
|
16121
|
+
const actions = Array.isArray(scan.todos?.actions) ? scan.todos.actions : [];
|
|
16122
|
+
console.log(`health_scan status=${scan.status} loops=${scan.counts.loops} findings=${scan.counts.findings} ` + `reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} ` + `latest=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} ` + `daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings} ` + `report=${scan.reports?.markdown ?? "none"} todos_actions=${actions.length}`);
|
|
16123
|
+
for (const finding of scan.findings) {
|
|
16124
|
+
console.log(`${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop?.name ?? ""} ${finding.message}`);
|
|
16125
|
+
}
|
|
16126
|
+
}
|
|
16127
|
+
if (!process.exitCode && scan.status !== "ok")
|
|
16128
|
+
process.exitCode = scan.status === "critical" ? 2 : 1;
|
|
16129
|
+
} finally {
|
|
16130
|
+
store.close();
|
|
16131
|
+
}
|
|
16132
|
+
}));
|
|
14750
16133
|
health.command("route-tasks").description("upsert deduped todos tasks for failed loop health expectations").option("--project <path>", "todos project path", defaultLoopsProject()).option("--task-list <slug>", "todos task-list slug", "loop-error-self-heal").option("--limit <n>", "maximum loops to inspect", "200").option("--max-actions <n>", "maximum todos tasks to upsert", "5").option("--include-inactive", "also route stopped or expired loops").option("--auto-route", "opt routed tasks into task-created headless worker/verifier automation").option("--route-project-path <path>", "fallback project path for --auto-route when the failed loop has no cwd").option("--evidence-dir <path>", "write the route result JSON to this directory").option("--dry-run", "print intended task upserts without mutating todos").action(runAction((opts) => {
|
|
14751
16134
|
const store = new Store;
|
|
14752
16135
|
try {
|
|
@@ -15105,7 +16488,7 @@ function listManagedBackups(dir) {
|
|
|
15105
16488
|
const group = modern ? `reason:${modern[1]}` : legacy ? `legacy:${legacy[1]}` : undefined;
|
|
15106
16489
|
if (!group)
|
|
15107
16490
|
continue;
|
|
15108
|
-
const path =
|
|
16491
|
+
const path = join12(dir, name);
|
|
15109
16492
|
try {
|
|
15110
16493
|
entries.push({ path, group, timeMs: statSync2(path).mtimeMs });
|
|
15111
16494
|
} catch {}
|
|
@@ -15120,7 +16503,7 @@ function listStrayTempFiles(dirs) {
|
|
|
15120
16503
|
for (const name of readdirSync3(dir)) {
|
|
15121
16504
|
if (!name.endsWith(".tmp"))
|
|
15122
16505
|
continue;
|
|
15123
|
-
const path =
|
|
16506
|
+
const path = join12(dir, name);
|
|
15124
16507
|
try {
|
|
15125
16508
|
if (statSync2(path).isFile())
|
|
15126
16509
|
stray.push(path);
|
|
@@ -15143,7 +16526,7 @@ program.command("gc").description("prune old run history, rotate database backup
|
|
|
15143
16526
|
} finally {
|
|
15144
16527
|
store.close();
|
|
15145
16528
|
}
|
|
15146
|
-
const backupsDir =
|
|
16529
|
+
const backupsDir = join12(dataDir(), "backups");
|
|
15147
16530
|
const grouped = new Map;
|
|
15148
16531
|
for (const entry of listManagedBackups(backupsDir)) {
|
|
15149
16532
|
const group = grouped.get(entry.group) ?? [];
|