@hasna/loops 0.4.13 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +74 -3
  2. package/README.md +54 -12
  3. package/dist/api/index.d.ts +35 -0
  4. package/dist/api/index.js +695 -10
  5. package/dist/cli/index.js +1633 -366
  6. package/dist/cli/ui.d.ts +44 -0
  7. package/dist/daemon/index.js +948 -212
  8. package/dist/generated/storage-kit/health.d.ts +19 -0
  9. package/dist/generated/storage-kit/index.d.ts +7 -0
  10. package/dist/generated/storage-kit/migrations.d.ts +47 -0
  11. package/dist/generated/storage-kit/mode.d.ts +47 -0
  12. package/dist/generated/storage-kit/pool.d.ts +33 -0
  13. package/dist/generated/storage-kit/query.d.ts +35 -0
  14. package/dist/generated/storage-kit/tls.d.ts +25 -0
  15. package/dist/index.d.ts +2 -2
  16. package/dist/index.js +3567 -2010
  17. package/dist/lib/health.d.ts +83 -3
  18. package/dist/lib/mode.js +14 -2
  19. package/dist/lib/scheduler.d.ts +5 -2
  20. package/dist/lib/storage/index.d.ts +1 -0
  21. package/dist/lib/storage/index.js +1329 -211
  22. package/dist/lib/storage/pg-executor.d.ts +27 -0
  23. package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
  24. package/dist/lib/storage/postgres-loop-storage.d.ts +114 -0
  25. package/dist/lib/storage/sqlite.js +336 -8
  26. package/dist/lib/store.d.ts +234 -0
  27. package/dist/lib/store.js +350 -8
  28. package/dist/mcp/index.js +1067 -306
  29. package/dist/runner/index.js +124 -25
  30. package/dist/sdk/http.d.ts +115 -0
  31. package/dist/sdk/http.js +145 -0
  32. package/dist/sdk/index.d.ts +5 -1
  33. package/dist/sdk/index.js +1052 -312
  34. package/dist/serve/index.d.ts +4 -0
  35. package/dist/serve/index.js +7619 -0
  36. package/dist/types.d.ts +3 -0
  37. package/docs/CUTOVER-RUNBOOK.md +61 -0
  38. package/docs/USAGE.md +50 -11
  39. package/package.json +14 -2
package/dist/mcp/index.js CHANGED
@@ -3,9 +3,9 @@
3
3
 
4
4
  // src/lib/store.ts
5
5
  import { Database } from "bun:sqlite";
6
- import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
7
- import { tmpdir } from "os";
8
- import { basename as basename2, dirname as dirname2, join as join3 } from "path";
6
+ import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
7
+ import { tmpdir as tmpdir2 } from "os";
8
+ import { basename as basename2, dirname as dirname2, join as join4 } from "path";
9
9
 
10
10
  // src/lib/errors.ts
11
11
  class CodedError extends Error {
@@ -1204,6 +1204,196 @@ function discardWorkflowRunManifest(staged) {
1204
1204
  } catch {}
1205
1205
  }
1206
1206
 
1207
+ // src/lib/route/todos-cli.ts
1208
+ import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
1209
+ import { spawnSync as spawnSync2 } from "child_process";
1210
+ import { join as join3 } from "path";
1211
+ import { homedir as homedir2, tmpdir } from "os";
1212
+
1213
+ // src/lib/format.ts
1214
+ var TEXT_OUTPUT_LIMIT = 32 * 1024;
1215
+ var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
1216
+ function redact(value, visible = 0) {
1217
+ if (!value)
1218
+ return value;
1219
+ if (value.length <= visible)
1220
+ return value;
1221
+ if (visible <= 0)
1222
+ return `[redacted ${value.length} chars]`;
1223
+ return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
1224
+ }
1225
+ function truncateTextOutput(value) {
1226
+ if (value.length <= TEXT_OUTPUT_LIMIT)
1227
+ return value;
1228
+ return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
1229
+ [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
1230
+ }
1231
+ function redactSensitivePayload(value, key) {
1232
+ if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
1233
+ if (typeof value === "string")
1234
+ return redact(value);
1235
+ if (value === undefined || value === null)
1236
+ return value;
1237
+ return "[redacted]";
1238
+ }
1239
+ if (Array.isArray(value))
1240
+ return value.map((item) => redactSensitivePayload(item));
1241
+ if (value && typeof value === "object") {
1242
+ return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
1243
+ }
1244
+ return value;
1245
+ }
1246
+ function textOutputBlocks(value, opts = {}) {
1247
+ const indent = opts.indent ?? "";
1248
+ const nested = `${indent} `;
1249
+ const blocks = [];
1250
+ for (const [label, output] of [
1251
+ ["stdout", value.stdout],
1252
+ ["stderr", value.stderr]
1253
+ ]) {
1254
+ if (!output)
1255
+ continue;
1256
+ blocks.push(`${indent}${label}:`);
1257
+ for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
1258
+ blocks.push(`${nested}${line}`);
1259
+ }
1260
+ }
1261
+ return blocks;
1262
+ }
1263
+ function publicLoop(loop) {
1264
+ const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
1265
+ return {
1266
+ ...loop,
1267
+ target
1268
+ };
1269
+ }
1270
+ function publicRun(run, showOutput = false, opts = {}) {
1271
+ return {
1272
+ ...run,
1273
+ stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
1274
+ stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
1275
+ error: opts.redactError ? redact(run.error) : run.error
1276
+ };
1277
+ }
1278
+ function publicExecutorResult(result, showOutput = false) {
1279
+ return {
1280
+ ...result,
1281
+ stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
1282
+ stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
1283
+ error: redact(result.error)
1284
+ };
1285
+ }
1286
+ function publicWorkflow(workflow) {
1287
+ return {
1288
+ ...workflow,
1289
+ steps: workflow.steps.map((step) => ({
1290
+ ...step,
1291
+ target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
1292
+ }))
1293
+ };
1294
+ }
1295
+ function publicWorkflowRun(run) {
1296
+ return { ...run, error: redact(run.error) };
1297
+ }
1298
+ function publicWorkflowInvocation(invocation) {
1299
+ return redactSensitivePayload(invocation);
1300
+ }
1301
+ function publicWorkflowWorkItem(item) {
1302
+ return { ...item, lastReason: redact(item.lastReason, 240) };
1303
+ }
1304
+ function publicWorkflowStepRun(run, showOutput = false) {
1305
+ return {
1306
+ ...run,
1307
+ stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
1308
+ stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
1309
+ error: redact(run.error)
1310
+ };
1311
+ }
1312
+ function publicWorkflowEvent(event) {
1313
+ return { ...event, payload: redactSensitivePayload(event.payload) };
1314
+ }
1315
+ function publicGoal(goal) {
1316
+ return {
1317
+ ...goal,
1318
+ objective: redact(goal.objective, 120)
1319
+ };
1320
+ }
1321
+ function publicGoalRun(run) {
1322
+ return {
1323
+ ...run,
1324
+ evidence: redactSensitivePayload(run.evidence),
1325
+ rawResponse: redactSensitivePayload(run.rawResponse)
1326
+ };
1327
+ }
1328
+
1329
+ // src/lib/route/todos-cli.ts
1330
+ function defaultLoopsProject() {
1331
+ return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
1332
+ }
1333
+ function runLocalCommand(command, args, opts = {}) {
1334
+ const result = spawnSync2(command, args, {
1335
+ input: opts.input,
1336
+ encoding: "utf8",
1337
+ timeout: opts.timeoutMs ?? 30000,
1338
+ maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
1339
+ env: process.env
1340
+ });
1341
+ return {
1342
+ ok: result.status === 0,
1343
+ status: result.status,
1344
+ stdout: result.stdout || "",
1345
+ stderr: result.stderr || "",
1346
+ error: result.error ? String(result.error.message || result.error) : ""
1347
+ };
1348
+ }
1349
+ function runLocalCommandWithStdoutFile(command, args, opts = {}) {
1350
+ const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
1351
+ const stdoutPath = join3(tempDir, "stdout");
1352
+ const stdoutFd = openSync(stdoutPath, "w");
1353
+ let result;
1354
+ try {
1355
+ result = spawnSync2(command, args, {
1356
+ input: opts.input,
1357
+ encoding: "utf8",
1358
+ timeout: opts.timeoutMs ?? 30000,
1359
+ maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
1360
+ env: process.env,
1361
+ stdio: ["pipe", stdoutFd, "pipe"]
1362
+ });
1363
+ } finally {
1364
+ closeSync(stdoutFd);
1365
+ }
1366
+ try {
1367
+ return {
1368
+ ok: result.status === 0,
1369
+ status: result.status,
1370
+ stdout: readFileSync3(stdoutPath, "utf8"),
1371
+ stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
1372
+ error: result.error ? String(result.error.message || result.error) : ""
1373
+ };
1374
+ } finally {
1375
+ rmSync2(tempDir, { recursive: true, force: true });
1376
+ }
1377
+ }
1378
+ function ensureTodosTaskList(project, slug, name, description) {
1379
+ runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
1380
+ const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
1381
+ if (!list.ok)
1382
+ throw new Error(list.stderr || list.error || "failed to list todos task lists");
1383
+ const values = JSON.parse(list.stdout || "[]");
1384
+ const found = values.find((entry) => entry.slug === slug);
1385
+ if (!found)
1386
+ throw new Error(`todos task list not found after ensure: ${slug}`);
1387
+ return found.id;
1388
+ }
1389
+ function todosMutationSummary(result) {
1390
+ return {
1391
+ ok: result.ok,
1392
+ status: result.status,
1393
+ error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
1394
+ };
1395
+ }
1396
+
1207
1397
  // src/lib/store.ts
1208
1398
  var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
1209
1399
  var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
@@ -1213,6 +1403,7 @@ var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "s
1213
1403
  var PRUNE_BATCH_SIZE = 400;
1214
1404
  var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
1215
1405
  var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
1406
+ var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
1216
1407
  function rowToLoop(row) {
1217
1408
  return {
1218
1409
  id: row.id,
@@ -1263,6 +1454,9 @@ function rowToRun(row) {
1263
1454
  updatedAt: row.updated_at
1264
1455
  };
1265
1456
  }
1457
+ function latestRunTime(row) {
1458
+ return row.finished_at ?? row.started_at ?? row.created_at;
1459
+ }
1266
1460
  function rowToWorkflow(row) {
1267
1461
  return {
1268
1462
  id: row.id,
@@ -1517,7 +1711,7 @@ class Store {
1517
1711
  const file = path ?? dbPath();
1518
1712
  if (file !== ":memory:")
1519
1713
  ensurePrivateStorePath(file);
1520
- this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
1714
+ this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
1521
1715
  if (file === ":memory:")
1522
1716
  this.memoryRootDir = this.rootDir;
1523
1717
  this.db = new Database(file);
@@ -2015,7 +2209,32 @@ class Store {
2015
2209
  } else {
2016
2210
  rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
2017
2211
  }
2018
- return rows.map(rowToLoop);
2212
+ return this.withLatestRunSummaries(rows.map(rowToLoop));
2213
+ }
2214
+ withLatestRunSummaries(loops) {
2215
+ if (loops.length === 0)
2216
+ return loops;
2217
+ const placeholders = loops.map(() => "?").join(",");
2218
+ const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
2219
+ FROM (
2220
+ SELECT loop_id, id, status, started_at, finished_at, created_at,
2221
+ ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
2222
+ FROM loop_runs
2223
+ WHERE loop_id IN (${placeholders})
2224
+ )
2225
+ WHERE rn = 1`).all(...loops.map((loop) => loop.id));
2226
+ const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
2227
+ return loops.map((loop) => {
2228
+ const latest = latestByLoopId.get(loop.id);
2229
+ if (!latest)
2230
+ return loop;
2231
+ return {
2232
+ ...loop,
2233
+ latestRunId: latest.id,
2234
+ latestRunStatus: latest.status,
2235
+ lastRunAt: latestRunTime(latest)
2236
+ };
2237
+ });
2019
2238
  }
2020
2239
  dueLoops(now, limit = 500) {
2021
2240
  const rows = this.db.query(`SELECT * FROM loops
@@ -2134,6 +2353,45 @@ class Store {
2134
2353
  throw error;
2135
2354
  }
2136
2355
  }
2356
+ updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
2357
+ const updated = (opts.now ?? new Date).toISOString();
2358
+ this.db.exec("BEGIN IMMEDIATE");
2359
+ try {
2360
+ const current = this.requireUniqueLoop(idOrName);
2361
+ if (current.archivedAt)
2362
+ throw new LoopArchivedError(current.name || current.id);
2363
+ if (current.target.type !== "agent")
2364
+ throw new Error(`loop is not an agent loop: ${idOrName}`);
2365
+ if (this.hasRunningRun(current.id))
2366
+ throw new Error(`refusing to update running loop: ${current.id}`);
2367
+ const target = { ...current.target, timeoutMs };
2368
+ if (timeoutMs === null && target.idleTimeoutMs !== undefined)
2369
+ delete target.idleTimeoutMs;
2370
+ const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
2371
+ WHERE id=$id
2372
+ AND ($daemonLeaseId IS NULL OR EXISTS (
2373
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
2374
+ ))`).run({
2375
+ $id: current.id,
2376
+ $target: JSON.stringify(target),
2377
+ $updated: updated,
2378
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
2379
+ $now: updated
2380
+ });
2381
+ if (res.changes !== 1)
2382
+ throw new Error("daemon lease lost");
2383
+ this.db.exec("COMMIT");
2384
+ const after = this.getLoop(current.id);
2385
+ if (!after)
2386
+ throw new Error(`loop not found after timeout update: ${current.id}`);
2387
+ return after;
2388
+ } catch (error) {
2389
+ try {
2390
+ this.db.exec("ROLLBACK");
2391
+ } catch {}
2392
+ throw error;
2393
+ }
2394
+ }
2137
2395
  createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
2138
2396
  const normalized = normalizeCreateWorkflowInput(workflowInput);
2139
2397
  const updated = (opts.now ?? new Date).toISOString();
@@ -2464,6 +2722,59 @@ class Store {
2464
2722
  updated
2465
2723
  });
2466
2724
  }
2725
+ taskLifecycleTodosPointerContext(workflowRunId) {
2726
+ const run = this.getWorkflowRun(workflowRunId);
2727
+ if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
2728
+ return;
2729
+ const workItem = this.getWorkflowWorkItem(run.workItemId);
2730
+ if (!workItem || workItem.routeKey !== "todos-task")
2731
+ return;
2732
+ const invocation = this.getWorkflowInvocation(run.invocationId);
2733
+ if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
2734
+ return;
2735
+ const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
2736
+ const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
2737
+ if (!projectPath || !taskId)
2738
+ return;
2739
+ return {
2740
+ projectPath,
2741
+ taskId,
2742
+ invocationId: invocation.id,
2743
+ workflowRunId: run.id,
2744
+ manifestPath: run.manifestPath
2745
+ };
2746
+ }
2747
+ syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
2748
+ const context = this.taskLifecycleTodosPointerContext(workflowRunId);
2749
+ if (!context)
2750
+ return;
2751
+ const result = runLocalCommand("todos", [
2752
+ "--project",
2753
+ context.projectPath,
2754
+ "task",
2755
+ "workflow-pointers",
2756
+ context.taskId,
2757
+ "--clear",
2758
+ "--invocation",
2759
+ context.invocationId,
2760
+ "--run",
2761
+ context.workflowRunId,
2762
+ "--manifest",
2763
+ context.manifestPath,
2764
+ "--state",
2765
+ "succeeded",
2766
+ "--actor",
2767
+ "openloops:task-lifecycle"
2768
+ ]);
2769
+ this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
2770
+ projectPath: context.projectPath,
2771
+ taskId: context.taskId,
2772
+ invocationId: context.invocationId,
2773
+ workflowRunId: context.workflowRunId,
2774
+ manifestPath: context.manifestPath,
2775
+ mutation: todosMutationSummary(result)
2776
+ });
2777
+ }
2467
2778
  createWorkflowInvocation(input) {
2468
2779
  const now = nowIso();
2469
2780
  const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
@@ -3426,6 +3737,8 @@ class Store {
3426
3737
  const run = this.getWorkflowRun(workflowRunId);
3427
3738
  if (!run)
3428
3739
  throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
3740
+ if (changed && status === "succeeded")
3741
+ this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
3429
3742
  return run;
3430
3743
  }
3431
3744
  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -3487,6 +3800,17 @@ class Store {
3487
3800
  const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
3488
3801
  return (row?.count ?? 0) > 0;
3489
3802
  }
3803
+ hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
3804
+ const rows = this.db.query(`SELECT * FROM loop_runs
3805
+ WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
3806
+ return rows.some((row) => {
3807
+ if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
3808
+ return true;
3809
+ if (isRecordedProcessAlive(row.pid, row.process_started_at))
3810
+ return true;
3811
+ return this.hasLiveWorkflowStepProcesses(row.id);
3812
+ });
3813
+ }
3490
3814
  markRunPid(id, pid, claimedBy, opts = {}) {
3491
3815
  const now = (opts.now ?? new Date).toISOString();
3492
3816
  const startedMs = processStartTimeMs(pid);
@@ -3619,6 +3943,10 @@ class Store {
3619
3943
  loop = currentLoop;
3620
3944
  const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
3621
3945
  const existing = this.getRunBySlot(loop.id, scheduledFor);
3946
+ if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
3947
+ this.db.exec("COMMIT");
3948
+ return;
3949
+ }
3622
3950
  if (existing) {
3623
3951
  if (existing.status === "running") {
3624
3952
  if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
@@ -4190,9 +4518,9 @@ class Store {
4190
4518
  const runDir = dirname2(manifestPath);
4191
4519
  try {
4192
4520
  if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
4193
- rmSync2(runDir, { recursive: true, force: true });
4521
+ rmSync3(runDir, { recursive: true, force: true });
4194
4522
  } else {
4195
- rmSync2(manifestPath, { force: true });
4523
+ rmSync3(manifestPath, { force: true });
4196
4524
  }
4197
4525
  } catch {}
4198
4526
  }
@@ -4259,7 +4587,7 @@ class Store {
4259
4587
  this.db.close();
4260
4588
  if (this.memoryRootDir) {
4261
4589
  try {
4262
- rmSync2(this.memoryRootDir, { recursive: true, force: true });
4590
+ rmSync3(this.memoryRootDir, { recursive: true, force: true });
4263
4591
  } catch {}
4264
4592
  this.memoryRootDir = undefined;
4265
4593
  }
@@ -4268,7 +4596,7 @@ class Store {
4268
4596
  // package.json
4269
4597
  var package_default = {
4270
4598
  name: "@hasna/loops",
4271
- version: "0.4.13",
4599
+ version: "0.4.14",
4272
4600
  description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
4273
4601
  type: "module",
4274
4602
  main: "dist/index.js",
@@ -4277,6 +4605,7 @@ var package_default = {
4277
4605
  loops: "dist/cli/index.js",
4278
4606
  "loops-daemon": "dist/daemon/index.js",
4279
4607
  "loops-api": "dist/api/index.js",
4608
+ "loops-serve": "dist/serve/index.js",
4280
4609
  "loops-runner": "dist/runner/index.js",
4281
4610
  "loops-mcp": "dist/mcp/index.js"
4282
4611
  },
@@ -4289,6 +4618,14 @@ var package_default = {
4289
4618
  types: "./dist/sdk/index.d.ts",
4290
4619
  import: "./dist/sdk/index.js"
4291
4620
  },
4621
+ "./sdk/http": {
4622
+ types: "./dist/sdk/http.d.ts",
4623
+ import: "./dist/sdk/http.js"
4624
+ },
4625
+ "./serve": {
4626
+ types: "./dist/serve/index.d.ts",
4627
+ import: "./dist/serve/index.js"
4628
+ },
4292
4629
  "./mcp": {
4293
4630
  types: "./dist/mcp/index.d.ts",
4294
4631
  import: "./dist/mcp/index.js"
@@ -4334,7 +4671,7 @@ var package_default = {
4334
4671
  "LICENSE"
4335
4672
  ],
4336
4673
  scripts: {
4337
- build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
4674
+ build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
4338
4675
  "build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
4339
4676
  typecheck: "tsc --noEmit",
4340
4677
  test: "bun test",
@@ -4369,11 +4706,13 @@ var package_default = {
4369
4706
  bun: ">=1.0.0"
4370
4707
  },
4371
4708
  dependencies: {
4709
+ "@hasna/contracts": "^0.4.1",
4372
4710
  "@hasna/events": "^0.1.9",
4373
4711
  "@modelcontextprotocol/sdk": "^1.29.0",
4374
4712
  "@openrouter/ai-sdk-provider": "2.9.1",
4375
4713
  ai: "6.0.204",
4376
4714
  commander: "^13.1.0",
4715
+ pg: "^8.13.1",
4377
4716
  zod: "4.4.3"
4378
4717
  },
4379
4718
  optionalDependencies: {
@@ -4381,6 +4720,7 @@ var package_default = {
4381
4720
  },
4382
4721
  devDependencies: {
4383
4722
  "@types/bun": "latest",
4723
+ "@types/pg": "^8.11.10",
4384
4724
  typescript: "^5.7.3"
4385
4725
  },
4386
4726
  publishConfig: {
@@ -4594,10 +4934,10 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
4594
4934
  import { z as z2 } from "zod/v3";
4595
4935
 
4596
4936
  // src/daemon/control.ts
4597
- import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync3, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
4598
- import { spawnSync as spawnSync2 } from "child_process";
4937
+ import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
4938
+ import { spawnSync as spawnSync3 } from "child_process";
4599
4939
  import { hostname } from "os";
4600
- import { dirname as dirname3, join as join4 } from "path";
4940
+ import { dirname as dirname3, join as join5 } from "path";
4601
4941
 
4602
4942
  // src/daemon/loop.ts
4603
4943
  function realSleep(ms) {
@@ -4627,7 +4967,7 @@ function readPidRecord(path = pidFilePath()) {
4627
4967
  return;
4628
4968
  let raw;
4629
4969
  try {
4630
- raw = readFileSync3(path, "utf8").trim();
4970
+ raw = readFileSync4(path, "utf8").trim();
4631
4971
  } catch {
4632
4972
  return;
4633
4973
  }
@@ -4653,7 +4993,7 @@ function writePid(pid = process.pid, path = pidFilePath()) {
4653
4993
  writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
4654
4994
  }
4655
4995
  function removePid(path = pidFilePath()) {
4656
- rmSync3(path, { force: true });
4996
+ rmSync4(path, { force: true });
4657
4997
  }
4658
4998
  function isAlive(pid, startedAt) {
4659
4999
  try {
@@ -4683,7 +5023,7 @@ function ownProcessGroupId() {
4683
5023
  return pgrp;
4684
5024
  }
4685
5025
  try {
4686
- const run = spawnSync2("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
5026
+ const run = spawnSync3("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
4687
5027
  const pgid = Number(run.stdout.trim());
4688
5028
  if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
4689
5029
  return pgid;
@@ -4779,7 +5119,7 @@ async function stopDaemon(opts = {}) {
4779
5119
  if (!state.running || !state.pid)
4780
5120
  return { wasRunning: false, stopped: false, forced: false };
4781
5121
  const sleep = opts.sleep ?? realSleep;
4782
- const store = new Store(join4(dirname3(path), "loops.db"));
5122
+ const store = new Store(join5(dirname3(path), "loops.db"));
4783
5123
  try {
4784
5124
  const lease = store.getDaemonLease();
4785
5125
  const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
@@ -4817,18 +5157,18 @@ async function stopDaemon(opts = {}) {
4817
5157
  }
4818
5158
 
4819
5159
  // src/lib/doctor.ts
4820
- import { spawnSync as spawnSync5 } from "child_process";
5160
+ import { spawnSync as spawnSync6 } from "child_process";
4821
5161
  import { accessSync as accessSync2, constants as constants2 } from "fs";
4822
5162
 
4823
5163
  // src/lib/executor.ts
4824
- import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
5164
+ import { spawn as spawn2, spawnSync as spawnSync5 } from "child_process";
4825
5165
  import { randomBytes as randomBytes2 } from "crypto";
4826
5166
  import { once } from "events";
4827
5167
  import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
4828
5168
  import { dirname as dirname4, resolve as resolve2 } from "path";
4829
5169
 
4830
5170
  // src/lib/accounts.ts
4831
- import { spawnSync as spawnSync3 } from "child_process";
5171
+ import { spawnSync as spawnSync4 } from "child_process";
4832
5172
  import { existsSync as existsSync3 } from "fs";
4833
5173
  var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
4834
5174
  var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
@@ -4933,7 +5273,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
4933
5273
  if (!account)
4934
5274
  return {};
4935
5275
  const tool = requiredAccountTool(account, toolHint);
4936
- const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
5276
+ const result = spawnSync4("accounts", ["env", account.profile, "--tool", tool], {
4937
5277
  encoding: "utf8",
4938
5278
  env,
4939
5279
  stdio: ["ignore", "pipe", "pipe"],
@@ -4949,8 +5289,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
4949
5289
 
4950
5290
  // src/lib/env.ts
4951
5291
  import { accessSync, constants } from "fs";
4952
- import { homedir as homedir2 } from "os";
4953
- import { delimiter, join as join5 } from "path";
5292
+ import { homedir as homedir3 } from "os";
5293
+ import { delimiter, join as join6 } from "path";
4954
5294
  function compactPathParts(parts) {
4955
5295
  const seen = new Set;
4956
5296
  const result = [];
@@ -4964,16 +5304,16 @@ function compactPathParts(parts) {
4964
5304
  return result;
4965
5305
  }
4966
5306
  function commonExecutableDirs(env = process.env) {
4967
- const home = env.HOME || homedir2();
5307
+ const home = env.HOME || homedir3();
4968
5308
  return compactPathParts([
4969
- join5(home, ".local", "bin"),
4970
- join5(home, ".bun", "bin"),
4971
- join5(home, ".cargo", "bin"),
4972
- join5(home, ".npm-global", "bin"),
4973
- join5(home, "bin"),
4974
- env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
5309
+ join6(home, ".local", "bin"),
5310
+ join6(home, ".bun", "bin"),
5311
+ join6(home, ".cargo", "bin"),
5312
+ join6(home, ".npm-global", "bin"),
5313
+ join6(home, "bin"),
5314
+ env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
4975
5315
  env.PNPM_HOME,
4976
- env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
5316
+ env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
4977
5317
  "/opt/homebrew/bin",
4978
5318
  "/usr/local/bin",
4979
5319
  "/usr/bin",
@@ -4997,7 +5337,7 @@ function executableExists(command, env = process.env) {
4997
5337
  if (command.includes("/"))
4998
5338
  return isExecutable(command);
4999
5339
  for (const dir of (env.PATH ?? "").split(delimiter)) {
5000
- if (dir && isExecutable(join5(dir, command)))
5340
+ if (dir && isExecutable(join6(dir, command)))
5001
5341
  return true;
5002
5342
  }
5003
5343
  return false;
@@ -5138,6 +5478,32 @@ function timeoutResult(startedAt, error, fields = {}) {
5138
5478
  function successResult(startedAt, fields = {}) {
5139
5479
  return buildResult("succeeded", startedAt, fields);
5140
5480
  }
5481
+ function codewithJsonlHasTerminalSuccess(stdout) {
5482
+ for (const line of stdout.split(/\r?\n/)) {
5483
+ const trimmed = line.trim();
5484
+ if (!trimmed || !trimmed.startsWith("{"))
5485
+ continue;
5486
+ let event;
5487
+ try {
5488
+ event = JSON.parse(trimmed);
5489
+ } catch {
5490
+ continue;
5491
+ }
5492
+ if (!event || typeof event !== "object")
5493
+ continue;
5494
+ const record = event;
5495
+ if (record.type === "task_complete")
5496
+ return true;
5497
+ const payload = record.payload;
5498
+ if (payload && typeof payload === "object" && payload.type === "task_complete") {
5499
+ return true;
5500
+ }
5501
+ }
5502
+ return false;
5503
+ }
5504
+ function codewithJsonlReconciledSuccess(spec, fields) {
5505
+ return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
5506
+ }
5141
5507
  function notifySpawn(pid, opts) {
5142
5508
  if (!pid)
5143
5509
  return;
@@ -5153,10 +5519,48 @@ function shellQuote(value) {
5153
5519
  return `'${value.replace(/'/g, `'\\''`)}'`;
5154
5520
  }
5155
5521
  function codewithProfileCandidateFromLine(line) {
5156
- const cols = line.trim().split(/\s+/);
5157
- if (cols[0] === "*")
5158
- return cols[1];
5159
- return cols[0];
5522
+ const trimmed = line.trim();
5523
+ if (!trimmed || trimmed === "No auth profiles saved.")
5524
+ return;
5525
+ const cols = trimmed.split(/\s+/);
5526
+ const candidate = cols[0] === "*" ? cols[1] : cols[0];
5527
+ if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
5528
+ return;
5529
+ if (candidate === '"name":') {
5530
+ const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
5531
+ if (jsonName)
5532
+ return jsonName;
5533
+ }
5534
+ return candidate;
5535
+ }
5536
+ function codewithProfileCandidatesFromJson(value) {
5537
+ const candidates = [];
5538
+ const root = value && typeof value === "object" ? value : undefined;
5539
+ const addProfile = (entry) => {
5540
+ if (!entry || typeof entry !== "object")
5541
+ return;
5542
+ const name = entry.name;
5543
+ if (typeof name === "string" && name)
5544
+ candidates.push(name);
5545
+ };
5546
+ const data = root?.data;
5547
+ if (Array.isArray(data))
5548
+ data.forEach(addProfile);
5549
+ const profiles = root?.profiles;
5550
+ if (Array.isArray(profiles))
5551
+ profiles.forEach(addProfile);
5552
+ return candidates;
5553
+ }
5554
+ function codewithProfileCandidatesFromOutput(output) {
5555
+ const trimmed = output.trim();
5556
+ const jsonStart = trimmed.indexOf("{");
5557
+ const jsonEnd = trimmed.lastIndexOf("}");
5558
+ if (jsonStart >= 0 && jsonEnd >= jsonStart) {
5559
+ try {
5560
+ return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
5561
+ } catch {}
5562
+ }
5563
+ return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
5160
5564
  }
5161
5565
  function codewithProfileForError(profile) {
5162
5566
  return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
@@ -5249,6 +5653,7 @@ function commandSpec(target, opts) {
5249
5653
  preflightAnyOf: invocation.preflightAnyOf,
5250
5654
  stdin: invocation.stdin,
5251
5655
  allowlist: agentTarget.allowlist,
5656
+ agentProvider: agentTarget.provider,
5252
5657
  worktree: agentTarget.worktree
5253
5658
  };
5254
5659
  }
@@ -5408,7 +5813,7 @@ function remotePreflightScript(spec, metadata) {
5408
5813
  }
5409
5814
  if (spec.nativeAuthProfile?.provider === "codewith") {
5410
5815
  const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
5411
- lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", `__OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", "}", `if ! printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk 'NR > 1 { candidate = ($1 == "*" ? $2 : $1); if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'; then`, ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", "fi");
5816
+ lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
5412
5817
  }
5413
5818
  return lines.join(`
5414
5819
  `);
@@ -5426,6 +5831,12 @@ function transportEnv(opts) {
5426
5831
  return env;
5427
5832
  }
5428
5833
  function assertCodewithProfileListed(profile, result) {
5834
+ const profiles = codewithProfileSetFromResult(result);
5835
+ if (!profiles.has(profile)) {
5836
+ throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
5837
+ }
5838
+ }
5839
+ function codewithProfileSetFromResult(result) {
5429
5840
  if (result.error) {
5430
5841
  throw new Error(`codewith auth profile preflight failed: ${result.error}`);
5431
5842
  }
@@ -5433,32 +5844,42 @@ function assertCodewithProfileListed(profile, result) {
5433
5844
  const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
5434
5845
  throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
5435
5846
  }
5436
- const profiles = new Set((result.stdout || "").split(/\r?\n/).slice(1).map(codewithProfileCandidateFromLine).filter(Boolean));
5437
- if (!profiles.has(profile)) {
5438
- throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
5439
- }
5847
+ return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
5440
5848
  }
5441
5849
  function preflightNativeAuthProfileSync(spec, env) {
5442
5850
  if (spec.nativeAuthProfile?.provider !== "codewith")
5443
5851
  return;
5444
- const result = spawnSync4(spec.command, ["profile", "list"], {
5445
- encoding: "utf8",
5446
- env,
5447
- stdio: ["ignore", "pipe", "pipe"],
5448
- timeout: 15000
5449
- });
5450
- assertCodewithProfileListed(spec.nativeAuthProfile.profile, {
5451
- status: result.status,
5452
- stdout: result.stdout ?? "",
5453
- stderr: result.stderr ?? "",
5454
- error: result.error?.message
5455
- });
5852
+ const runProfileList = (args) => {
5853
+ const result = spawnSync5(spec.command, args, {
5854
+ encoding: "utf8",
5855
+ env,
5856
+ stdio: ["ignore", "pipe", "pipe"],
5857
+ timeout: 15000
5858
+ });
5859
+ return {
5860
+ status: result.status,
5861
+ stdout: result.stdout ?? "",
5862
+ stderr: result.stderr ?? "",
5863
+ error: result.error?.message
5864
+ };
5865
+ };
5866
+ const jsonResult = runProfileList(["profile", "list", "--json"]);
5867
+ try {
5868
+ if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
5869
+ return;
5870
+ } catch {}
5871
+ assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
5456
5872
  }
5457
5873
  async function preflightNativeAuthProfile(spec, env) {
5458
5874
  if (spec.nativeAuthProfile?.provider !== "codewith")
5459
5875
  return;
5460
- const result = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
5461
- assertCodewithProfileListed(spec.nativeAuthProfile.profile, result);
5876
+ const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
5877
+ try {
5878
+ if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
5879
+ return;
5880
+ } catch {}
5881
+ const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
5882
+ assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
5462
5883
  }
5463
5884
  function resolvedDirEquals(left, right) {
5464
5885
  try {
@@ -5561,7 +5982,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
5561
5982
  }
5562
5983
  function preflightRemoteSpec(spec, machine, metadata, opts) {
5563
5984
  const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
5564
- const result = spawnSync4(plan.command, plan.args, {
5985
+ const result = spawnSync5(plan.command, plan.args, {
5565
5986
  encoding: "utf8",
5566
5987
  env: transportEnv(opts),
5567
5988
  input: remotePreflightScript(spec, metadata),
@@ -5660,6 +6081,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
5660
6081
  if (timedOut || idleTimedOut) {
5661
6082
  return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
5662
6083
  }
6084
+ if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
6085
+ return successResult(startedAt, fields);
6086
+ }
5663
6087
  if (error || exitCode !== 0) {
5664
6088
  return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
5665
6089
  }
@@ -5795,6 +6219,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
5795
6219
  if (timedOut || idleTimedOut) {
5796
6220
  return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
5797
6221
  }
6222
+ if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
6223
+ return successResult(startedAt, fields);
6224
+ }
5798
6225
  if (error || exitCode !== 0) {
5799
6226
  return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
5800
6227
  }
@@ -5825,239 +6252,20 @@ async function executeLoop(loop, run, opts = {}) {
5825
6252
  }, { ...opts, machine: opts.machine ?? loop.machine });
5826
6253
  }
5827
6254
 
5828
- // src/lib/doctor.ts
5829
- var PROVIDER_COMMANDS = [
5830
- "claude",
5831
- "agent",
5832
- "codewith",
5833
- "aicopilot",
5834
- "opencode",
5835
- "codex"
5836
- ];
5837
- function hasCommand(command) {
5838
- const result = spawnSync5("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
5839
- return (result.status ?? 1) === 0;
5840
- }
5841
- function commandVersion(command) {
5842
- const result = spawnSync5(command, ["--version"], {
5843
- encoding: "utf8",
5844
- stdio: ["ignore", "pipe", "pipe"]
5845
- });
5846
- if ((result.status ?? 1) !== 0)
5847
- return;
5848
- return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
5849
- }
5850
- function runDoctor(store) {
5851
- const checks = [];
5852
- try {
5853
- const dir = ensureDataDir();
5854
- accessSync2(dir, constants2.R_OK | constants2.W_OK);
5855
- checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
5856
- } catch (error) {
5857
- checks.push({
5858
- id: "data-dir",
5859
- status: "fail",
5860
- message: "data directory is not writable",
5861
- detail: error instanceof Error ? error.message : String(error)
5862
- });
5863
- }
5864
- const bunVersion = commandVersion("bun");
5865
- checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
5866
- const accountsVersion = commandVersion("accounts");
5867
- checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
5868
- try {
5869
- const machines = listOpenMachines();
5870
- const local = machines.find((machine) => machine.local);
5871
- checks.push({
5872
- id: "machines",
5873
- status: "ok",
5874
- message: `OpenMachines topology available (${machines.length} machine(s))`,
5875
- detail: local ? `local=${local.id}` : undefined
5876
- });
5877
- } catch (error) {
5878
- checks.push({
5879
- id: "machines",
5880
- status: "warn",
5881
- message: "OpenMachines topology is not available; machine-assigned loops will fail",
5882
- detail: error instanceof Error ? error.message : String(error)
5883
- });
5884
- }
5885
- for (const command of PROVIDER_COMMANDS) {
5886
- checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
5887
- }
5888
- const status = daemonStatus(store);
5889
- checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
5890
- const failedRuns = store.countRuns("failed");
5891
- checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
5892
- const deployment = buildDeploymentStatus();
5893
- const schedulerState = deployment.schedulerState;
5894
- checks.push({
5895
- id: "scheduler-state",
5896
- status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
5897
- message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
5898
- detail: [
5899
- `route_state=${schedulerState.routeAdmission.stateStore}`,
5900
- `active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
5901
- `gates=${schedulerState.routeAdmission.gates.join(",")}`,
5902
- `artifacts=${schedulerState.localStore.runArtifacts}`,
5903
- `remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
5904
- `remote_apply=${String(schedulerState.remoteStore.applySupported)}`
5905
- ].join(" ")
5906
- });
5907
- for (const loop of store.listLoops({ status: "active" })) {
5908
- try {
5909
- if (loop.target.type === "workflow") {
5910
- const workflow = store.requireWorkflow(loop.target.workflowId);
5911
- for (const step of workflowExecutionOrder(workflow)) {
5912
- preflightTarget({
5913
- ...step.target,
5914
- account: step.account ?? step.target.account,
5915
- timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
5916
- }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
5917
- }
5918
- } else {
5919
- preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
5920
- }
5921
- checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
5922
- } catch (error) {
5923
- checks.push({
5924
- id: `loop:${loop.id}:preflight`,
5925
- status: "fail",
5926
- message: `active loop target preflight failed: ${loop.name}`,
5927
- detail: error instanceof Error ? error.message : String(error)
5928
- });
5929
- }
5930
- }
5931
- return {
5932
- ok: checks.every((check) => check.status !== "fail"),
5933
- checks
5934
- };
5935
- }
5936
-
5937
- // src/lib/format.ts
5938
- var TEXT_OUTPUT_LIMIT = 32 * 1024;
5939
- var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
5940
- function redact(value, visible = 0) {
5941
- if (!value)
5942
- return value;
5943
- if (value.length <= visible)
5944
- return value;
5945
- if (visible <= 0)
5946
- return `[redacted ${value.length} chars]`;
5947
- return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
5948
- }
5949
- function truncateTextOutput(value) {
5950
- if (value.length <= TEXT_OUTPUT_LIMIT)
5951
- return value;
5952
- return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
5953
- [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
5954
- }
5955
- function redactSensitivePayload(value, key) {
5956
- if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
5957
- if (typeof value === "string")
5958
- return redact(value);
5959
- if (value === undefined || value === null)
5960
- return value;
5961
- return "[redacted]";
5962
- }
5963
- if (Array.isArray(value))
5964
- return value.map((item) => redactSensitivePayload(item));
5965
- if (value && typeof value === "object") {
5966
- return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
5967
- }
5968
- return value;
5969
- }
5970
- function textOutputBlocks(value, opts = {}) {
5971
- const indent = opts.indent ?? "";
5972
- const nested = `${indent} `;
5973
- const blocks = [];
5974
- for (const [label, output] of [
5975
- ["stdout", value.stdout],
5976
- ["stderr", value.stderr]
5977
- ]) {
5978
- if (!output)
5979
- continue;
5980
- blocks.push(`${indent}${label}:`);
5981
- for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
5982
- blocks.push(`${nested}${line}`);
5983
- }
5984
- }
5985
- return blocks;
5986
- }
5987
- function publicLoop(loop) {
5988
- const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
5989
- return {
5990
- ...loop,
5991
- target
5992
- };
5993
- }
5994
- function publicRun(run, showOutput = false, opts = {}) {
5995
- return {
5996
- ...run,
5997
- stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
5998
- stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
5999
- error: opts.redactError ? redact(run.error) : run.error
6000
- };
6001
- }
6002
- function publicExecutorResult(result, showOutput = false) {
6003
- return {
6004
- ...result,
6005
- stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
6006
- stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
6007
- error: redact(result.error)
6008
- };
6009
- }
6010
- function publicWorkflow(workflow) {
6011
- return {
6012
- ...workflow,
6013
- steps: workflow.steps.map((step) => ({
6014
- ...step,
6015
- target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
6016
- }))
6017
- };
6018
- }
6019
- function publicWorkflowRun(run) {
6020
- return { ...run, error: redact(run.error) };
6021
- }
6022
- function publicWorkflowInvocation(invocation) {
6023
- return redactSensitivePayload(invocation);
6024
- }
6025
- function publicWorkflowWorkItem(item) {
6026
- return { ...item, lastReason: redact(item.lastReason, 240) };
6027
- }
6028
- function publicWorkflowStepRun(run, showOutput = false) {
6029
- return {
6030
- ...run,
6031
- stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
6032
- stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
6033
- error: redact(run.error)
6034
- };
6035
- }
6036
- function publicWorkflowEvent(event) {
6037
- return { ...event, payload: redactSensitivePayload(event.payload) };
6038
- }
6039
- function publicGoal(goal) {
6040
- return {
6041
- ...goal,
6042
- objective: redact(goal.objective, 120)
6043
- };
6044
- }
6045
- function publicGoalRun(run) {
6046
- return {
6047
- ...run,
6048
- evidence: redactSensitivePayload(run.evidence),
6049
- rawResponse: redactSensitivePayload(run.rawResponse)
6050
- };
6051
- }
6052
-
6053
6255
  // src/lib/health.ts
6054
6256
  import { createHash as createHash2 } from "crypto";
6055
- import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
6257
+ import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
6258
+ import { join as join7 } from "path";
6056
6259
  var EVIDENCE_CHARS = 2000;
6057
6260
  var FINGERPRINT_EVIDENCE_CHARS = 120;
6261
+ var DEFAULT_SCAN_LIMIT = 200;
6262
+ var DEFAULT_SCAN_STATUSES = ["active", "paused"];
6263
+ var DEFAULT_MAX_FINDINGS = 100;
6264
+ var MIN_STALE_RUNNING_MS = 10 * 60000;
6058
6265
  var CLASSIFICATIONS = [
6059
6266
  "rate_limit",
6060
6267
  "auth",
6268
+ "provider_unavailable",
6061
6269
  "model_not_found",
6062
6270
  "context_length",
6063
6271
  "schema_response_format",
@@ -6066,10 +6274,12 @@ var CLASSIFICATIONS = [
6066
6274
  "route_functional",
6067
6275
  "timeout",
6068
6276
  "sigsegv",
6277
+ "restart_interrupted",
6069
6278
  "skipped_previous_active",
6070
6279
  "circuit_breaker",
6071
6280
  "unknown"
6072
6281
  ];
6282
+ var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
6073
6283
  function bounded(value, limit = EVIDENCE_CHARS) {
6074
6284
  if (!value)
6075
6285
  return;
@@ -6083,7 +6293,7 @@ function redactedEvidence(value) {
6083
6293
  }
6084
6294
  function searchableText(run) {
6085
6295
  return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
6086
- `).toLowerCase();
6296
+ `);
6087
6297
  }
6088
6298
  function isRecord(value) {
6089
6299
  return Boolean(value && typeof value === "object" && !Array.isArray(value));
@@ -6108,13 +6318,36 @@ function stableFingerprint(parts) {
6108
6318
  return createHash2("sha256").update(parts.join(`
6109
6319
  `)).digest("hex").slice(0, 16);
6110
6320
  }
6111
- function stableFailureFingerprint(run, classification) {
6321
+ function stableScanFingerprint(parts) {
6322
+ return `openloops:health-scan:${stableFingerprint(parts)}`;
6323
+ }
6324
+ function safeHost(value) {
6325
+ if (!value)
6326
+ return;
6327
+ let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
6328
+ host = host.replace(/^\[|\]$/g, "");
6329
+ return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
6330
+ }
6331
+ function isCursorHost(host) {
6332
+ return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
6333
+ }
6334
+ function providerUnavailableSummary(rawText) {
6335
+ const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
6336
+ if (dns) {
6337
+ const host = safeHost(dns[2]);
6338
+ if (isCursorHost(host))
6339
+ return `provider DNS lookup failed: ${dns[1]} ${host}`;
6340
+ }
6341
+ return;
6342
+ }
6343
+ function stableFailureFingerprint(run, classification, summary) {
6344
+ const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
6112
6345
  return stableFingerprint([
6113
6346
  run.loopId,
6114
6347
  classification,
6115
6348
  String(run.status),
6116
6349
  String(run.exitCode ?? ""),
6117
- (run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6350
+ evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6118
6351
  ]);
6119
6352
  }
6120
6353
  function stableRouteFunctionalFingerprint(loop, reason) {
@@ -6132,13 +6365,279 @@ function healthRun(run) {
6132
6365
  stderr: redactedEvidence(run.stderr)
6133
6366
  };
6134
6367
  }
6368
+ function compactText(value, limit = 500) {
6369
+ const text = redact(bounded(value, limit));
6370
+ return text?.replace(/\s+/g, " ").trim();
6371
+ }
6372
+ function publicDoctorCheck(check) {
6373
+ return {
6374
+ id: check.id,
6375
+ status: check.status,
6376
+ message: redact(bounded(check.message, 500)) ?? "",
6377
+ detail: redact(bounded(check.detail, 800))
6378
+ };
6379
+ }
6380
+ function publicDoctorReport(report) {
6381
+ return {
6382
+ ok: report.ok,
6383
+ checks: report.checks.map(publicDoctorCheck)
6384
+ };
6385
+ }
6386
+ function includedStatusSet(statuses) {
6387
+ const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
6388
+ return new Set(values);
6389
+ }
6390
+ function statusCounts(loops) {
6391
+ return {
6392
+ loops: loops.length,
6393
+ active: loops.filter((loop) => loop.status === "active").length,
6394
+ paused: loops.filter((loop) => loop.status === "paused").length,
6395
+ stopped: loops.filter((loop) => loop.status === "stopped").length,
6396
+ expired: loops.filter((loop) => loop.status === "expired").length
6397
+ };
6398
+ }
6399
+ function compareLoopsForScan(left, right) {
6400
+ const statusOrder = left.status.localeCompare(right.status);
6401
+ if (statusOrder !== 0)
6402
+ return statusOrder;
6403
+ return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
6404
+ }
6405
+ function scanLoops(store, statuses, opts) {
6406
+ const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
6407
+ return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
6408
+ }
6409
+ function healthReportForLoops(store, loops, generatedAt) {
6410
+ const expectations = loops.map((loop) => expectationForLoop(store, loop));
6411
+ const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
6412
+ for (const expectation of expectations) {
6413
+ if (expectation.failure)
6414
+ classifications[expectation.failure.classification] += 1;
6415
+ }
6416
+ const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
6417
+ const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
6418
+ return {
6419
+ ok: unhealthy === 0,
6420
+ generatedAt,
6421
+ summary: {
6422
+ loops: expectations.length,
6423
+ healthy: expectations.length - unhealthy,
6424
+ unhealthy,
6425
+ warnings
6426
+ },
6427
+ classifications,
6428
+ expectations
6429
+ };
6430
+ }
6431
+ function ageMs(run, now) {
6432
+ const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
6433
+ const time = Date.parse(stamp);
6434
+ return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
6435
+ }
6436
+ function shortLoop(loop) {
6437
+ return {
6438
+ id: loop.id,
6439
+ name: loop.name,
6440
+ status: loop.status,
6441
+ nextRunAt: loop.nextRunAt,
6442
+ leaseMs: loop.leaseMs
6443
+ };
6444
+ }
6445
+ function priorityForSeverity(severity) {
6446
+ if (severity === "critical")
6447
+ return "critical";
6448
+ if (severity === "high")
6449
+ return "high";
6450
+ if (severity === "low")
6451
+ return "low";
6452
+ return "medium";
6453
+ }
6454
+ function recommendedFindingTask(finding, route) {
6455
+ const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
6456
+ if (finding.classification)
6457
+ tags.push(finding.classification);
6458
+ const description = [
6459
+ `OpenLoops health scan found a ${finding.kind} issue.`,
6460
+ finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
6461
+ finding.run ? `Run: ${finding.run.id}` : undefined,
6462
+ finding.classification ? `Classification: ${finding.classification}` : undefined,
6463
+ finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
6464
+ finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
6465
+ `Fingerprint: ${finding.fingerprint}`,
6466
+ `Severity: ${finding.severity}`,
6467
+ `No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
6468
+ route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
6469
+ route?.provider ? `Provider: ${route.provider}` : undefined,
6470
+ "",
6471
+ finding.message,
6472
+ finding.run?.error ? `Error:
6473
+ ${finding.run.error}` : undefined,
6474
+ finding.run?.stderr ? `Stderr:
6475
+ ${finding.run.stderr}` : undefined
6476
+ ].filter(Boolean).join(`
6477
+
6478
+ `);
6479
+ return {
6480
+ title: finding.title,
6481
+ description,
6482
+ priority: priorityForSeverity(finding.severity),
6483
+ tags,
6484
+ dedupeKey: finding.fingerprint,
6485
+ search: { query: finding.fingerprint },
6486
+ compatibilityFallback: {
6487
+ search: ["todos", "search", finding.fingerprint, "--json"],
6488
+ add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
6489
+ comment: ["todos", "comment", "<task-id>", description]
6490
+ },
6491
+ futureNativeUpsert: {
6492
+ command: "todos task upsert",
6493
+ fields: {
6494
+ title: finding.title,
6495
+ description,
6496
+ priority: priorityForSeverity(finding.severity),
6497
+ tags,
6498
+ dedupeKey: finding.fingerprint,
6499
+ routeSource: route?.source ?? "openloops",
6500
+ routeKind: route?.kind ?? "health_scan",
6501
+ routeLoopId: finding.loop?.id ?? "",
6502
+ routeLoopName: finding.loop?.name ?? ""
6503
+ }
6504
+ }
6505
+ };
6506
+ }
6507
+ function daemonFinding(daemon) {
6508
+ if (daemon.running && !daemon.stale)
6509
+ return;
6510
+ const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
6511
+ const severity = "critical";
6512
+ const finding = {
6513
+ kind: "daemon",
6514
+ severity,
6515
+ fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
6516
+ title: "OpenLoops daemon health issue",
6517
+ message: reason
6518
+ };
6519
+ return {
6520
+ ...finding,
6521
+ recommendedTask: recommendedFindingTask(finding, undefined)
6522
+ };
6523
+ }
6524
+ function doctorSeverity(check) {
6525
+ if (check.status === "fail" && check.id === "data-dir")
6526
+ return "critical";
6527
+ if (check.status === "fail")
6528
+ return "high";
6529
+ return "medium";
6530
+ }
6531
+ function doctorFinding(check, loop, route) {
6532
+ if (check.status === "ok" || check.id === "loop-runs")
6533
+ return;
6534
+ const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
6535
+ const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
6536
+ const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
6537
+ const finding = {
6538
+ kind,
6539
+ severity,
6540
+ fingerprint,
6541
+ title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
6542
+ message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
6543
+ loop: loop ? shortLoop(loop) : undefined,
6544
+ route,
6545
+ doctorCheck: publicDoctorCheck(check)
6546
+ };
6547
+ return {
6548
+ ...finding,
6549
+ recommendedTask: recommendedFindingTask(finding, route)
6550
+ };
6551
+ }
6552
+ function latestRunFinding(expectation) {
6553
+ if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
6554
+ return;
6555
+ const failure = expectation.failure;
6556
+ if (!failure)
6557
+ return;
6558
+ const severity = expectation.loop.status === "active" ? "high" : "medium";
6559
+ return {
6560
+ kind: "latest-run",
6561
+ severity,
6562
+ fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
6563
+ title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
6564
+ message: expectation.check.message,
6565
+ loop: expectation.loop,
6566
+ run: expectation.latestRun,
6567
+ route: expectation.route,
6568
+ classification: failure.classification,
6569
+ doctorCheck: undefined,
6570
+ recommendedTask: expectation.recommendedTask
6571
+ };
6572
+ }
6573
+ function staleRunningFinding(loop, expectation, now, staleRunningMs) {
6574
+ const run = expectation.latestRun;
6575
+ if (loop.status !== "active" || run?.status !== "running")
6576
+ return;
6577
+ const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
6578
+ const age = ageMs(run, now);
6579
+ if (age <= threshold)
6580
+ return;
6581
+ const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
6582
+ const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
6583
+ const finding = {
6584
+ kind: "stale-running",
6585
+ severity: "critical",
6586
+ fingerprint,
6587
+ title: `OpenLoops stale running run - ${loop.name}`,
6588
+ message,
6589
+ loop: shortLoop(loop),
6590
+ run,
6591
+ route: expectation.route,
6592
+ ageMs: age,
6593
+ staleThresholdMs: threshold
6594
+ };
6595
+ return {
6596
+ ...finding,
6597
+ recommendedTask: recommendedFindingTask(finding, expectation.route)
6598
+ };
6599
+ }
6600
+ function scanStatus(findings) {
6601
+ if (findings.some((finding) => finding.severity === "critical"))
6602
+ return "critical";
6603
+ if (findings.length > 0)
6604
+ return "degraded";
6605
+ return "ok";
6606
+ }
6607
+ function timestampDir(root, generatedAt) {
6608
+ const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
6609
+ return join7(root, stamp);
6610
+ }
6611
+ function healthScanMarkdown(scan) {
6612
+ return [
6613
+ "# OpenLoops Health Scan",
6614
+ "",
6615
+ `- status: ${scan.status}`,
6616
+ `- generated_at: ${scan.generatedAt}`,
6617
+ `- included_statuses: ${scan.includedStatuses.join(",")}`,
6618
+ `- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
6619
+ `- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
6620
+ scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
6621
+ scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
6622
+ scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
6623
+ "",
6624
+ "## Findings",
6625
+ scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
6626
+ `) : "None."
6627
+ ].join(`
6628
+ `);
6629
+ }
6135
6630
  function classifyRunFailure(run) {
6136
6631
  if (run.status === "succeeded" || run.status === "running")
6137
6632
  return;
6138
- const text = searchableText(run);
6633
+ const rawText = searchableText(run);
6634
+ const text = rawText.toLowerCase();
6139
6635
  let classification = "unknown";
6636
+ let summary;
6140
6637
  if (run.status === "timed_out")
6141
6638
  classification = "timeout";
6639
+ else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
6640
+ classification = "restart_interrupted";
6142
6641
  else if (run.status === "skipped" && /circuit breaker open/.test(text))
6143
6642
  classification = "circuit_breaker";
6144
6643
  else if (run.status === "skipped" && /previous run still active/.test(text))
@@ -6147,8 +6646,10 @@ function classifyRunFailure(run) {
6147
6646
  classification = "preflight";
6148
6647
  else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
6149
6648
  classification = "rate_limit";
6150
- else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
6649
+ else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
6151
6650
  classification = "auth";
6651
+ else if (summary = providerUnavailableSummary(rawText))
6652
+ classification = "provider_unavailable";
6152
6653
  else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
6153
6654
  classification = "model_not_found";
6154
6655
  else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
@@ -6161,8 +6662,9 @@ function classifyRunFailure(run) {
6161
6662
  classification = "sigsegv";
6162
6663
  return {
6163
6664
  classification,
6164
- fingerprint: stableFailureFingerprint(run, classification),
6665
+ fingerprint: stableFailureFingerprint(run, classification, summary),
6165
6666
  evidence: {
6667
+ summary,
6166
6668
  error: redactedEvidence(run.error),
6167
6669
  stdout: redactedEvidence(run.stdout),
6168
6670
  stderr: redactedEvidence(run.stderr),
@@ -6207,7 +6709,7 @@ function routeEvidenceReport(run) {
6207
6709
  const evidencePath = stringValue(stdoutReport?.evidencePath);
6208
6710
  if (evidencePath && existsSync4(evidencePath)) {
6209
6711
  try {
6210
- return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
6712
+ return parseJsonObject(readFileSync5(evidencePath, "utf8")) ?? stdoutReport;
6211
6713
  } catch {
6212
6714
  return stdoutReport;
6213
6715
  }
@@ -6339,6 +6841,12 @@ function targetRoute(loop) {
6339
6841
  loopName: loop.name
6340
6842
  };
6341
6843
  }
6844
+ function expectationLoop(loop) {
6845
+ return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
6846
+ }
6847
+ function hasPendingRetry(loop, run) {
6848
+ return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
6849
+ }
6342
6850
  function recommendedTask(loop, run, failure, route) {
6343
6851
  const title = `BUG: open-loops loop failure - ${loop.name}`;
6344
6852
  const description = [
@@ -6350,6 +6858,7 @@ function recommendedTask(loop, run, failure, route) {
6350
6858
  `No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
6351
6859
  route.cwd ? `Route cwd: ${route.cwd}` : undefined,
6352
6860
  route.provider ? `Provider: ${route.provider}` : undefined,
6861
+ failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
6353
6862
  failure.evidence.error ? `Error:
6354
6863
  ${failure.evidence.error}` : undefined,
6355
6864
  failure.evidence.stderr ? `Stderr:
@@ -6359,7 +6868,7 @@ ${failure.evidence.stderr}` : undefined
6359
6868
  `);
6360
6869
  const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
6361
6870
  const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
6362
- const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
6871
+ const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
6363
6872
  return {
6364
6873
  title,
6365
6874
  description,
@@ -6393,7 +6902,7 @@ function expectationForLoop(store, loop) {
6393
6902
  const route = targetRoute(loop);
6394
6903
  if (!latestRun) {
6395
6904
  return {
6396
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6905
+ loop: expectationLoop(loop),
6397
6906
  ok: true,
6398
6907
  check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
6399
6908
  route
@@ -6402,7 +6911,7 @@ function expectationForLoop(store, loop) {
6402
6911
  const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
6403
6912
  if (routeFailure) {
6404
6913
  return {
6405
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6914
+ loop: expectationLoop(loop),
6406
6915
  ok: false,
6407
6916
  check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
6408
6917
  latestRun: healthRun(latestRun),
@@ -6413,7 +6922,7 @@ function expectationForLoop(store, loop) {
6413
6922
  }
6414
6923
  if (latestRun.status === "succeeded") {
6415
6924
  return {
6416
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6925
+ loop: expectationLoop(loop),
6417
6926
  ok: true,
6418
6927
  check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
6419
6928
  latestRun: healthRun(latestRun),
@@ -6424,7 +6933,7 @@ function expectationForLoop(store, loop) {
6424
6933
  if (failure?.classification === "circuit_breaker") {
6425
6934
  if (loop.status !== "paused") {
6426
6935
  return {
6427
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6936
+ loop: expectationLoop(loop),
6428
6937
  ok: true,
6429
6938
  check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
6430
6939
  latestRun: healthRun(latestRun),
@@ -6432,7 +6941,7 @@ function expectationForLoop(store, loop) {
6432
6941
  };
6433
6942
  }
6434
6943
  return {
6435
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6944
+ loop: expectationLoop(loop),
6436
6945
  ok: false,
6437
6946
  check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
6438
6947
  latestRun: healthRun(latestRun),
@@ -6441,8 +6950,33 @@ function expectationForLoop(store, loop) {
6441
6950
  recommendedTask: recommendedTask(loop, latestRun, failure, route)
6442
6951
  };
6443
6952
  }
6953
+ if (failure?.classification === "restart_interrupted") {
6954
+ return {
6955
+ loop: expectationLoop(loop),
6956
+ ok: true,
6957
+ check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
6958
+ latestRun: healthRun(latestRun),
6959
+ failure,
6960
+ route
6961
+ };
6962
+ }
6963
+ if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
6964
+ const message = [
6965
+ "provider unavailable/network failure; retry is scheduled",
6966
+ loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
6967
+ failure.evidence.summary
6968
+ ].filter(Boolean).join("; ");
6969
+ return {
6970
+ loop: expectationLoop(loop),
6971
+ ok: true,
6972
+ check: { id: "latest-run-succeeded", status: "warn", message },
6973
+ latestRun: healthRun(latestRun),
6974
+ failure,
6975
+ route
6976
+ };
6977
+ }
6444
6978
  return {
6445
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6979
+ loop: expectationLoop(loop),
6446
6980
  ok: false,
6447
6981
  check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
6448
6982
  latestRun: healthRun(latestRun),
@@ -6474,6 +7008,206 @@ function buildHealthReport(store, opts = {}) {
6474
7008
  expectations
6475
7009
  };
6476
7010
  }
7011
+ function buildHealthScan(store, opts = {}) {
7012
+ const generatedAt = (opts.now ?? new Date).toISOString();
7013
+ const now = opts.now ?? new Date(generatedAt);
7014
+ const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
7015
+ const loops = scanLoops(store, includeStatuses, opts);
7016
+ const health = healthReportForLoops(store, loops, generatedAt);
7017
+ const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
7018
+ const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
7019
+ const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
7020
+ const allFindings = [];
7021
+ const pushFinding = (finding) => {
7022
+ if (!finding)
7023
+ return;
7024
+ allFindings.push(finding);
7025
+ };
7026
+ if (opts.daemon)
7027
+ pushFinding(daemonFinding(opts.daemon));
7028
+ if (opts.latestRun !== false) {
7029
+ for (const loop of loops) {
7030
+ const expectation = expectationsByLoopId.get(loop.id);
7031
+ if (!expectation)
7032
+ continue;
7033
+ pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
7034
+ pushFinding(latestRunFinding(expectation));
7035
+ }
7036
+ }
7037
+ const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
7038
+ if (doctor) {
7039
+ for (const check of doctor.checks) {
7040
+ const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
7041
+ const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
7042
+ const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
7043
+ pushFinding(doctorFinding(check, loop, route));
7044
+ }
7045
+ }
7046
+ const findings = allFindings.slice(0, maxFindings);
7047
+ const status = scanStatus(allFindings);
7048
+ const baseCounts = statusCounts(loops);
7049
+ return {
7050
+ ok: status === "ok",
7051
+ status,
7052
+ generatedAt,
7053
+ includedStatuses: includeStatuses,
7054
+ counts: {
7055
+ ...baseCounts,
7056
+ latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
7057
+ staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
7058
+ daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
7059
+ doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
7060
+ preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
7061
+ findings: allFindings.length,
7062
+ reportedFindings: findings.length,
7063
+ truncatedFindings: Math.max(0, allFindings.length - findings.length)
7064
+ },
7065
+ daemon: opts.daemon ? {
7066
+ running: opts.daemon.running,
7067
+ stale: opts.daemon.stale,
7068
+ pid: opts.daemon.pid,
7069
+ host: opts.daemon.host,
7070
+ loops: opts.daemon.loops,
7071
+ runs: opts.daemon.runs,
7072
+ logPath: opts.daemon.logPath
7073
+ } : undefined,
7074
+ doctor,
7075
+ health,
7076
+ selfHeals: opts.selfHeals ?? [],
7077
+ findings
7078
+ };
7079
+ }
7080
+ function writeHealthScanReports(scan, opts = {}) {
7081
+ const root = opts.reportDir ?? join7(dataDir(), "reports", "health-scan");
7082
+ const dir = timestampDir(root, scan.generatedAt);
7083
+ mkdirSync6(dir, { recursive: true, mode: 448 });
7084
+ const json = join7(dir, "summary.json");
7085
+ const markdown = join7(dir, "report.md");
7086
+ const withReports = {
7087
+ ...scan,
7088
+ reports: { dir, json, markdown }
7089
+ };
7090
+ writeFileSync3(json, JSON.stringify(withReports, null, 2), { mode: 384 });
7091
+ writeFileSync3(markdown, healthScanMarkdown(withReports), { mode: 384 });
7092
+ return withReports;
7093
+ }
7094
+
7095
+ // src/lib/doctor.ts
7096
+ var PROVIDER_COMMANDS = [
7097
+ "claude",
7098
+ "agent",
7099
+ "codewith",
7100
+ "aicopilot",
7101
+ "opencode",
7102
+ "codex"
7103
+ ];
7104
+ function hasCommand(command) {
7105
+ const result = spawnSync6("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
7106
+ return (result.status ?? 1) === 0;
7107
+ }
7108
+ function commandVersion(command) {
7109
+ const result = spawnSync6(command, ["--version"], {
7110
+ encoding: "utf8",
7111
+ stdio: ["ignore", "pipe", "pipe"]
7112
+ });
7113
+ if ((result.status ?? 1) !== 0)
7114
+ return;
7115
+ return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
7116
+ }
7117
+ function runDoctor(store) {
7118
+ const checks = [];
7119
+ try {
7120
+ const dir = ensureDataDir();
7121
+ accessSync2(dir, constants2.R_OK | constants2.W_OK);
7122
+ checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
7123
+ } catch (error) {
7124
+ checks.push({
7125
+ id: "data-dir",
7126
+ status: "fail",
7127
+ message: "data directory is not writable",
7128
+ detail: error instanceof Error ? error.message : String(error)
7129
+ });
7130
+ }
7131
+ const bunVersion = commandVersion("bun");
7132
+ checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
7133
+ const accountsVersion = commandVersion("accounts");
7134
+ checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
7135
+ try {
7136
+ const machines = listOpenMachines();
7137
+ const local = machines.find((machine) => machine.local);
7138
+ checks.push({
7139
+ id: "machines",
7140
+ status: "ok",
7141
+ message: `OpenMachines topology available (${machines.length} machine(s))`,
7142
+ detail: local ? `local=${local.id}` : undefined
7143
+ });
7144
+ } catch (error) {
7145
+ checks.push({
7146
+ id: "machines",
7147
+ status: "warn",
7148
+ message: "OpenMachines topology is not available; machine-assigned loops will fail",
7149
+ detail: error instanceof Error ? error.message : String(error)
7150
+ });
7151
+ }
7152
+ for (const command of PROVIDER_COMMANDS) {
7153
+ checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
7154
+ }
7155
+ const status = daemonStatus(store);
7156
+ checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
7157
+ const failedRuns = store.countRuns("failed");
7158
+ const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
7159
+ checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
7160
+ if (restartInterruptedRuns > 0) {
7161
+ checks.push({
7162
+ id: "loop-runs:restart-interrupted",
7163
+ status: "warn",
7164
+ message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
7165
+ });
7166
+ }
7167
+ const deployment = buildDeploymentStatus();
7168
+ const schedulerState = deployment.schedulerState;
7169
+ checks.push({
7170
+ id: "scheduler-state",
7171
+ status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
7172
+ message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
7173
+ detail: [
7174
+ `route_state=${schedulerState.routeAdmission.stateStore}`,
7175
+ `active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
7176
+ `gates=${schedulerState.routeAdmission.gates.join(",")}`,
7177
+ `artifacts=${schedulerState.localStore.runArtifacts}`,
7178
+ `remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
7179
+ `remote_apply=${String(schedulerState.remoteStore.applySupported)}`
7180
+ ].join(" ")
7181
+ });
7182
+ for (const loop of store.listLoops({ status: "active" })) {
7183
+ try {
7184
+ if (loop.target.type === "workflow") {
7185
+ const workflow = store.requireWorkflow(loop.target.workflowId);
7186
+ for (const step of workflowExecutionOrder(workflow)) {
7187
+ preflightTarget({
7188
+ ...step.target,
7189
+ account: step.account ?? step.target.account,
7190
+ timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
7191
+ }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
7192
+ }
7193
+ } else {
7194
+ preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
7195
+ }
7196
+ checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
7197
+ } catch (error) {
7198
+ checks.push({
7199
+ id: `loop:${loop.id}:preflight`,
7200
+ status: "fail",
7201
+ message: `active loop target preflight failed: ${loop.name}`,
7202
+ detail: error instanceof Error ? error.message : String(error)
7203
+ });
7204
+ }
7205
+ }
7206
+ return {
7207
+ ok: checks.every((check) => check.status !== "fail"),
7208
+ checks
7209
+ };
7210
+ }
6477
7211
 
6478
7212
  // src/lib/goal/metadata.ts
6479
7213
  function goalExecutionContext(parts) {
@@ -7473,7 +8207,7 @@ var MAX_RETRY_EXPONENT = 20;
7473
8207
  function retryBackoffDelayMs(loop, run, random = Math.random) {
7474
8208
  const attempt = Math.max(1, run.attempt);
7475
8209
  const failure = classifyRunFailure(run);
7476
- const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
8210
+ const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
7477
8211
  const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
7478
8212
  const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
7479
8213
  const jitter = 0.5 + random();
@@ -7600,20 +8334,21 @@ async function executeClaimedRun(deps) {
7600
8334
  daemonLeaseId: deps.daemonLeaseId,
7601
8335
  onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
7602
8336
  })))(deps.loop, deps.run);
8337
+ const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
7603
8338
  deps.beforeFinalize?.(deps.loop, deps.run);
7604
8339
  return deps.store.finalizeRun(deps.run.id, {
7605
- status: result.status,
7606
- finishedAt: result.finishedAt,
7607
- durationMs: result.durationMs,
7608
- stdout: result.stdout,
7609
- stderr: result.stderr,
7610
- exitCode: result.exitCode,
7611
- error: result.error,
7612
- pid: result.pid
8340
+ status: finalResult.status,
8341
+ finishedAt: finalResult.finishedAt,
8342
+ durationMs: finalResult.durationMs,
8343
+ stdout: finalResult.stdout,
8344
+ stderr: finalResult.stderr,
8345
+ exitCode: finalResult.exitCode,
8346
+ error: finalResult.error,
8347
+ pid: finalResult.pid
7613
8348
  }, {
7614
8349
  claimedBy: deps.runnerId,
7615
8350
  daemonLeaseId: deps.daemonLeaseId,
7616
- now: deps.now?.() ?? new Date(result.finishedAt)
8351
+ now: deps.now?.() ?? new Date(finalResult.finishedAt)
7617
8352
  });
7618
8353
  } catch (err) {
7619
8354
  deps.onError?.(deps.loop, err);
@@ -8131,6 +8866,32 @@ var TOOL_REGISTRATIONS = [
8131
8866
  },
8132
8867
  handler: ({ includeArchived, includeInactive, limit }) => withStore((store) => buildHealthReport(store, { includeArchived, includeInactive, limit }))
8133
8868
  },
8869
+ {
8870
+ name: "loops_health_scan",
8871
+ description: "Build a read-only OpenLoops health scan with bounded daemon, doctor/preflight, latest-run, and stale-running findings.",
8872
+ readOnly: true,
8873
+ annotations: READ_ONLY_ANNOTATIONS,
8874
+ inputSchema: {
8875
+ includeStatuses: z2.array(z2.enum(LOOP_STATUSES)).optional().describe("Loop statuses to inventory (default active and paused)."),
8876
+ includeArchived: z2.boolean().optional().describe("Include archived loops in the scan (default false)."),
8877
+ latestRun: z2.boolean().optional().describe("Include latest-run and stale-running checks (default true)."),
8878
+ doctor: z2.boolean().optional().describe("Include doctor/preflight checks (default false)."),
8879
+ daemon: z2.boolean().optional().describe("Include daemon status checks (default false)."),
8880
+ staleRunningMs: z2.number().int().positive().optional().describe("Minimum age before a running latest run is stale; loop lease and 10m still apply."),
8881
+ maxFindings: z2.number().int().min(0).max(MAX_LIMIT).optional().describe(`Maximum findings to return (0-${MAX_LIMIT}, default 100).`),
8882
+ limit: limitSchema
8883
+ },
8884
+ handler: ({ includeStatuses, includeArchived, latestRun, doctor, daemon, staleRunningMs, maxFindings, limit }) => withStore((store) => buildHealthScan(store, {
8885
+ includeStatuses,
8886
+ includeArchived,
8887
+ latestRun,
8888
+ doctor: doctor ? runDoctor(store) : undefined,
8889
+ daemon: daemon ? daemonStatus(store) : undefined,
8890
+ staleRunningMs,
8891
+ maxFindings,
8892
+ limit
8893
+ }))
8894
+ },
8134
8895
  {
8135
8896
  name: "loops_diagnose",
8136
8897
  aliases: ["loop_diagnose"],