@hasna/loops 0.4.13 → 0.4.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +74 -3
  2. package/README.md +54 -12
  3. package/dist/api/index.d.ts +35 -0
  4. package/dist/api/index.js +695 -10
  5. package/dist/cli/index.js +1633 -366
  6. package/dist/cli/ui.d.ts +44 -0
  7. package/dist/daemon/index.js +948 -212
  8. package/dist/generated/storage-kit/health.d.ts +19 -0
  9. package/dist/generated/storage-kit/index.d.ts +7 -0
  10. package/dist/generated/storage-kit/migrations.d.ts +47 -0
  11. package/dist/generated/storage-kit/mode.d.ts +47 -0
  12. package/dist/generated/storage-kit/pool.d.ts +33 -0
  13. package/dist/generated/storage-kit/query.d.ts +35 -0
  14. package/dist/generated/storage-kit/tls.d.ts +25 -0
  15. package/dist/index.d.ts +2 -2
  16. package/dist/index.js +3567 -2010
  17. package/dist/lib/health.d.ts +83 -3
  18. package/dist/lib/mode.js +14 -2
  19. package/dist/lib/scheduler.d.ts +5 -2
  20. package/dist/lib/storage/index.d.ts +1 -0
  21. package/dist/lib/storage/index.js +1329 -211
  22. package/dist/lib/storage/pg-executor.d.ts +27 -0
  23. package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
  24. package/dist/lib/storage/postgres-loop-storage.d.ts +114 -0
  25. package/dist/lib/storage/sqlite.js +336 -8
  26. package/dist/lib/store.d.ts +234 -0
  27. package/dist/lib/store.js +350 -8
  28. package/dist/mcp/index.js +1067 -306
  29. package/dist/runner/index.js +124 -25
  30. package/dist/sdk/http.d.ts +115 -0
  31. package/dist/sdk/http.js +145 -0
  32. package/dist/sdk/index.d.ts +5 -1
  33. package/dist/sdk/index.js +1052 -312
  34. package/dist/serve/index.d.ts +4 -0
  35. package/dist/serve/index.js +7619 -0
  36. package/dist/types.d.ts +3 -0
  37. package/docs/CUTOVER-RUNBOOK.md +61 -0
  38. package/docs/USAGE.md +50 -11
  39. package/package.json +14 -2
package/dist/sdk/index.js CHANGED
@@ -1,9 +1,9 @@
1
1
  // @bun
2
2
  // src/lib/store.ts
3
3
  import { Database } from "bun:sqlite";
4
- import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
5
- import { tmpdir } from "os";
6
- import { basename as basename2, dirname as dirname2, join as join3 } from "path";
4
+ import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
5
+ import { tmpdir as tmpdir2 } from "os";
6
+ import { basename as basename2, dirname as dirname2, join as join4 } from "path";
7
7
 
8
8
  // src/lib/errors.ts
9
9
  class CodedError extends Error {
@@ -1202,6 +1202,196 @@ function discardWorkflowRunManifest(staged) {
1202
1202
  } catch {}
1203
1203
  }
1204
1204
 
1205
+ // src/lib/route/todos-cli.ts
1206
+ import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
1207
+ import { spawnSync as spawnSync2 } from "child_process";
1208
+ import { join as join3 } from "path";
1209
+ import { homedir as homedir2, tmpdir } from "os";
1210
+
1211
+ // src/lib/format.ts
1212
+ var TEXT_OUTPUT_LIMIT = 32 * 1024;
1213
+ var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
1214
+ function redact(value, visible = 0) {
1215
+ if (!value)
1216
+ return value;
1217
+ if (value.length <= visible)
1218
+ return value;
1219
+ if (visible <= 0)
1220
+ return `[redacted ${value.length} chars]`;
1221
+ return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
1222
+ }
1223
+ function truncateTextOutput(value) {
1224
+ if (value.length <= TEXT_OUTPUT_LIMIT)
1225
+ return value;
1226
+ return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
1227
+ [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
1228
+ }
1229
+ function redactSensitivePayload(value, key) {
1230
+ if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
1231
+ if (typeof value === "string")
1232
+ return redact(value);
1233
+ if (value === undefined || value === null)
1234
+ return value;
1235
+ return "[redacted]";
1236
+ }
1237
+ if (Array.isArray(value))
1238
+ return value.map((item) => redactSensitivePayload(item));
1239
+ if (value && typeof value === "object") {
1240
+ return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
1241
+ }
1242
+ return value;
1243
+ }
1244
+ function textOutputBlocks(value, opts = {}) {
1245
+ const indent = opts.indent ?? "";
1246
+ const nested = `${indent} `;
1247
+ const blocks = [];
1248
+ for (const [label, output] of [
1249
+ ["stdout", value.stdout],
1250
+ ["stderr", value.stderr]
1251
+ ]) {
1252
+ if (!output)
1253
+ continue;
1254
+ blocks.push(`${indent}${label}:`);
1255
+ for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
1256
+ blocks.push(`${nested}${line}`);
1257
+ }
1258
+ }
1259
+ return blocks;
1260
+ }
1261
+ function publicLoop(loop) {
1262
+ const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
1263
+ return {
1264
+ ...loop,
1265
+ target
1266
+ };
1267
+ }
1268
+ function publicRun(run, showOutput = false, opts = {}) {
1269
+ return {
1270
+ ...run,
1271
+ stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
1272
+ stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
1273
+ error: opts.redactError ? redact(run.error) : run.error
1274
+ };
1275
+ }
1276
+ function publicExecutorResult(result, showOutput = false) {
1277
+ return {
1278
+ ...result,
1279
+ stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
1280
+ stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
1281
+ error: redact(result.error)
1282
+ };
1283
+ }
1284
+ function publicWorkflow(workflow) {
1285
+ return {
1286
+ ...workflow,
1287
+ steps: workflow.steps.map((step) => ({
1288
+ ...step,
1289
+ target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
1290
+ }))
1291
+ };
1292
+ }
1293
+ function publicWorkflowRun(run) {
1294
+ return { ...run, error: redact(run.error) };
1295
+ }
1296
+ function publicWorkflowInvocation(invocation) {
1297
+ return redactSensitivePayload(invocation);
1298
+ }
1299
+ function publicWorkflowWorkItem(item) {
1300
+ return { ...item, lastReason: redact(item.lastReason, 240) };
1301
+ }
1302
+ function publicWorkflowStepRun(run, showOutput = false) {
1303
+ return {
1304
+ ...run,
1305
+ stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
1306
+ stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
1307
+ error: redact(run.error)
1308
+ };
1309
+ }
1310
+ function publicWorkflowEvent(event) {
1311
+ return { ...event, payload: redactSensitivePayload(event.payload) };
1312
+ }
1313
+ function publicGoal(goal) {
1314
+ return {
1315
+ ...goal,
1316
+ objective: redact(goal.objective, 120)
1317
+ };
1318
+ }
1319
+ function publicGoalRun(run) {
1320
+ return {
1321
+ ...run,
1322
+ evidence: redactSensitivePayload(run.evidence),
1323
+ rawResponse: redactSensitivePayload(run.rawResponse)
1324
+ };
1325
+ }
1326
+
1327
+ // src/lib/route/todos-cli.ts
1328
+ function defaultLoopsProject() {
1329
+ return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
1330
+ }
1331
+ function runLocalCommand(command, args, opts = {}) {
1332
+ const result = spawnSync2(command, args, {
1333
+ input: opts.input,
1334
+ encoding: "utf8",
1335
+ timeout: opts.timeoutMs ?? 30000,
1336
+ maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
1337
+ env: process.env
1338
+ });
1339
+ return {
1340
+ ok: result.status === 0,
1341
+ status: result.status,
1342
+ stdout: result.stdout || "",
1343
+ stderr: result.stderr || "",
1344
+ error: result.error ? String(result.error.message || result.error) : ""
1345
+ };
1346
+ }
1347
+ function runLocalCommandWithStdoutFile(command, args, opts = {}) {
1348
+ const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
1349
+ const stdoutPath = join3(tempDir, "stdout");
1350
+ const stdoutFd = openSync(stdoutPath, "w");
1351
+ let result;
1352
+ try {
1353
+ result = spawnSync2(command, args, {
1354
+ input: opts.input,
1355
+ encoding: "utf8",
1356
+ timeout: opts.timeoutMs ?? 30000,
1357
+ maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
1358
+ env: process.env,
1359
+ stdio: ["pipe", stdoutFd, "pipe"]
1360
+ });
1361
+ } finally {
1362
+ closeSync(stdoutFd);
1363
+ }
1364
+ try {
1365
+ return {
1366
+ ok: result.status === 0,
1367
+ status: result.status,
1368
+ stdout: readFileSync3(stdoutPath, "utf8"),
1369
+ stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
1370
+ error: result.error ? String(result.error.message || result.error) : ""
1371
+ };
1372
+ } finally {
1373
+ rmSync2(tempDir, { recursive: true, force: true });
1374
+ }
1375
+ }
1376
+ function ensureTodosTaskList(project, slug, name, description) {
1377
+ runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
1378
+ const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
1379
+ if (!list.ok)
1380
+ throw new Error(list.stderr || list.error || "failed to list todos task lists");
1381
+ const values = JSON.parse(list.stdout || "[]");
1382
+ const found = values.find((entry) => entry.slug === slug);
1383
+ if (!found)
1384
+ throw new Error(`todos task list not found after ensure: ${slug}`);
1385
+ return found.id;
1386
+ }
1387
+ function todosMutationSummary(result) {
1388
+ return {
1389
+ ok: result.ok,
1390
+ status: result.status,
1391
+ error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
1392
+ };
1393
+ }
1394
+
1205
1395
  // src/lib/store.ts
1206
1396
  var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
1207
1397
  var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
@@ -1211,6 +1401,7 @@ var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "s
1211
1401
  var PRUNE_BATCH_SIZE = 400;
1212
1402
  var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
1213
1403
  var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
1404
+ var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
1214
1405
  function rowToLoop(row) {
1215
1406
  return {
1216
1407
  id: row.id,
@@ -1261,6 +1452,9 @@ function rowToRun(row) {
1261
1452
  updatedAt: row.updated_at
1262
1453
  };
1263
1454
  }
1455
+ function latestRunTime(row) {
1456
+ return row.finished_at ?? row.started_at ?? row.created_at;
1457
+ }
1264
1458
  function rowToWorkflow(row) {
1265
1459
  return {
1266
1460
  id: row.id,
@@ -1515,7 +1709,7 @@ class Store {
1515
1709
  const file = path ?? dbPath();
1516
1710
  if (file !== ":memory:")
1517
1711
  ensurePrivateStorePath(file);
1518
- this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
1712
+ this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
1519
1713
  if (file === ":memory:")
1520
1714
  this.memoryRootDir = this.rootDir;
1521
1715
  this.db = new Database(file);
@@ -2013,7 +2207,32 @@ class Store {
2013
2207
  } else {
2014
2208
  rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
2015
2209
  }
2016
- return rows.map(rowToLoop);
2210
+ return this.withLatestRunSummaries(rows.map(rowToLoop));
2211
+ }
2212
+ withLatestRunSummaries(loops) {
2213
+ if (loops.length === 0)
2214
+ return loops;
2215
+ const placeholders = loops.map(() => "?").join(",");
2216
+ const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
2217
+ FROM (
2218
+ SELECT loop_id, id, status, started_at, finished_at, created_at,
2219
+ ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
2220
+ FROM loop_runs
2221
+ WHERE loop_id IN (${placeholders})
2222
+ )
2223
+ WHERE rn = 1`).all(...loops.map((loop) => loop.id));
2224
+ const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
2225
+ return loops.map((loop) => {
2226
+ const latest = latestByLoopId.get(loop.id);
2227
+ if (!latest)
2228
+ return loop;
2229
+ return {
2230
+ ...loop,
2231
+ latestRunId: latest.id,
2232
+ latestRunStatus: latest.status,
2233
+ lastRunAt: latestRunTime(latest)
2234
+ };
2235
+ });
2017
2236
  }
2018
2237
  dueLoops(now, limit = 500) {
2019
2238
  const rows = this.db.query(`SELECT * FROM loops
@@ -2132,6 +2351,45 @@ class Store {
2132
2351
  throw error;
2133
2352
  }
2134
2353
  }
2354
+ updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
2355
+ const updated = (opts.now ?? new Date).toISOString();
2356
+ this.db.exec("BEGIN IMMEDIATE");
2357
+ try {
2358
+ const current = this.requireUniqueLoop(idOrName);
2359
+ if (current.archivedAt)
2360
+ throw new LoopArchivedError(current.name || current.id);
2361
+ if (current.target.type !== "agent")
2362
+ throw new Error(`loop is not an agent loop: ${idOrName}`);
2363
+ if (this.hasRunningRun(current.id))
2364
+ throw new Error(`refusing to update running loop: ${current.id}`);
2365
+ const target = { ...current.target, timeoutMs };
2366
+ if (timeoutMs === null && target.idleTimeoutMs !== undefined)
2367
+ delete target.idleTimeoutMs;
2368
+ const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
2369
+ WHERE id=$id
2370
+ AND ($daemonLeaseId IS NULL OR EXISTS (
2371
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
2372
+ ))`).run({
2373
+ $id: current.id,
2374
+ $target: JSON.stringify(target),
2375
+ $updated: updated,
2376
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
2377
+ $now: updated
2378
+ });
2379
+ if (res.changes !== 1)
2380
+ throw new Error("daemon lease lost");
2381
+ this.db.exec("COMMIT");
2382
+ const after = this.getLoop(current.id);
2383
+ if (!after)
2384
+ throw new Error(`loop not found after timeout update: ${current.id}`);
2385
+ return after;
2386
+ } catch (error) {
2387
+ try {
2388
+ this.db.exec("ROLLBACK");
2389
+ } catch {}
2390
+ throw error;
2391
+ }
2392
+ }
2135
2393
  createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
2136
2394
  const normalized = normalizeCreateWorkflowInput(workflowInput);
2137
2395
  const updated = (opts.now ?? new Date).toISOString();
@@ -2462,6 +2720,59 @@ class Store {
2462
2720
  updated
2463
2721
  });
2464
2722
  }
2723
+ taskLifecycleTodosPointerContext(workflowRunId) {
2724
+ const run = this.getWorkflowRun(workflowRunId);
2725
+ if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
2726
+ return;
2727
+ const workItem = this.getWorkflowWorkItem(run.workItemId);
2728
+ if (!workItem || workItem.routeKey !== "todos-task")
2729
+ return;
2730
+ const invocation = this.getWorkflowInvocation(run.invocationId);
2731
+ if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
2732
+ return;
2733
+ const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
2734
+ const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
2735
+ if (!projectPath || !taskId)
2736
+ return;
2737
+ return {
2738
+ projectPath,
2739
+ taskId,
2740
+ invocationId: invocation.id,
2741
+ workflowRunId: run.id,
2742
+ manifestPath: run.manifestPath
2743
+ };
2744
+ }
2745
+ syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
2746
+ const context = this.taskLifecycleTodosPointerContext(workflowRunId);
2747
+ if (!context)
2748
+ return;
2749
+ const result = runLocalCommand("todos", [
2750
+ "--project",
2751
+ context.projectPath,
2752
+ "task",
2753
+ "workflow-pointers",
2754
+ context.taskId,
2755
+ "--clear",
2756
+ "--invocation",
2757
+ context.invocationId,
2758
+ "--run",
2759
+ context.workflowRunId,
2760
+ "--manifest",
2761
+ context.manifestPath,
2762
+ "--state",
2763
+ "succeeded",
2764
+ "--actor",
2765
+ "openloops:task-lifecycle"
2766
+ ]);
2767
+ this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
2768
+ projectPath: context.projectPath,
2769
+ taskId: context.taskId,
2770
+ invocationId: context.invocationId,
2771
+ workflowRunId: context.workflowRunId,
2772
+ manifestPath: context.manifestPath,
2773
+ mutation: todosMutationSummary(result)
2774
+ });
2775
+ }
2465
2776
  createWorkflowInvocation(input) {
2466
2777
  const now = nowIso();
2467
2778
  const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
@@ -3424,6 +3735,8 @@ class Store {
3424
3735
  const run = this.getWorkflowRun(workflowRunId);
3425
3736
  if (!run)
3426
3737
  throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
3738
+ if (changed && status === "succeeded")
3739
+ this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
3427
3740
  return run;
3428
3741
  }
3429
3742
  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -3485,6 +3798,17 @@ class Store {
3485
3798
  const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
3486
3799
  return (row?.count ?? 0) > 0;
3487
3800
  }
3801
+ hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
3802
+ const rows = this.db.query(`SELECT * FROM loop_runs
3803
+ WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
3804
+ return rows.some((row) => {
3805
+ if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
3806
+ return true;
3807
+ if (isRecordedProcessAlive(row.pid, row.process_started_at))
3808
+ return true;
3809
+ return this.hasLiveWorkflowStepProcesses(row.id);
3810
+ });
3811
+ }
3488
3812
  markRunPid(id, pid, claimedBy, opts = {}) {
3489
3813
  const now = (opts.now ?? new Date).toISOString();
3490
3814
  const startedMs = processStartTimeMs(pid);
@@ -3617,6 +3941,10 @@ class Store {
3617
3941
  loop = currentLoop;
3618
3942
  const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
3619
3943
  const existing = this.getRunBySlot(loop.id, scheduledFor);
3944
+ if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
3945
+ this.db.exec("COMMIT");
3946
+ return;
3947
+ }
3620
3948
  if (existing) {
3621
3949
  if (existing.status === "running") {
3622
3950
  if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
@@ -4188,9 +4516,9 @@ class Store {
4188
4516
  const runDir = dirname2(manifestPath);
4189
4517
  try {
4190
4518
  if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
4191
- rmSync2(runDir, { recursive: true, force: true });
4519
+ rmSync3(runDir, { recursive: true, force: true });
4192
4520
  } else {
4193
- rmSync2(manifestPath, { force: true });
4521
+ rmSync3(manifestPath, { force: true });
4194
4522
  }
4195
4523
  } catch {}
4196
4524
  }
@@ -4257,7 +4585,7 @@ class Store {
4257
4585
  this.db.close();
4258
4586
  if (this.memoryRootDir) {
4259
4587
  try {
4260
- rmSync2(this.memoryRootDir, { recursive: true, force: true });
4588
+ rmSync3(this.memoryRootDir, { recursive: true, force: true });
4261
4589
  } catch {}
4262
4590
  this.memoryRootDir = undefined;
4263
4591
  }
@@ -4266,7 +4594,7 @@ class Store {
4266
4594
  // package.json
4267
4595
  var package_default = {
4268
4596
  name: "@hasna/loops",
4269
- version: "0.4.13",
4597
+ version: "0.4.14",
4270
4598
  description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
4271
4599
  type: "module",
4272
4600
  main: "dist/index.js",
@@ -4275,6 +4603,7 @@ var package_default = {
4275
4603
  loops: "dist/cli/index.js",
4276
4604
  "loops-daemon": "dist/daemon/index.js",
4277
4605
  "loops-api": "dist/api/index.js",
4606
+ "loops-serve": "dist/serve/index.js",
4278
4607
  "loops-runner": "dist/runner/index.js",
4279
4608
  "loops-mcp": "dist/mcp/index.js"
4280
4609
  },
@@ -4287,6 +4616,14 @@ var package_default = {
4287
4616
  types: "./dist/sdk/index.d.ts",
4288
4617
  import: "./dist/sdk/index.js"
4289
4618
  },
4619
+ "./sdk/http": {
4620
+ types: "./dist/sdk/http.d.ts",
4621
+ import: "./dist/sdk/http.js"
4622
+ },
4623
+ "./serve": {
4624
+ types: "./dist/serve/index.d.ts",
4625
+ import: "./dist/serve/index.js"
4626
+ },
4290
4627
  "./mcp": {
4291
4628
  types: "./dist/mcp/index.d.ts",
4292
4629
  import: "./dist/mcp/index.js"
@@ -4332,7 +4669,7 @@ var package_default = {
4332
4669
  "LICENSE"
4333
4670
  ],
4334
4671
  scripts: {
4335
- build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
4672
+ build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
4336
4673
  "build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
4337
4674
  typecheck: "tsc --noEmit",
4338
4675
  test: "bun test",
@@ -4367,11 +4704,13 @@ var package_default = {
4367
4704
  bun: ">=1.0.0"
4368
4705
  },
4369
4706
  dependencies: {
4707
+ "@hasna/contracts": "^0.4.1",
4370
4708
  "@hasna/events": "^0.1.9",
4371
4709
  "@modelcontextprotocol/sdk": "^1.29.0",
4372
4710
  "@openrouter/ai-sdk-provider": "2.9.1",
4373
4711
  ai: "6.0.204",
4374
4712
  commander: "^13.1.0",
4713
+ pg: "^8.13.1",
4375
4714
  zod: "4.4.3"
4376
4715
  },
4377
4716
  optionalDependencies: {
@@ -4379,6 +4718,7 @@ var package_default = {
4379
4718
  },
4380
4719
  devDependencies: {
4381
4720
  "@types/bun": "latest",
4721
+ "@types/pg": "^8.11.10",
4382
4722
  typescript: "^5.7.3"
4383
4723
  },
4384
4724
  publishConfig: {
@@ -4586,15 +4926,11 @@ function deploymentStatusLine(status) {
4586
4926
  ].join(" ");
4587
4927
  }
4588
4928
 
4589
- // src/lib/doctor.ts
4590
- import { spawnSync as spawnSync5 } from "child_process";
4591
- import { accessSync as accessSync2, constants as constants2 } from "fs";
4592
-
4593
4929
  // src/daemon/control.ts
4594
- import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync3, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
4595
- import { spawnSync as spawnSync2 } from "child_process";
4930
+ import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
4931
+ import { spawnSync as spawnSync3 } from "child_process";
4596
4932
  import { hostname } from "os";
4597
- import { dirname as dirname3, join as join4 } from "path";
4933
+ import { dirname as dirname3, join as join5 } from "path";
4598
4934
 
4599
4935
  // src/daemon/loop.ts
4600
4936
  function realSleep(ms) {
@@ -4624,7 +4960,7 @@ function readPidRecord(path = pidFilePath()) {
4624
4960
  return;
4625
4961
  let raw;
4626
4962
  try {
4627
- raw = readFileSync3(path, "utf8").trim();
4963
+ raw = readFileSync4(path, "utf8").trim();
4628
4964
  } catch {
4629
4965
  return;
4630
4966
  }
@@ -4650,7 +4986,7 @@ function writePid(pid = process.pid, path = pidFilePath()) {
4650
4986
  writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
4651
4987
  }
4652
4988
  function removePid(path = pidFilePath()) {
4653
- rmSync3(path, { force: true });
4989
+ rmSync4(path, { force: true });
4654
4990
  }
4655
4991
  function isAlive(pid, startedAt) {
4656
4992
  try {
@@ -4680,7 +5016,7 @@ function ownProcessGroupId() {
4680
5016
  return pgrp;
4681
5017
  }
4682
5018
  try {
4683
- const run = spawnSync2("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
5019
+ const run = spawnSync3("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
4684
5020
  const pgid = Number(run.stdout.trim());
4685
5021
  if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
4686
5022
  return pgid;
@@ -4776,7 +5112,7 @@ async function stopDaemon(opts = {}) {
4776
5112
  if (!state.running || !state.pid)
4777
5113
  return { wasRunning: false, stopped: false, forced: false };
4778
5114
  const sleep = opts.sleep ?? realSleep;
4779
- const store = new Store(join4(dirname3(path), "loops.db"));
5115
+ const store = new Store(join5(dirname3(path), "loops.db"));
4780
5116
  try {
4781
5117
  const lease = store.getDaemonLease();
4782
5118
  const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
@@ -4813,15 +5149,19 @@ async function stopDaemon(opts = {}) {
4813
5149
  }
4814
5150
  }
4815
5151
 
5152
+ // src/lib/doctor.ts
5153
+ import { spawnSync as spawnSync6 } from "child_process";
5154
+ import { accessSync as accessSync2, constants as constants2 } from "fs";
5155
+
4816
5156
  // src/lib/executor.ts
4817
- import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
5157
+ import { spawn as spawn2, spawnSync as spawnSync5 } from "child_process";
4818
5158
  import { randomBytes as randomBytes2 } from "crypto";
4819
5159
  import { once } from "events";
4820
5160
  import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
4821
5161
  import { dirname as dirname4, resolve as resolve2 } from "path";
4822
5162
 
4823
5163
  // src/lib/accounts.ts
4824
- import { spawnSync as spawnSync3 } from "child_process";
5164
+ import { spawnSync as spawnSync4 } from "child_process";
4825
5165
  import { existsSync as existsSync3 } from "fs";
4826
5166
  var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
4827
5167
  var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
@@ -4926,7 +5266,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
4926
5266
  if (!account)
4927
5267
  return {};
4928
5268
  const tool = requiredAccountTool(account, toolHint);
4929
- const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
5269
+ const result = spawnSync4("accounts", ["env", account.profile, "--tool", tool], {
4930
5270
  encoding: "utf8",
4931
5271
  env,
4932
5272
  stdio: ["ignore", "pipe", "pipe"],
@@ -4942,8 +5282,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
4942
5282
 
4943
5283
  // src/lib/env.ts
4944
5284
  import { accessSync, constants } from "fs";
4945
- import { homedir as homedir2 } from "os";
4946
- import { delimiter, join as join5 } from "path";
5285
+ import { homedir as homedir3 } from "os";
5286
+ import { delimiter, join as join6 } from "path";
4947
5287
  function compactPathParts(parts) {
4948
5288
  const seen = new Set;
4949
5289
  const result = [];
@@ -4957,16 +5297,16 @@ function compactPathParts(parts) {
4957
5297
  return result;
4958
5298
  }
4959
5299
  function commonExecutableDirs(env = process.env) {
4960
- const home = env.HOME || homedir2();
5300
+ const home = env.HOME || homedir3();
4961
5301
  return compactPathParts([
4962
- join5(home, ".local", "bin"),
4963
- join5(home, ".bun", "bin"),
4964
- join5(home, ".cargo", "bin"),
4965
- join5(home, ".npm-global", "bin"),
4966
- join5(home, "bin"),
4967
- env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
5302
+ join6(home, ".local", "bin"),
5303
+ join6(home, ".bun", "bin"),
5304
+ join6(home, ".cargo", "bin"),
5305
+ join6(home, ".npm-global", "bin"),
5306
+ join6(home, "bin"),
5307
+ env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
4968
5308
  env.PNPM_HOME,
4969
- env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
5309
+ env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
4970
5310
  "/opt/homebrew/bin",
4971
5311
  "/usr/local/bin",
4972
5312
  "/usr/bin",
@@ -4990,7 +5330,7 @@ function executableExists(command, env = process.env) {
4990
5330
  if (command.includes("/"))
4991
5331
  return isExecutable(command);
4992
5332
  for (const dir of (env.PATH ?? "").split(delimiter)) {
4993
- if (dir && isExecutable(join5(dir, command)))
5333
+ if (dir && isExecutable(join6(dir, command)))
4994
5334
  return true;
4995
5335
  }
4996
5336
  return false;
@@ -5131,6 +5471,32 @@ function timeoutResult(startedAt, error, fields = {}) {
5131
5471
  function successResult(startedAt, fields = {}) {
5132
5472
  return buildResult("succeeded", startedAt, fields);
5133
5473
  }
5474
+ function codewithJsonlHasTerminalSuccess(stdout) {
5475
+ for (const line of stdout.split(/\r?\n/)) {
5476
+ const trimmed = line.trim();
5477
+ if (!trimmed || !trimmed.startsWith("{"))
5478
+ continue;
5479
+ let event;
5480
+ try {
5481
+ event = JSON.parse(trimmed);
5482
+ } catch {
5483
+ continue;
5484
+ }
5485
+ if (!event || typeof event !== "object")
5486
+ continue;
5487
+ const record = event;
5488
+ if (record.type === "task_complete")
5489
+ return true;
5490
+ const payload = record.payload;
5491
+ if (payload && typeof payload === "object" && payload.type === "task_complete") {
5492
+ return true;
5493
+ }
5494
+ }
5495
+ return false;
5496
+ }
5497
+ function codewithJsonlReconciledSuccess(spec, fields) {
5498
+ return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
5499
+ }
5134
5500
  function notifySpawn(pid, opts) {
5135
5501
  if (!pid)
5136
5502
  return;
@@ -5146,10 +5512,48 @@ function shellQuote(value) {
5146
5512
  return `'${value.replace(/'/g, `'\\''`)}'`;
5147
5513
  }
5148
5514
  function codewithProfileCandidateFromLine(line) {
5149
- const cols = line.trim().split(/\s+/);
5150
- if (cols[0] === "*")
5151
- return cols[1];
5152
- return cols[0];
5515
+ const trimmed = line.trim();
5516
+ if (!trimmed || trimmed === "No auth profiles saved.")
5517
+ return;
5518
+ const cols = trimmed.split(/\s+/);
5519
+ const candidate = cols[0] === "*" ? cols[1] : cols[0];
5520
+ if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
5521
+ return;
5522
+ if (candidate === '"name":') {
5523
+ const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
5524
+ if (jsonName)
5525
+ return jsonName;
5526
+ }
5527
+ return candidate;
5528
+ }
5529
+ function codewithProfileCandidatesFromJson(value) {
5530
+ const candidates = [];
5531
+ const root = value && typeof value === "object" ? value : undefined;
5532
+ const addProfile = (entry) => {
5533
+ if (!entry || typeof entry !== "object")
5534
+ return;
5535
+ const name = entry.name;
5536
+ if (typeof name === "string" && name)
5537
+ candidates.push(name);
5538
+ };
5539
+ const data = root?.data;
5540
+ if (Array.isArray(data))
5541
+ data.forEach(addProfile);
5542
+ const profiles = root?.profiles;
5543
+ if (Array.isArray(profiles))
5544
+ profiles.forEach(addProfile);
5545
+ return candidates;
5546
+ }
5547
+ function codewithProfileCandidatesFromOutput(output) {
5548
+ const trimmed = output.trim();
5549
+ const jsonStart = trimmed.indexOf("{");
5550
+ const jsonEnd = trimmed.lastIndexOf("}");
5551
+ if (jsonStart >= 0 && jsonEnd >= jsonStart) {
5552
+ try {
5553
+ return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
5554
+ } catch {}
5555
+ }
5556
+ return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
5153
5557
  }
5154
5558
  function codewithProfileForError(profile) {
5155
5559
  return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
@@ -5242,6 +5646,7 @@ function commandSpec(target, opts) {
5242
5646
  preflightAnyOf: invocation.preflightAnyOf,
5243
5647
  stdin: invocation.stdin,
5244
5648
  allowlist: agentTarget.allowlist,
5649
+ agentProvider: agentTarget.provider,
5245
5650
  worktree: agentTarget.worktree
5246
5651
  };
5247
5652
  }
@@ -5401,7 +5806,7 @@ function remotePreflightScript(spec, metadata) {
5401
5806
  }
5402
5807
  if (spec.nativeAuthProfile?.provider === "codewith") {
5403
5808
  const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
5404
- lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", `__OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", "}", `if ! printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk 'NR > 1 { candidate = ($1 == "*" ? $2 : $1); if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'; then`, ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", "fi");
5809
+ lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
5405
5810
  }
5406
5811
  return lines.join(`
5407
5812
  `);
@@ -5419,6 +5824,12 @@ function transportEnv(opts) {
5419
5824
  return env;
5420
5825
  }
5421
5826
  function assertCodewithProfileListed(profile, result) {
5827
+ const profiles = codewithProfileSetFromResult(result);
5828
+ if (!profiles.has(profile)) {
5829
+ throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
5830
+ }
5831
+ }
5832
+ function codewithProfileSetFromResult(result) {
5422
5833
  if (result.error) {
5423
5834
  throw new Error(`codewith auth profile preflight failed: ${result.error}`);
5424
5835
  }
@@ -5426,33 +5837,43 @@ function assertCodewithProfileListed(profile, result) {
5426
5837
  const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
5427
5838
  throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
5428
5839
  }
5429
- const profiles = new Set((result.stdout || "").split(/\r?\n/).slice(1).map(codewithProfileCandidateFromLine).filter(Boolean));
5430
- if (!profiles.has(profile)) {
5431
- throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
5432
- }
5840
+ return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
5433
5841
  }
5434
5842
  function preflightNativeAuthProfileSync(spec, env) {
5435
5843
  if (spec.nativeAuthProfile?.provider !== "codewith")
5436
5844
  return;
5437
- const result = spawnSync4(spec.command, ["profile", "list"], {
5438
- encoding: "utf8",
5439
- env,
5440
- stdio: ["ignore", "pipe", "pipe"],
5441
- timeout: 15000
5442
- });
5443
- assertCodewithProfileListed(spec.nativeAuthProfile.profile, {
5444
- status: result.status,
5445
- stdout: result.stdout ?? "",
5446
- stderr: result.stderr ?? "",
5447
- error: result.error?.message
5448
- });
5845
+ const runProfileList = (args) => {
5846
+ const result = spawnSync5(spec.command, args, {
5847
+ encoding: "utf8",
5848
+ env,
5849
+ stdio: ["ignore", "pipe", "pipe"],
5850
+ timeout: 15000
5851
+ });
5852
+ return {
5853
+ status: result.status,
5854
+ stdout: result.stdout ?? "",
5855
+ stderr: result.stderr ?? "",
5856
+ error: result.error?.message
5857
+ };
5858
+ };
5859
+ const jsonResult = runProfileList(["profile", "list", "--json"]);
5860
+ try {
5861
+ if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
5862
+ return;
5863
+ } catch {}
5864
+ assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
5449
5865
  }
5450
5866
  async function preflightNativeAuthProfile(spec, env) {
5451
5867
  if (spec.nativeAuthProfile?.provider !== "codewith")
5452
5868
  return;
5453
- const result = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
5454
- assertCodewithProfileListed(spec.nativeAuthProfile.profile, result);
5455
- }
5869
+ const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
5870
+ try {
5871
+ if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
5872
+ return;
5873
+ } catch {}
5874
+ const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
5875
+ assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
5876
+ }
5456
5877
  function resolvedDirEquals(left, right) {
5457
5878
  try {
5458
5879
  return realpathSync(left) === realpathSync(right);
@@ -5554,7 +5975,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
5554
5975
  }
5555
5976
  function preflightRemoteSpec(spec, machine, metadata, opts) {
5556
5977
  const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
5557
- const result = spawnSync4(plan.command, plan.args, {
5978
+ const result = spawnSync5(plan.command, plan.args, {
5558
5979
  encoding: "utf8",
5559
5980
  env: transportEnv(opts),
5560
5981
  input: remotePreflightScript(spec, metadata),
@@ -5653,6 +6074,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
5653
6074
  if (timedOut || idleTimedOut) {
5654
6075
  return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
5655
6076
  }
6077
+ if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
6078
+ return successResult(startedAt, fields);
6079
+ }
5656
6080
  if (error || exitCode !== 0) {
5657
6081
  return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
5658
6082
  }
@@ -5788,6 +6212,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
5788
6212
  if (timedOut || idleTimedOut) {
5789
6213
  return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
5790
6214
  }
6215
+ if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
6216
+ return successResult(startedAt, fields);
6217
+ }
5791
6218
  if (error || exitCode !== 0) {
5792
6219
  return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
5793
6220
  }
@@ -5818,241 +6245,20 @@ async function executeLoop(loop, run, opts = {}) {
5818
6245
  }, { ...opts, machine: opts.machine ?? loop.machine });
5819
6246
  }
5820
6247
 
5821
- // src/lib/doctor.ts
5822
- var PROVIDER_COMMANDS = [
5823
- "claude",
5824
- "agent",
5825
- "codewith",
5826
- "aicopilot",
5827
- "opencode",
5828
- "codex"
5829
- ];
5830
- function hasCommand(command) {
5831
- const result = spawnSync5("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
5832
- return (result.status ?? 1) === 0;
5833
- }
5834
- function commandVersion(command) {
5835
- const result = spawnSync5(command, ["--version"], {
5836
- encoding: "utf8",
5837
- stdio: ["ignore", "pipe", "pipe"]
5838
- });
5839
- if ((result.status ?? 1) !== 0)
5840
- return;
5841
- return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
5842
- }
5843
- function runDoctor(store) {
5844
- const checks = [];
5845
- try {
5846
- const dir = ensureDataDir();
5847
- accessSync2(dir, constants2.R_OK | constants2.W_OK);
5848
- checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
5849
- } catch (error) {
5850
- checks.push({
5851
- id: "data-dir",
5852
- status: "fail",
5853
- message: "data directory is not writable",
5854
- detail: error instanceof Error ? error.message : String(error)
5855
- });
5856
- }
5857
- const bunVersion = commandVersion("bun");
5858
- checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
5859
- const accountsVersion = commandVersion("accounts");
5860
- checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
5861
- try {
5862
- const machines = listOpenMachines();
5863
- const local = machines.find((machine) => machine.local);
5864
- checks.push({
5865
- id: "machines",
5866
- status: "ok",
5867
- message: `OpenMachines topology available (${machines.length} machine(s))`,
5868
- detail: local ? `local=${local.id}` : undefined
5869
- });
5870
- } catch (error) {
5871
- checks.push({
5872
- id: "machines",
5873
- status: "warn",
5874
- message: "OpenMachines topology is not available; machine-assigned loops will fail",
5875
- detail: error instanceof Error ? error.message : String(error)
5876
- });
5877
- }
5878
- for (const command of PROVIDER_COMMANDS) {
5879
- checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
5880
- }
5881
- const status = daemonStatus(store);
5882
- checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
5883
- const failedRuns = store.countRuns("failed");
5884
- checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
5885
- const deployment = buildDeploymentStatus();
5886
- const schedulerState = deployment.schedulerState;
5887
- checks.push({
5888
- id: "scheduler-state",
5889
- status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
5890
- message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
5891
- detail: [
5892
- `route_state=${schedulerState.routeAdmission.stateStore}`,
5893
- `active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
5894
- `gates=${schedulerState.routeAdmission.gates.join(",")}`,
5895
- `artifacts=${schedulerState.localStore.runArtifacts}`,
5896
- `remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
5897
- `remote_apply=${String(schedulerState.remoteStore.applySupported)}`
5898
- ].join(" ")
5899
- });
5900
- for (const loop of store.listLoops({ status: "active" })) {
5901
- try {
5902
- if (loop.target.type === "workflow") {
5903
- const workflow = store.requireWorkflow(loop.target.workflowId);
5904
- for (const step of workflowExecutionOrder(workflow)) {
5905
- preflightTarget({
5906
- ...step.target,
5907
- account: step.account ?? step.target.account,
5908
- timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
5909
- }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
5910
- }
5911
- } else {
5912
- preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
5913
- }
5914
- checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
5915
- } catch (error) {
5916
- checks.push({
5917
- id: `loop:${loop.id}:preflight`,
5918
- status: "fail",
5919
- message: `active loop target preflight failed: ${loop.name}`,
5920
- detail: error instanceof Error ? error.message : String(error)
5921
- });
5922
- }
5923
- }
5924
- return {
5925
- ok: checks.every((check) => check.status !== "fail"),
5926
- checks
5927
- };
5928
- }
5929
-
5930
6248
  // src/lib/health.ts
5931
6249
  import { createHash as createHash2 } from "crypto";
5932
- import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
5933
-
5934
- // src/lib/format.ts
5935
- var TEXT_OUTPUT_LIMIT = 32 * 1024;
5936
- var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
5937
- function redact(value, visible = 0) {
5938
- if (!value)
5939
- return value;
5940
- if (value.length <= visible)
5941
- return value;
5942
- if (visible <= 0)
5943
- return `[redacted ${value.length} chars]`;
5944
- return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
5945
- }
5946
- function truncateTextOutput(value) {
5947
- if (value.length <= TEXT_OUTPUT_LIMIT)
5948
- return value;
5949
- return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
5950
- [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
5951
- }
5952
- function redactSensitivePayload(value, key) {
5953
- if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
5954
- if (typeof value === "string")
5955
- return redact(value);
5956
- if (value === undefined || value === null)
5957
- return value;
5958
- return "[redacted]";
5959
- }
5960
- if (Array.isArray(value))
5961
- return value.map((item) => redactSensitivePayload(item));
5962
- if (value && typeof value === "object") {
5963
- return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
5964
- }
5965
- return value;
5966
- }
5967
- function textOutputBlocks(value, opts = {}) {
5968
- const indent = opts.indent ?? "";
5969
- const nested = `${indent} `;
5970
- const blocks = [];
5971
- for (const [label, output] of [
5972
- ["stdout", value.stdout],
5973
- ["stderr", value.stderr]
5974
- ]) {
5975
- if (!output)
5976
- continue;
5977
- blocks.push(`${indent}${label}:`);
5978
- for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
5979
- blocks.push(`${nested}${line}`);
5980
- }
5981
- }
5982
- return blocks;
5983
- }
5984
- function publicLoop(loop) {
5985
- const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
5986
- return {
5987
- ...loop,
5988
- target
5989
- };
5990
- }
5991
- function publicRun(run, showOutput = false, opts = {}) {
5992
- return {
5993
- ...run,
5994
- stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
5995
- stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
5996
- error: opts.redactError ? redact(run.error) : run.error
5997
- };
5998
- }
5999
- function publicExecutorResult(result, showOutput = false) {
6000
- return {
6001
- ...result,
6002
- stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
6003
- stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
6004
- error: redact(result.error)
6005
- };
6006
- }
6007
- function publicWorkflow(workflow) {
6008
- return {
6009
- ...workflow,
6010
- steps: workflow.steps.map((step) => ({
6011
- ...step,
6012
- target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
6013
- }))
6014
- };
6015
- }
6016
- function publicWorkflowRun(run) {
6017
- return { ...run, error: redact(run.error) };
6018
- }
6019
- function publicWorkflowInvocation(invocation) {
6020
- return redactSensitivePayload(invocation);
6021
- }
6022
- function publicWorkflowWorkItem(item) {
6023
- return { ...item, lastReason: redact(item.lastReason, 240) };
6024
- }
6025
- function publicWorkflowStepRun(run, showOutput = false) {
6026
- return {
6027
- ...run,
6028
- stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
6029
- stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
6030
- error: redact(run.error)
6031
- };
6032
- }
6033
- function publicWorkflowEvent(event) {
6034
- return { ...event, payload: redactSensitivePayload(event.payload) };
6035
- }
6036
- function publicGoal(goal) {
6037
- return {
6038
- ...goal,
6039
- objective: redact(goal.objective, 120)
6040
- };
6041
- }
6042
- function publicGoalRun(run) {
6043
- return {
6044
- ...run,
6045
- evidence: redactSensitivePayload(run.evidence),
6046
- rawResponse: redactSensitivePayload(run.rawResponse)
6047
- };
6048
- }
6049
-
6050
- // src/lib/health.ts
6250
+ import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
6251
+ import { join as join7 } from "path";
6051
6252
  var EVIDENCE_CHARS = 2000;
6052
6253
  var FINGERPRINT_EVIDENCE_CHARS = 120;
6254
+ var DEFAULT_SCAN_LIMIT = 200;
6255
+ var DEFAULT_SCAN_STATUSES = ["active", "paused"];
6256
+ var DEFAULT_MAX_FINDINGS = 100;
6257
+ var MIN_STALE_RUNNING_MS = 10 * 60000;
6053
6258
  var CLASSIFICATIONS = [
6054
6259
  "rate_limit",
6055
6260
  "auth",
6261
+ "provider_unavailable",
6056
6262
  "model_not_found",
6057
6263
  "context_length",
6058
6264
  "schema_response_format",
@@ -6061,10 +6267,12 @@ var CLASSIFICATIONS = [
6061
6267
  "route_functional",
6062
6268
  "timeout",
6063
6269
  "sigsegv",
6270
+ "restart_interrupted",
6064
6271
  "skipped_previous_active",
6065
6272
  "circuit_breaker",
6066
6273
  "unknown"
6067
6274
  ];
6275
+ var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
6068
6276
  function bounded(value, limit = EVIDENCE_CHARS) {
6069
6277
  if (!value)
6070
6278
  return;
@@ -6078,7 +6286,7 @@ function redactedEvidence(value) {
6078
6286
  }
6079
6287
  function searchableText(run) {
6080
6288
  return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
6081
- `).toLowerCase();
6289
+ `);
6082
6290
  }
6083
6291
  function isRecord(value) {
6084
6292
  return Boolean(value && typeof value === "object" && !Array.isArray(value));
@@ -6103,13 +6311,36 @@ function stableFingerprint(parts) {
6103
6311
  return createHash2("sha256").update(parts.join(`
6104
6312
  `)).digest("hex").slice(0, 16);
6105
6313
  }
6106
- function stableFailureFingerprint(run, classification) {
6314
+ function stableScanFingerprint(parts) {
6315
+ return `openloops:health-scan:${stableFingerprint(parts)}`;
6316
+ }
6317
+ function safeHost(value) {
6318
+ if (!value)
6319
+ return;
6320
+ let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
6321
+ host = host.replace(/^\[|\]$/g, "");
6322
+ return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
6323
+ }
6324
+ function isCursorHost(host) {
6325
+ return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
6326
+ }
6327
+ function providerUnavailableSummary(rawText) {
6328
+ const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
6329
+ if (dns) {
6330
+ const host = safeHost(dns[2]);
6331
+ if (isCursorHost(host))
6332
+ return `provider DNS lookup failed: ${dns[1]} ${host}`;
6333
+ }
6334
+ return;
6335
+ }
6336
+ function stableFailureFingerprint(run, classification, summary) {
6337
+ const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
6107
6338
  return stableFingerprint([
6108
6339
  run.loopId,
6109
6340
  classification,
6110
6341
  String(run.status),
6111
6342
  String(run.exitCode ?? ""),
6112
- (run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6343
+ evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6113
6344
  ]);
6114
6345
  }
6115
6346
  function stableRouteFunctionalFingerprint(loop, reason) {
@@ -6127,13 +6358,279 @@ function healthRun(run) {
6127
6358
  stderr: redactedEvidence(run.stderr)
6128
6359
  };
6129
6360
  }
6361
+ function compactText(value, limit = 500) {
6362
+ const text = redact(bounded(value, limit));
6363
+ return text?.replace(/\s+/g, " ").trim();
6364
+ }
6365
+ function publicDoctorCheck(check) {
6366
+ return {
6367
+ id: check.id,
6368
+ status: check.status,
6369
+ message: redact(bounded(check.message, 500)) ?? "",
6370
+ detail: redact(bounded(check.detail, 800))
6371
+ };
6372
+ }
6373
+ function publicDoctorReport(report) {
6374
+ return {
6375
+ ok: report.ok,
6376
+ checks: report.checks.map(publicDoctorCheck)
6377
+ };
6378
+ }
6379
+ function includedStatusSet(statuses) {
6380
+ const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
6381
+ return new Set(values);
6382
+ }
6383
+ function statusCounts(loops) {
6384
+ return {
6385
+ loops: loops.length,
6386
+ active: loops.filter((loop) => loop.status === "active").length,
6387
+ paused: loops.filter((loop) => loop.status === "paused").length,
6388
+ stopped: loops.filter((loop) => loop.status === "stopped").length,
6389
+ expired: loops.filter((loop) => loop.status === "expired").length
6390
+ };
6391
+ }
6392
+ function compareLoopsForScan(left, right) {
6393
+ const statusOrder = left.status.localeCompare(right.status);
6394
+ if (statusOrder !== 0)
6395
+ return statusOrder;
6396
+ return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
6397
+ }
6398
+ function scanLoops(store, statuses, opts) {
6399
+ const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
6400
+ return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
6401
+ }
6402
+ function healthReportForLoops(store, loops, generatedAt) {
6403
+ const expectations = loops.map((loop) => expectationForLoop(store, loop));
6404
+ const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
6405
+ for (const expectation of expectations) {
6406
+ if (expectation.failure)
6407
+ classifications[expectation.failure.classification] += 1;
6408
+ }
6409
+ const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
6410
+ const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
6411
+ return {
6412
+ ok: unhealthy === 0,
6413
+ generatedAt,
6414
+ summary: {
6415
+ loops: expectations.length,
6416
+ healthy: expectations.length - unhealthy,
6417
+ unhealthy,
6418
+ warnings
6419
+ },
6420
+ classifications,
6421
+ expectations
6422
+ };
6423
+ }
6424
+ function ageMs(run, now) {
6425
+ const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
6426
+ const time = Date.parse(stamp);
6427
+ return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
6428
+ }
6429
+ function shortLoop(loop) {
6430
+ return {
6431
+ id: loop.id,
6432
+ name: loop.name,
6433
+ status: loop.status,
6434
+ nextRunAt: loop.nextRunAt,
6435
+ leaseMs: loop.leaseMs
6436
+ };
6437
+ }
6438
+ function priorityForSeverity(severity) {
6439
+ if (severity === "critical")
6440
+ return "critical";
6441
+ if (severity === "high")
6442
+ return "high";
6443
+ if (severity === "low")
6444
+ return "low";
6445
+ return "medium";
6446
+ }
6447
+ function recommendedFindingTask(finding, route) {
6448
+ const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
6449
+ if (finding.classification)
6450
+ tags.push(finding.classification);
6451
+ const description = [
6452
+ `OpenLoops health scan found a ${finding.kind} issue.`,
6453
+ finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
6454
+ finding.run ? `Run: ${finding.run.id}` : undefined,
6455
+ finding.classification ? `Classification: ${finding.classification}` : undefined,
6456
+ finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
6457
+ finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
6458
+ `Fingerprint: ${finding.fingerprint}`,
6459
+ `Severity: ${finding.severity}`,
6460
+ `No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
6461
+ route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
6462
+ route?.provider ? `Provider: ${route.provider}` : undefined,
6463
+ "",
6464
+ finding.message,
6465
+ finding.run?.error ? `Error:
6466
+ ${finding.run.error}` : undefined,
6467
+ finding.run?.stderr ? `Stderr:
6468
+ ${finding.run.stderr}` : undefined
6469
+ ].filter(Boolean).join(`
6470
+
6471
+ `);
6472
+ return {
6473
+ title: finding.title,
6474
+ description,
6475
+ priority: priorityForSeverity(finding.severity),
6476
+ tags,
6477
+ dedupeKey: finding.fingerprint,
6478
+ search: { query: finding.fingerprint },
6479
+ compatibilityFallback: {
6480
+ search: ["todos", "search", finding.fingerprint, "--json"],
6481
+ add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
6482
+ comment: ["todos", "comment", "<task-id>", description]
6483
+ },
6484
+ futureNativeUpsert: {
6485
+ command: "todos task upsert",
6486
+ fields: {
6487
+ title: finding.title,
6488
+ description,
6489
+ priority: priorityForSeverity(finding.severity),
6490
+ tags,
6491
+ dedupeKey: finding.fingerprint,
6492
+ routeSource: route?.source ?? "openloops",
6493
+ routeKind: route?.kind ?? "health_scan",
6494
+ routeLoopId: finding.loop?.id ?? "",
6495
+ routeLoopName: finding.loop?.name ?? ""
6496
+ }
6497
+ }
6498
+ };
6499
+ }
6500
+ function daemonFinding(daemon) {
6501
+ if (daemon.running && !daemon.stale)
6502
+ return;
6503
+ const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
6504
+ const severity = "critical";
6505
+ const finding = {
6506
+ kind: "daemon",
6507
+ severity,
6508
+ fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
6509
+ title: "OpenLoops daemon health issue",
6510
+ message: reason
6511
+ };
6512
+ return {
6513
+ ...finding,
6514
+ recommendedTask: recommendedFindingTask(finding, undefined)
6515
+ };
6516
+ }
6517
+ function doctorSeverity(check) {
6518
+ if (check.status === "fail" && check.id === "data-dir")
6519
+ return "critical";
6520
+ if (check.status === "fail")
6521
+ return "high";
6522
+ return "medium";
6523
+ }
6524
+ function doctorFinding(check, loop, route) {
6525
+ if (check.status === "ok" || check.id === "loop-runs")
6526
+ return;
6527
+ const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
6528
+ const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
6529
+ const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
6530
+ const finding = {
6531
+ kind,
6532
+ severity,
6533
+ fingerprint,
6534
+ title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
6535
+ message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
6536
+ loop: loop ? shortLoop(loop) : undefined,
6537
+ route,
6538
+ doctorCheck: publicDoctorCheck(check)
6539
+ };
6540
+ return {
6541
+ ...finding,
6542
+ recommendedTask: recommendedFindingTask(finding, route)
6543
+ };
6544
+ }
6545
+ function latestRunFinding(expectation) {
6546
+ if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
6547
+ return;
6548
+ const failure = expectation.failure;
6549
+ if (!failure)
6550
+ return;
6551
+ const severity = expectation.loop.status === "active" ? "high" : "medium";
6552
+ return {
6553
+ kind: "latest-run",
6554
+ severity,
6555
+ fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
6556
+ title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
6557
+ message: expectation.check.message,
6558
+ loop: expectation.loop,
6559
+ run: expectation.latestRun,
6560
+ route: expectation.route,
6561
+ classification: failure.classification,
6562
+ doctorCheck: undefined,
6563
+ recommendedTask: expectation.recommendedTask
6564
+ };
6565
+ }
6566
+ function staleRunningFinding(loop, expectation, now, staleRunningMs) {
6567
+ const run = expectation.latestRun;
6568
+ if (loop.status !== "active" || run?.status !== "running")
6569
+ return;
6570
+ const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
6571
+ const age = ageMs(run, now);
6572
+ if (age <= threshold)
6573
+ return;
6574
+ const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
6575
+ const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
6576
+ const finding = {
6577
+ kind: "stale-running",
6578
+ severity: "critical",
6579
+ fingerprint,
6580
+ title: `OpenLoops stale running run - ${loop.name}`,
6581
+ message,
6582
+ loop: shortLoop(loop),
6583
+ run,
6584
+ route: expectation.route,
6585
+ ageMs: age,
6586
+ staleThresholdMs: threshold
6587
+ };
6588
+ return {
6589
+ ...finding,
6590
+ recommendedTask: recommendedFindingTask(finding, expectation.route)
6591
+ };
6592
+ }
6593
+ function scanStatus(findings) {
6594
+ if (findings.some((finding) => finding.severity === "critical"))
6595
+ return "critical";
6596
+ if (findings.length > 0)
6597
+ return "degraded";
6598
+ return "ok";
6599
+ }
6600
+ function timestampDir(root, generatedAt) {
6601
+ const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
6602
+ return join7(root, stamp);
6603
+ }
6604
+ function healthScanMarkdown(scan) {
6605
+ return [
6606
+ "# OpenLoops Health Scan",
6607
+ "",
6608
+ `- status: ${scan.status}`,
6609
+ `- generated_at: ${scan.generatedAt}`,
6610
+ `- included_statuses: ${scan.includedStatuses.join(",")}`,
6611
+ `- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
6612
+ `- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
6613
+ scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
6614
+ scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
6615
+ scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
6616
+ "",
6617
+ "## Findings",
6618
+ scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
6619
+ `) : "None."
6620
+ ].join(`
6621
+ `);
6622
+ }
6130
6623
  function classifyRunFailure(run) {
6131
6624
  if (run.status === "succeeded" || run.status === "running")
6132
6625
  return;
6133
- const text = searchableText(run);
6626
+ const rawText = searchableText(run);
6627
+ const text = rawText.toLowerCase();
6134
6628
  let classification = "unknown";
6629
+ let summary;
6135
6630
  if (run.status === "timed_out")
6136
6631
  classification = "timeout";
6632
+ else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
6633
+ classification = "restart_interrupted";
6137
6634
  else if (run.status === "skipped" && /circuit breaker open/.test(text))
6138
6635
  classification = "circuit_breaker";
6139
6636
  else if (run.status === "skipped" && /previous run still active/.test(text))
@@ -6142,8 +6639,10 @@ function classifyRunFailure(run) {
6142
6639
  classification = "preflight";
6143
6640
  else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
6144
6641
  classification = "rate_limit";
6145
- else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
6642
+ else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
6146
6643
  classification = "auth";
6644
+ else if (summary = providerUnavailableSummary(rawText))
6645
+ classification = "provider_unavailable";
6147
6646
  else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
6148
6647
  classification = "model_not_found";
6149
6648
  else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
@@ -6156,8 +6655,9 @@ function classifyRunFailure(run) {
6156
6655
  classification = "sigsegv";
6157
6656
  return {
6158
6657
  classification,
6159
- fingerprint: stableFailureFingerprint(run, classification),
6658
+ fingerprint: stableFailureFingerprint(run, classification, summary),
6160
6659
  evidence: {
6660
+ summary,
6161
6661
  error: redactedEvidence(run.error),
6162
6662
  stdout: redactedEvidence(run.stdout),
6163
6663
  stderr: redactedEvidence(run.stderr),
@@ -6202,7 +6702,7 @@ function routeEvidenceReport(run) {
6202
6702
  const evidencePath = stringValue(stdoutReport?.evidencePath);
6203
6703
  if (evidencePath && existsSync4(evidencePath)) {
6204
6704
  try {
6205
- return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
6705
+ return parseJsonObject(readFileSync5(evidencePath, "utf8")) ?? stdoutReport;
6206
6706
  } catch {
6207
6707
  return stdoutReport;
6208
6708
  }
@@ -6334,6 +6834,12 @@ function targetRoute(loop) {
6334
6834
  loopName: loop.name
6335
6835
  };
6336
6836
  }
6837
+ function expectationLoop(loop) {
6838
+ return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
6839
+ }
6840
+ function hasPendingRetry(loop, run) {
6841
+ return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
6842
+ }
6337
6843
  function recommendedTask(loop, run, failure, route) {
6338
6844
  const title = `BUG: open-loops loop failure - ${loop.name}`;
6339
6845
  const description = [
@@ -6345,6 +6851,7 @@ function recommendedTask(loop, run, failure, route) {
6345
6851
  `No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
6346
6852
  route.cwd ? `Route cwd: ${route.cwd}` : undefined,
6347
6853
  route.provider ? `Provider: ${route.provider}` : undefined,
6854
+ failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
6348
6855
  failure.evidence.error ? `Error:
6349
6856
  ${failure.evidence.error}` : undefined,
6350
6857
  failure.evidence.stderr ? `Stderr:
@@ -6354,7 +6861,7 @@ ${failure.evidence.stderr}` : undefined
6354
6861
  `);
6355
6862
  const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
6356
6863
  const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
6357
- const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
6864
+ const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
6358
6865
  return {
6359
6866
  title,
6360
6867
  description,
@@ -6388,7 +6895,7 @@ function expectationForLoop(store, loop) {
6388
6895
  const route = targetRoute(loop);
6389
6896
  if (!latestRun) {
6390
6897
  return {
6391
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6898
+ loop: expectationLoop(loop),
6392
6899
  ok: true,
6393
6900
  check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
6394
6901
  route
@@ -6397,7 +6904,7 @@ function expectationForLoop(store, loop) {
6397
6904
  const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
6398
6905
  if (routeFailure) {
6399
6906
  return {
6400
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6907
+ loop: expectationLoop(loop),
6401
6908
  ok: false,
6402
6909
  check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
6403
6910
  latestRun: healthRun(latestRun),
@@ -6408,7 +6915,7 @@ function expectationForLoop(store, loop) {
6408
6915
  }
6409
6916
  if (latestRun.status === "succeeded") {
6410
6917
  return {
6411
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6918
+ loop: expectationLoop(loop),
6412
6919
  ok: true,
6413
6920
  check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
6414
6921
  latestRun: healthRun(latestRun),
@@ -6419,7 +6926,7 @@ function expectationForLoop(store, loop) {
6419
6926
  if (failure?.classification === "circuit_breaker") {
6420
6927
  if (loop.status !== "paused") {
6421
6928
  return {
6422
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6929
+ loop: expectationLoop(loop),
6423
6930
  ok: true,
6424
6931
  check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
6425
6932
  latestRun: healthRun(latestRun),
@@ -6427,7 +6934,7 @@ function expectationForLoop(store, loop) {
6427
6934
  };
6428
6935
  }
6429
6936
  return {
6430
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6937
+ loop: expectationLoop(loop),
6431
6938
  ok: false,
6432
6939
  check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
6433
6940
  latestRun: healthRun(latestRun),
@@ -6436,8 +6943,33 @@ function expectationForLoop(store, loop) {
6436
6943
  recommendedTask: recommendedTask(loop, latestRun, failure, route)
6437
6944
  };
6438
6945
  }
6946
+ if (failure?.classification === "restart_interrupted") {
6947
+ return {
6948
+ loop: expectationLoop(loop),
6949
+ ok: true,
6950
+ check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
6951
+ latestRun: healthRun(latestRun),
6952
+ failure,
6953
+ route
6954
+ };
6955
+ }
6956
+ if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
6957
+ const message = [
6958
+ "provider unavailable/network failure; retry is scheduled",
6959
+ loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
6960
+ failure.evidence.summary
6961
+ ].filter(Boolean).join("; ");
6962
+ return {
6963
+ loop: expectationLoop(loop),
6964
+ ok: true,
6965
+ check: { id: "latest-run-succeeded", status: "warn", message },
6966
+ latestRun: healthRun(latestRun),
6967
+ failure,
6968
+ route
6969
+ };
6970
+ }
6439
6971
  return {
6440
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
6972
+ loop: expectationLoop(loop),
6441
6973
  ok: false,
6442
6974
  check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
6443
6975
  latestRun: healthRun(latestRun),
@@ -6469,6 +7001,206 @@ function buildHealthReport(store, opts = {}) {
6469
7001
  expectations
6470
7002
  };
6471
7003
  }
7004
+ function buildHealthScan(store, opts = {}) {
7005
+ const generatedAt = (opts.now ?? new Date).toISOString();
7006
+ const now = opts.now ?? new Date(generatedAt);
7007
+ const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
7008
+ const loops = scanLoops(store, includeStatuses, opts);
7009
+ const health = healthReportForLoops(store, loops, generatedAt);
7010
+ const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
7011
+ const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
7012
+ const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
7013
+ const allFindings = [];
7014
+ const pushFinding = (finding) => {
7015
+ if (!finding)
7016
+ return;
7017
+ allFindings.push(finding);
7018
+ };
7019
+ if (opts.daemon)
7020
+ pushFinding(daemonFinding(opts.daemon));
7021
+ if (opts.latestRun !== false) {
7022
+ for (const loop of loops) {
7023
+ const expectation = expectationsByLoopId.get(loop.id);
7024
+ if (!expectation)
7025
+ continue;
7026
+ pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
7027
+ pushFinding(latestRunFinding(expectation));
7028
+ }
7029
+ }
7030
+ const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
7031
+ if (doctor) {
7032
+ for (const check of doctor.checks) {
7033
+ const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
7034
+ const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
7035
+ const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
7036
+ pushFinding(doctorFinding(check, loop, route));
7037
+ }
7038
+ }
7039
+ const findings = allFindings.slice(0, maxFindings);
7040
+ const status = scanStatus(allFindings);
7041
+ const baseCounts = statusCounts(loops);
7042
+ return {
7043
+ ok: status === "ok",
7044
+ status,
7045
+ generatedAt,
7046
+ includedStatuses: includeStatuses,
7047
+ counts: {
7048
+ ...baseCounts,
7049
+ latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
7050
+ staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
7051
+ daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
7052
+ doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
7053
+ preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
7054
+ findings: allFindings.length,
7055
+ reportedFindings: findings.length,
7056
+ truncatedFindings: Math.max(0, allFindings.length - findings.length)
7057
+ },
7058
+ daemon: opts.daemon ? {
7059
+ running: opts.daemon.running,
7060
+ stale: opts.daemon.stale,
7061
+ pid: opts.daemon.pid,
7062
+ host: opts.daemon.host,
7063
+ loops: opts.daemon.loops,
7064
+ runs: opts.daemon.runs,
7065
+ logPath: opts.daemon.logPath
7066
+ } : undefined,
7067
+ doctor,
7068
+ health,
7069
+ selfHeals: opts.selfHeals ?? [],
7070
+ findings
7071
+ };
7072
+ }
7073
+ function writeHealthScanReports(scan, opts = {}) {
7074
+ const root = opts.reportDir ?? join7(dataDir(), "reports", "health-scan");
7075
+ const dir = timestampDir(root, scan.generatedAt);
7076
+ mkdirSync6(dir, { recursive: true, mode: 448 });
7077
+ const json = join7(dir, "summary.json");
7078
+ const markdown = join7(dir, "report.md");
7079
+ const withReports = {
7080
+ ...scan,
7081
+ reports: { dir, json, markdown }
7082
+ };
7083
+ writeFileSync3(json, JSON.stringify(withReports, null, 2), { mode: 384 });
7084
+ writeFileSync3(markdown, healthScanMarkdown(withReports), { mode: 384 });
7085
+ return withReports;
7086
+ }
7087
+
7088
+ // src/lib/doctor.ts
7089
+ var PROVIDER_COMMANDS = [
7090
+ "claude",
7091
+ "agent",
7092
+ "codewith",
7093
+ "aicopilot",
7094
+ "opencode",
7095
+ "codex"
7096
+ ];
7097
+ function hasCommand(command) {
7098
+ const result = spawnSync6("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
7099
+ return (result.status ?? 1) === 0;
7100
+ }
7101
+ function commandVersion(command) {
7102
+ const result = spawnSync6(command, ["--version"], {
7103
+ encoding: "utf8",
7104
+ stdio: ["ignore", "pipe", "pipe"]
7105
+ });
7106
+ if ((result.status ?? 1) !== 0)
7107
+ return;
7108
+ return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
7109
+ }
7110
+ function runDoctor(store) {
7111
+ const checks = [];
7112
+ try {
7113
+ const dir = ensureDataDir();
7114
+ accessSync2(dir, constants2.R_OK | constants2.W_OK);
7115
+ checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
7116
+ } catch (error) {
7117
+ checks.push({
7118
+ id: "data-dir",
7119
+ status: "fail",
7120
+ message: "data directory is not writable",
7121
+ detail: error instanceof Error ? error.message : String(error)
7122
+ });
7123
+ }
7124
+ const bunVersion = commandVersion("bun");
7125
+ checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
7126
+ const accountsVersion = commandVersion("accounts");
7127
+ checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
7128
+ try {
7129
+ const machines = listOpenMachines();
7130
+ const local = machines.find((machine) => machine.local);
7131
+ checks.push({
7132
+ id: "machines",
7133
+ status: "ok",
7134
+ message: `OpenMachines topology available (${machines.length} machine(s))`,
7135
+ detail: local ? `local=${local.id}` : undefined
7136
+ });
7137
+ } catch (error) {
7138
+ checks.push({
7139
+ id: "machines",
7140
+ status: "warn",
7141
+ message: "OpenMachines topology is not available; machine-assigned loops will fail",
7142
+ detail: error instanceof Error ? error.message : String(error)
7143
+ });
7144
+ }
7145
+ for (const command of PROVIDER_COMMANDS) {
7146
+ checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
7147
+ }
7148
+ const status = daemonStatus(store);
7149
+ checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
7150
+ const failedRuns = store.countRuns("failed");
7151
+ const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
7152
+ checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
7153
+ if (restartInterruptedRuns > 0) {
7154
+ checks.push({
7155
+ id: "loop-runs:restart-interrupted",
7156
+ status: "warn",
7157
+ message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
7158
+ });
7159
+ }
7160
+ const deployment = buildDeploymentStatus();
7161
+ const schedulerState = deployment.schedulerState;
7162
+ checks.push({
7163
+ id: "scheduler-state",
7164
+ status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
7165
+ message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
7166
+ detail: [
7167
+ `route_state=${schedulerState.routeAdmission.stateStore}`,
7168
+ `active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
7169
+ `gates=${schedulerState.routeAdmission.gates.join(",")}`,
7170
+ `artifacts=${schedulerState.localStore.runArtifacts}`,
7171
+ `remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
7172
+ `remote_apply=${String(schedulerState.remoteStore.applySupported)}`
7173
+ ].join(" ")
7174
+ });
7175
+ for (const loop of store.listLoops({ status: "active" })) {
7176
+ try {
7177
+ if (loop.target.type === "workflow") {
7178
+ const workflow = store.requireWorkflow(loop.target.workflowId);
7179
+ for (const step of workflowExecutionOrder(workflow)) {
7180
+ preflightTarget({
7181
+ ...step.target,
7182
+ account: step.account ?? step.target.account,
7183
+ timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
7184
+ }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
7185
+ }
7186
+ } else {
7187
+ preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
7188
+ }
7189
+ checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
7190
+ } catch (error) {
7191
+ checks.push({
7192
+ id: `loop:${loop.id}:preflight`,
7193
+ status: "fail",
7194
+ message: `active loop target preflight failed: ${loop.name}`,
7195
+ detail: error instanceof Error ? error.message : String(error)
7196
+ });
7197
+ }
7198
+ }
7199
+ return {
7200
+ ok: checks.every((check) => check.status !== "fail"),
7201
+ checks
7202
+ };
7203
+ }
6472
7204
 
6473
7205
  // src/lib/migration.ts
6474
7206
  import { createHash as createHash3 } from "crypto";
@@ -7977,7 +8709,7 @@ var MAX_RETRY_EXPONENT = 20;
7977
8709
  function retryBackoffDelayMs(loop, run, random = Math.random) {
7978
8710
  const attempt = Math.max(1, run.attempt);
7979
8711
  const failure = classifyRunFailure(run);
7980
- const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
8712
+ const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
7981
8713
  const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
7982
8714
  const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
7983
8715
  const jitter = 0.5 + random();
@@ -8104,20 +8836,21 @@ async function executeClaimedRun(deps) {
8104
8836
  daemonLeaseId: deps.daemonLeaseId,
8105
8837
  onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
8106
8838
  })))(deps.loop, deps.run);
8839
+ const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
8107
8840
  deps.beforeFinalize?.(deps.loop, deps.run);
8108
8841
  return deps.store.finalizeRun(deps.run.id, {
8109
- status: result.status,
8110
- finishedAt: result.finishedAt,
8111
- durationMs: result.durationMs,
8112
- stdout: result.stdout,
8113
- stderr: result.stderr,
8114
- exitCode: result.exitCode,
8115
- error: result.error,
8116
- pid: result.pid
8842
+ status: finalResult.status,
8843
+ finishedAt: finalResult.finishedAt,
8844
+ durationMs: finalResult.durationMs,
8845
+ stdout: finalResult.stdout,
8846
+ stderr: finalResult.stderr,
8847
+ exitCode: finalResult.exitCode,
8848
+ error: finalResult.error,
8849
+ pid: finalResult.pid
8117
8850
  }, {
8118
8851
  claimedBy: deps.runnerId,
8119
8852
  daemonLeaseId: deps.daemonLeaseId,
8120
- now: deps.now?.() ?? new Date(result.finishedAt)
8853
+ now: deps.now?.() ?? new Date(finalResult.finishedAt)
8121
8854
  });
8122
8855
  } catch (err) {
8123
8856
  deps.onError?.(deps.loop, err);
@@ -8418,6 +9151,13 @@ class LoopsClient {
8418
9151
  health(opts = {}) {
8419
9152
  return buildHealthReport(this.store, opts);
8420
9153
  }
9154
+ healthScan(opts = {}) {
9155
+ return buildHealthScan(this.store, {
9156
+ ...opts,
9157
+ doctor: opts.doctor ? runDoctor(this.store) : undefined,
9158
+ daemon: opts.daemon ? daemonStatus(this.store) : undefined
9159
+ });
9160
+ }
8421
9161
  goal(idOrName) {
8422
9162
  const goal = this.store.getGoal(idOrName) ?? this.store.findGoalByLoop(idOrName) ?? this.store.findGoalByRunId(idOrName);
8423
9163
  return {