@hasna/loops 0.4.13 → 0.4.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +99 -3
  2. package/README.md +171 -39
  3. package/dist/api/index.d.ts +36 -0
  4. package/dist/api/index.js +1518 -15
  5. package/dist/cli/index.js +7280 -3213
  6. package/dist/cli/ui.d.ts +44 -0
  7. package/dist/daemon/index.js +1433 -303
  8. package/dist/generated/storage-kit/health.d.ts +19 -0
  9. package/dist/generated/storage-kit/index.d.ts +7 -0
  10. package/dist/generated/storage-kit/migrations.d.ts +47 -0
  11. package/dist/generated/storage-kit/mode.d.ts +47 -0
  12. package/dist/generated/storage-kit/pool.d.ts +33 -0
  13. package/dist/generated/storage-kit/query.d.ts +35 -0
  14. package/dist/generated/storage-kit/tls.d.ts +25 -0
  15. package/dist/index.d.ts +2 -2
  16. package/dist/index.js +8689 -4837
  17. package/dist/lib/cloud/mode.d.ts +17 -0
  18. package/dist/lib/cloud/resolve.d.ts +16 -0
  19. package/dist/lib/cloud/storage.d.ts +82 -0
  20. package/dist/lib/cloud/transport.d.ts +148 -0
  21. package/dist/lib/format.d.ts +2 -1
  22. package/dist/lib/health.d.ts +83 -3
  23. package/dist/lib/migration.d.ts +40 -0
  24. package/dist/lib/mode.js +15 -3
  25. package/dist/lib/route/index.d.ts +2 -0
  26. package/dist/lib/route/policies.d.ts +51 -0
  27. package/dist/lib/route/provider-admission.d.ts +37 -0
  28. package/dist/lib/route/throttle.d.ts +4 -0
  29. package/dist/lib/route/types.d.ts +8 -0
  30. package/dist/lib/run-receipts.d.ts +14 -0
  31. package/dist/lib/scheduler.d.ts +5 -2
  32. package/dist/lib/storage/contract.d.ts +7 -1
  33. package/dist/lib/storage/index.d.ts +1 -0
  34. package/dist/lib/storage/index.js +2028 -231
  35. package/dist/lib/storage/pg-executor.d.ts +27 -0
  36. package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
  37. package/dist/lib/storage/postgres-loop-storage.d.ts +120 -0
  38. package/dist/lib/storage/postgres-schema.js +29 -0
  39. package/dist/lib/storage/postgres.js +29 -0
  40. package/dist/lib/storage/sqlite.d.ts +6 -0
  41. package/dist/lib/storage/sqlite.js +748 -26
  42. package/dist/lib/store/index.d.ts +268 -0
  43. package/dist/lib/store.d.ts +282 -1
  44. package/dist/lib/store.js +746 -26
  45. package/dist/lib/template-kit.d.ts +25 -0
  46. package/dist/lib/templates.d.ts +16 -1
  47. package/dist/mcp/http.d.ts +16 -0
  48. package/dist/mcp/index.js +2885 -634
  49. package/dist/runner/index.js +198 -32
  50. package/dist/sdk/http.d.ts +182 -0
  51. package/dist/sdk/http.js +166 -0
  52. package/dist/sdk/index.d.ts +55 -18
  53. package/dist/sdk/index.js +2734 -617
  54. package/dist/serve/index.d.ts +4 -0
  55. package/dist/serve/index.js +9095 -0
  56. package/dist/types.d.ts +52 -0
  57. package/docs/AUTOMATION_RUNTIME_DESIGN.md +442 -1
  58. package/docs/CUTOVER-RUNBOOK.md +78 -0
  59. package/docs/DEPLOYMENT_MODES.md +35 -18
  60. package/docs/RUNTIME_BOUNDARY.md +203 -0
  61. package/docs/USAGE.md +195 -38
  62. package/package.json +15 -3
package/dist/sdk/index.js CHANGED
@@ -1,9 +1,9 @@
1
1
  // @bun
2
2
  // src/lib/store.ts
3
3
  import { Database } from "bun:sqlite";
4
- import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
5
- import { tmpdir } from "os";
6
- import { basename as basename2, dirname as dirname2, join as join3 } from "path";
4
+ import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
5
+ import { tmpdir as tmpdir2 } from "os";
6
+ import { basename as basename2, dirname as dirname2, join as join4 } from "path";
7
7
 
8
8
  // src/lib/errors.ts
9
9
  class CodedError extends Error {
@@ -1202,15 +1202,378 @@ function discardWorkflowRunManifest(staged) {
1202
1202
  } catch {}
1203
1203
  }
1204
1204
 
1205
+ // src/lib/run-receipts.ts
1206
+ import { createHash as createHash2 } from "crypto";
1207
+ import { hostname } from "os";
1208
+
1209
+ // src/lib/run-envelope.ts
1210
+ var ENVELOPE_EXCERPT_CHARS = 2048;
1211
+ var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
1212
+ function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
1213
+ if (!text)
1214
+ return;
1215
+ const scrubbed = scrubSecrets(text);
1216
+ if (scrubbed.length <= limit * 2)
1217
+ return scrubbed;
1218
+ const omitted = scrubbed.length - limit * 2;
1219
+ return `${scrubbed.slice(0, limit)}
1220
+ [... ${omitted} chars omitted ...]
1221
+ ${scrubbed.slice(-limit)}`;
1222
+ }
1223
+ function summarizeOutput(stdout, stderr) {
1224
+ return {
1225
+ stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
1226
+ stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
1227
+ stdoutExcerpt: boundedExcerpt(stdout),
1228
+ stderrExcerpt: boundedExcerpt(stderr)
1229
+ };
1230
+ }
1231
+ function summarizeExecutorResult(result) {
1232
+ return {
1233
+ ...summarizeOutput(result.stdout, result.stderr),
1234
+ status: result.status,
1235
+ exitCode: result.exitCode,
1236
+ durationMs: result.durationMs,
1237
+ error: boundedExcerpt(result.error)
1238
+ };
1239
+ }
1240
+ function isBlockedStepRun(step) {
1241
+ return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
1242
+ }
1243
+ function summarizeWorkflowStepRun(step) {
1244
+ return {
1245
+ ...summarizeOutput(step.stdout, step.stderr),
1246
+ stepId: step.stepId,
1247
+ status: step.status,
1248
+ exitCode: step.exitCode,
1249
+ durationMs: step.durationMs,
1250
+ error: boundedExcerpt(step.error),
1251
+ blocked: isBlockedStepRun(step)
1252
+ };
1253
+ }
1254
+ function workflowRunEnvelope(run, steps) {
1255
+ return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
1256
+ }
1257
+
1258
+ // src/lib/run-receipts.ts
1259
+ var RUN_RECEIPT_SUMMARY_TEXT_CHARS = 4096;
1260
+ var RUN_RECEIPT_MAX_IDS = 100;
1261
+ var RUN_RECEIPT_MAX_EVIDENCE_PATHS = 100;
1262
+ var RUN_RECEIPT_MAX_PATH_CHARS = 1024;
1263
+ function canonicalJson(value) {
1264
+ if (Array.isArray(value))
1265
+ return `[${value.map(canonicalJson).join(",")}]`;
1266
+ if (value && typeof value === "object") {
1267
+ return `{${Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
1268
+ }
1269
+ return JSON.stringify(value);
1270
+ }
1271
+ function digestReceipt(receipt) {
1272
+ return `sha256:${createHash2("sha256").update(canonicalJson(receipt)).digest("hex")}`;
1273
+ }
1274
+ function isoOrNull(value, label) {
1275
+ if (value === undefined || value === null || value === "")
1276
+ return null;
1277
+ const date = new Date(value);
1278
+ if (Number.isNaN(date.getTime()))
1279
+ throw new Error(`${label} must be a valid ISO date/time`);
1280
+ return date.toISOString();
1281
+ }
1282
+ function nonEmpty(value, label) {
1283
+ const trimmed = value?.trim();
1284
+ if (!trimmed)
1285
+ throw new Error(`${label} must be non-empty`);
1286
+ return trimmed;
1287
+ }
1288
+ function normalizedStringArray(values, opts) {
1289
+ const out = [];
1290
+ const seen = new Set;
1291
+ for (const raw of values ?? []) {
1292
+ const value = String(raw).trim();
1293
+ if (!value || seen.has(value))
1294
+ continue;
1295
+ seen.add(value);
1296
+ out.push(opts.itemMax ? value.slice(0, opts.itemMax) : value);
1297
+ if (out.length >= opts.max)
1298
+ break;
1299
+ }
1300
+ if ((values?.length ?? 0) > opts.max) {
1301
+ out.push(`[truncated ${values.length - opts.max} ${opts.label}]`);
1302
+ }
1303
+ return out;
1304
+ }
1305
+ function receiptMachine(input, opts) {
1306
+ if (typeof input.machine === "string")
1307
+ return nonEmpty(input.machine, "machine");
1308
+ if (input.machine && typeof input.machine === "object")
1309
+ return input.machine;
1310
+ if (opts.loop?.machine)
1311
+ return { ...opts.loop.machine };
1312
+ if (typeof opts.defaultMachine === "string")
1313
+ return nonEmpty(opts.defaultMachine, "machine");
1314
+ if (opts.defaultMachine && typeof opts.defaultMachine === "object")
1315
+ return opts.defaultMachine;
1316
+ return hostname();
1317
+ }
1318
+ function normalizeSummary(input, run) {
1319
+ const stdout = input.stdout ?? run?.stdout;
1320
+ const stderr = input.stderr ?? run?.stderr;
1321
+ const output = summarizeOutput(stdout, stderr);
1322
+ const summaryInput = input.summary;
1323
+ const provided = typeof summaryInput === "object" && summaryInput !== null ? summaryInput : {};
1324
+ const text = typeof summaryInput === "string" ? boundedExcerpt(summaryInput, Math.floor(RUN_RECEIPT_SUMMARY_TEXT_CHARS / 2)) : typeof provided.text === "string" ? boundedExcerpt(provided.text, Math.floor(RUN_RECEIPT_SUMMARY_TEXT_CHARS / 2)) : undefined;
1325
+ return {
1326
+ text,
1327
+ stdout_bytes: Number.isFinite(provided.stdout_bytes) ? Number(provided.stdout_bytes) : output.stdoutBytes,
1328
+ stderr_bytes: Number.isFinite(provided.stderr_bytes) ? Number(provided.stderr_bytes) : output.stderrBytes,
1329
+ stdout_excerpt: typeof provided.stdout_excerpt === "string" ? boundedExcerpt(provided.stdout_excerpt) : output.stdoutExcerpt,
1330
+ stderr_excerpt: typeof provided.stderr_excerpt === "string" ? boundedExcerpt(provided.stderr_excerpt) : output.stderrExcerpt,
1331
+ error: typeof provided.error === "string" ? boundedExcerpt(provided.error) : boundedExcerpt(input.error ?? run?.error),
1332
+ duration_ms: Number.isFinite(provided.duration_ms) ? Number(provided.duration_ms) : input.duration_ms ?? run?.durationMs
1333
+ };
1334
+ }
1335
+ function normalizeRunReceipt(input, opts = {}) {
1336
+ const now = (opts.now ?? new Date).toISOString();
1337
+ const run = opts.run;
1338
+ const loopId = input.loop_id ?? run?.loopId ?? opts.loop?.id;
1339
+ const normalized = {
1340
+ loop_id: nonEmpty(loopId, "loop_id"),
1341
+ run_id: nonEmpty(input.run_id ?? run?.id, "run_id"),
1342
+ machine: receiptMachine(input, opts),
1343
+ repo: nonEmpty(input.repo ?? opts.defaultRepo ?? targetRepo(opts.loop) ?? process.cwd(), "repo"),
1344
+ task_ids: normalizedStringArray(input.task_ids, { max: RUN_RECEIPT_MAX_IDS, label: "task_ids" }),
1345
+ knowledge_ids: normalizedStringArray(input.knowledge_ids, { max: RUN_RECEIPT_MAX_IDS, label: "knowledge_ids" }),
1346
+ started_at: isoOrNull(input.started_at ?? run?.startedAt, "started_at"),
1347
+ finished_at: isoOrNull(input.finished_at ?? run?.finishedAt, "finished_at"),
1348
+ status: nonEmpty(input.status ?? run?.status, "status"),
1349
+ exit_code: input.exit_code === undefined ? run?.exitCode ?? null : input.exit_code,
1350
+ summary: normalizeSummary(input, run),
1351
+ evidence_paths: normalizedStringArray(input.evidence_paths, {
1352
+ max: RUN_RECEIPT_MAX_EVIDENCE_PATHS,
1353
+ label: "evidence_paths",
1354
+ itemMax: RUN_RECEIPT_MAX_PATH_CHARS
1355
+ })
1356
+ };
1357
+ return {
1358
+ ...normalized,
1359
+ digest_id: input.digest_id?.trim() || digestReceipt(normalized),
1360
+ created_at: opts.existing?.created_at ?? now,
1361
+ updated_at: now
1362
+ };
1363
+ }
1364
+ function targetRepo(loop) {
1365
+ if (!loop)
1366
+ return;
1367
+ if (loop.target.type === "command")
1368
+ return loop.target.cwd;
1369
+ if (loop.target.type === "agent")
1370
+ return loop.target.cwd;
1371
+ return;
1372
+ }
1373
+
1374
+ // src/lib/route/todos-cli.ts
1375
+ import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
1376
+ import { spawnSync as spawnSync2 } from "child_process";
1377
+ import { join as join3 } from "path";
1378
+ import { homedir as homedir2, tmpdir } from "os";
1379
+
1380
+ // src/lib/format.ts
1381
+ var TEXT_OUTPUT_LIMIT = 32 * 1024;
1382
+ var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
1383
+ function redact(value, visible = 0) {
1384
+ if (!value)
1385
+ return value;
1386
+ if (value.length <= visible)
1387
+ return value;
1388
+ if (visible <= 0)
1389
+ return `[redacted ${value.length} chars]`;
1390
+ return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
1391
+ }
1392
+ function truncateTextOutput(value) {
1393
+ if (value.length <= TEXT_OUTPUT_LIMIT)
1394
+ return value;
1395
+ return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
1396
+ [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
1397
+ }
1398
+ function redactSensitivePayload(value, key) {
1399
+ if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
1400
+ if (typeof value === "string")
1401
+ return redact(value);
1402
+ if (value === undefined || value === null)
1403
+ return value;
1404
+ return "[redacted]";
1405
+ }
1406
+ if (Array.isArray(value))
1407
+ return value.map((item) => redactSensitivePayload(item));
1408
+ if (value && typeof value === "object") {
1409
+ return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
1410
+ }
1411
+ return value;
1412
+ }
1413
+ function textOutputBlocks(value, opts = {}) {
1414
+ const indent = opts.indent ?? "";
1415
+ const nested = `${indent} `;
1416
+ const blocks = [];
1417
+ for (const [label, output] of [
1418
+ ["stdout", value.stdout],
1419
+ ["stderr", value.stderr]
1420
+ ]) {
1421
+ if (!output)
1422
+ continue;
1423
+ blocks.push(`${indent}${label}:`);
1424
+ for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
1425
+ blocks.push(`${nested}${line}`);
1426
+ }
1427
+ }
1428
+ return blocks;
1429
+ }
1430
+ function publicLoop(loop) {
1431
+ const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
1432
+ return {
1433
+ ...loop,
1434
+ target
1435
+ };
1436
+ }
1437
+ function publicRun(run, showOutput = false, opts = {}) {
1438
+ return {
1439
+ ...run,
1440
+ stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
1441
+ stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
1442
+ error: opts.redactError ? redact(run.error) : run.error
1443
+ };
1444
+ }
1445
+ function publicRunReceipt(receipt) {
1446
+ return { ...receipt };
1447
+ }
1448
+ function publicExecutorResult(result, showOutput = false) {
1449
+ return {
1450
+ ...result,
1451
+ stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
1452
+ stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
1453
+ error: redact(result.error)
1454
+ };
1455
+ }
1456
+ function publicWorkflow(workflow) {
1457
+ return {
1458
+ ...workflow,
1459
+ steps: workflow.steps.map((step) => ({
1460
+ ...step,
1461
+ target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
1462
+ }))
1463
+ };
1464
+ }
1465
+ function publicWorkflowRun(run) {
1466
+ return { ...run, error: redact(run.error) };
1467
+ }
1468
+ function publicWorkflowInvocation(invocation) {
1469
+ return redactSensitivePayload(invocation);
1470
+ }
1471
+ function publicWorkflowWorkItem(item) {
1472
+ return { ...item, lastReason: redact(item.lastReason, 240) };
1473
+ }
1474
+ function publicWorkflowStepRun(run, showOutput = false) {
1475
+ return {
1476
+ ...run,
1477
+ stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
1478
+ stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
1479
+ error: redact(run.error)
1480
+ };
1481
+ }
1482
+ function publicWorkflowEvent(event) {
1483
+ return { ...event, payload: redactSensitivePayload(event.payload) };
1484
+ }
1485
+ function publicGoal(goal) {
1486
+ return {
1487
+ ...goal,
1488
+ objective: redact(goal.objective, 120)
1489
+ };
1490
+ }
1491
+ function publicGoalRun(run) {
1492
+ return {
1493
+ ...run,
1494
+ evidence: redactSensitivePayload(run.evidence),
1495
+ rawResponse: redactSensitivePayload(run.rawResponse)
1496
+ };
1497
+ }
1498
+
1499
+ // src/lib/route/todos-cli.ts
1500
+ function defaultLoopsProject() {
1501
+ return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
1502
+ }
1503
+ function runLocalCommand(command, args, opts = {}) {
1504
+ const result = spawnSync2(command, args, {
1505
+ input: opts.input,
1506
+ encoding: "utf8",
1507
+ timeout: opts.timeoutMs ?? 30000,
1508
+ maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
1509
+ env: process.env
1510
+ });
1511
+ return {
1512
+ ok: result.status === 0,
1513
+ status: result.status,
1514
+ stdout: result.stdout || "",
1515
+ stderr: result.stderr || "",
1516
+ error: result.error ? String(result.error.message || result.error) : ""
1517
+ };
1518
+ }
1519
+ function runLocalCommandWithStdoutFile(command, args, opts = {}) {
1520
+ const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
1521
+ const stdoutPath = join3(tempDir, "stdout");
1522
+ const stdoutFd = openSync(stdoutPath, "w");
1523
+ let result;
1524
+ try {
1525
+ result = spawnSync2(command, args, {
1526
+ input: opts.input,
1527
+ encoding: "utf8",
1528
+ timeout: opts.timeoutMs ?? 30000,
1529
+ maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
1530
+ env: process.env,
1531
+ stdio: ["pipe", stdoutFd, "pipe"]
1532
+ });
1533
+ } finally {
1534
+ closeSync(stdoutFd);
1535
+ }
1536
+ try {
1537
+ return {
1538
+ ok: result.status === 0,
1539
+ status: result.status,
1540
+ stdout: readFileSync3(stdoutPath, "utf8"),
1541
+ stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
1542
+ error: result.error ? String(result.error.message || result.error) : ""
1543
+ };
1544
+ } finally {
1545
+ rmSync2(tempDir, { recursive: true, force: true });
1546
+ }
1547
+ }
1548
+ function ensureTodosTaskList(project, slug, name, description) {
1549
+ runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
1550
+ const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
1551
+ if (!list.ok)
1552
+ throw new Error(list.stderr || list.error || "failed to list todos task lists");
1553
+ const values = JSON.parse(list.stdout || "[]");
1554
+ const found = values.find((entry) => entry.slug === slug);
1555
+ if (!found)
1556
+ throw new Error(`todos task list not found after ensure: ${slug}`);
1557
+ return found.id;
1558
+ }
1559
+ function todosMutationSummary(result) {
1560
+ return {
1561
+ ok: result.ok,
1562
+ status: result.status,
1563
+ error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
1564
+ };
1565
+ }
1566
+
1205
1567
  // src/lib/store.ts
1206
1568
  var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
1207
1569
  var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
1208
1570
  var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
1209
- var SCHEMA_USER_VERSION = 7;
1571
+ var SCHEMA_USER_VERSION = 8;
1210
1572
  var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
1211
1573
  var PRUNE_BATCH_SIZE = 400;
1212
1574
  var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
1213
1575
  var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
1576
+ var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
1214
1577
  function rowToLoop(row) {
1215
1578
  return {
1216
1579
  id: row.id,
@@ -1261,6 +1624,28 @@ function rowToRun(row) {
1261
1624
  updatedAt: row.updated_at
1262
1625
  };
1263
1626
  }
1627
+ function rowToRunReceipt(row) {
1628
+ return {
1629
+ loop_id: row.loop_id,
1630
+ run_id: row.run_id,
1631
+ machine: JSON.parse(row.machine_json),
1632
+ repo: row.repo,
1633
+ task_ids: JSON.parse(row.task_ids_json),
1634
+ knowledge_ids: JSON.parse(row.knowledge_ids_json),
1635
+ digest_id: row.digest_id,
1636
+ started_at: row.started_at,
1637
+ finished_at: row.finished_at,
1638
+ status: row.status,
1639
+ exit_code: row.exit_code,
1640
+ summary: JSON.parse(row.summary_json),
1641
+ evidence_paths: JSON.parse(row.evidence_paths_json),
1642
+ created_at: row.created_at,
1643
+ updated_at: row.updated_at
1644
+ };
1645
+ }
1646
+ function latestRunTime(row) {
1647
+ return row.finished_at ?? row.started_at ?? row.created_at;
1648
+ }
1264
1649
  function rowToWorkflow(row) {
1265
1650
  return {
1266
1651
  id: row.id,
@@ -1321,6 +1706,7 @@ function rowToWorkflowWorkItem(row) {
1321
1706
  subjectRef: row.subject_ref,
1322
1707
  projectKey: row.project_key ?? undefined,
1323
1708
  projectGroup: row.project_group ?? undefined,
1709
+ machineId: row.machine_id ?? undefined,
1324
1710
  routeScope: row.route_scope ?? undefined,
1325
1711
  priority: row.priority,
1326
1712
  status: row.status,
@@ -1478,6 +1864,7 @@ function scrubbedOrNull(value) {
1478
1864
  return value == null ? null : scrubSecrets(value);
1479
1865
  }
1480
1866
  var MAX_PERSISTED_RUN_OUTPUT_CHARS = 64 * 1024;
1867
+ var MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS = 64 * 1024;
1481
1868
  function clampPersistedRunOutput(value) {
1482
1869
  if (value == null || value.length <= MAX_PERSISTED_RUN_OUTPUT_CHARS)
1483
1870
  return value;
@@ -1492,6 +1879,42 @@ ${tail}`;
1492
1879
  function persistedRunOutput(value) {
1493
1880
  return clampPersistedRunOutput(scrubbedOrNull(value));
1494
1881
  }
1882
+ function clampTextToChars(value, maxChars, reason) {
1883
+ if (value.length <= maxChars)
1884
+ return value;
1885
+ const marker = `
1886
+ ...[truncated by ${reason}]...
1887
+ `;
1888
+ const budget = Math.max(0, maxChars - marker.length);
1889
+ const headLength = Math.ceil(budget / 2);
1890
+ const tailLength = Math.floor(budget / 2);
1891
+ return `${value.slice(0, headLength)}${marker}${value.slice(value.length - tailLength)}`;
1892
+ }
1893
+ function boundedWorkflowEventPayloadJson(scrubbedJson) {
1894
+ if (scrubbedJson.length <= MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS)
1895
+ return scrubbedJson;
1896
+ const base = {
1897
+ truncated: true,
1898
+ originalChars: scrubbedJson.length,
1899
+ maxChars: MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS,
1900
+ preview: ""
1901
+ };
1902
+ const baseChars = JSON.stringify(base).length;
1903
+ let previewBudget = Math.max(0, MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS - baseChars - 64);
1904
+ while (true) {
1905
+ const preview = clampTextToChars(scrubbedJson, previewBudget, "loops workflow-event payload retention");
1906
+ const bounded = JSON.stringify({ ...base, preview });
1907
+ if (bounded.length <= MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS || previewBudget === 0)
1908
+ return bounded;
1909
+ previewBudget = Math.max(0, previewBudget - (bounded.length - MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS) - 64);
1910
+ }
1911
+ }
1912
+ function persistedWorkflowEventPayload(payload) {
1913
+ if (payload == null)
1914
+ return null;
1915
+ const scrubbed = scrubSecretsDeep(payload);
1916
+ return boundedWorkflowEventPayloadJson(scrubSecrets(JSON.stringify(scrubbed)));
1917
+ }
1495
1918
  function chmodIfExists(path, mode) {
1496
1919
  try {
1497
1920
  if (existsSync(path))
@@ -1515,7 +1938,7 @@ class Store {
1515
1938
  const file = path ?? dbPath();
1516
1939
  if (file !== ":memory:")
1517
1940
  ensurePrivateStorePath(file);
1518
- this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
1941
+ this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
1519
1942
  if (file === ":memory:")
1520
1943
  this.memoryRootDir = this.rootDir;
1521
1944
  this.db = new Database(file);
@@ -1609,6 +2032,17 @@ class Store {
1609
2032
  this.addColumnIfMissing("workflow_work_items", "route_scope", "TEXT");
1610
2033
  this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_scope ON workflow_work_items(route_scope, status)");
1611
2034
  }
2035
+ },
2036
+ {
2037
+ id: "0009_run_receipts",
2038
+ apply: () => this.createRunReceiptsSchema()
2039
+ },
2040
+ {
2041
+ id: "0010_work_item_machine_id",
2042
+ apply: () => {
2043
+ this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
2044
+ this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
2045
+ }
1612
2046
  }
1613
2047
  ];
1614
2048
  }
@@ -1670,6 +2104,28 @@ class Store {
1670
2104
  CREATE INDEX IF NOT EXISTS idx_runs_status_lease ON loop_runs(status, lease_expires_at);
1671
2105
  CREATE INDEX IF NOT EXISTS idx_runs_scheduled ON loop_runs(scheduled_for);
1672
2106
 
2107
+ CREATE TABLE IF NOT EXISTS run_receipts (
2108
+ run_id TEXT PRIMARY KEY,
2109
+ loop_id TEXT NOT NULL,
2110
+ machine_json TEXT NOT NULL,
2111
+ repo TEXT NOT NULL,
2112
+ task_ids_json TEXT NOT NULL,
2113
+ knowledge_ids_json TEXT NOT NULL,
2114
+ digest_id TEXT NOT NULL,
2115
+ started_at TEXT,
2116
+ finished_at TEXT,
2117
+ status TEXT NOT NULL,
2118
+ exit_code INTEGER,
2119
+ summary_json TEXT NOT NULL,
2120
+ evidence_paths_json TEXT NOT NULL,
2121
+ created_at TEXT NOT NULL,
2122
+ updated_at TEXT NOT NULL
2123
+ );
2124
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_loop ON run_receipts(loop_id, created_at DESC);
2125
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_repo ON run_receipts(repo, created_at DESC);
2126
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_digest ON run_receipts(digest_id);
2127
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_status ON run_receipts(status, created_at DESC);
2128
+
1673
2129
  CREATE TABLE IF NOT EXISTS daemon_lease (
1674
2130
  id TEXT PRIMARY KEY,
1675
2131
  pid INTEGER NOT NULL,
@@ -1756,6 +2212,7 @@ class Store {
1756
2212
  subject_ref TEXT NOT NULL,
1757
2213
  project_key TEXT,
1758
2214
  project_group TEXT,
2215
+ machine_id TEXT,
1759
2216
  route_scope TEXT,
1760
2217
  priority INTEGER NOT NULL,
1761
2218
  status TEXT NOT NULL,
@@ -1774,10 +2231,10 @@ class Store {
1774
2231
  CREATE INDEX IF NOT EXISTS idx_workflow_work_items_project ON workflow_work_items(project_key, status);
1775
2232
  CREATE INDEX IF NOT EXISTS idx_workflow_work_items_group ON workflow_work_items(project_group, status);
1776
2233
  CREATE INDEX IF NOT EXISTS idx_workflow_work_items_invocation ON workflow_work_items(invocation_id);
1777
- -- idx_workflow_work_items_scope (route_scope, status) is created ONLY by
1778
- -- migration 0008_work_item_route_scope, never here: this baseline DDL
2234
+ -- New-column indexes (route_scope, machine_id, etc.) are created ONLY by
2235
+ -- their additive migrations, never here: this baseline DDL
1779
2236
  -- re-runs on EVERY open (0001 is not skip-guarded), and on a pre-0008
1780
- -- database the CREATE TABLE above is a no-op, so an index on route_scope
2237
+ -- database the CREATE TABLE above is a no-op, so an index on a new column
1781
2238
  -- here would execute before the column exists and crash the open
1782
2239
  -- ("no such column: route_scope"). New columns may be folded into the
1783
2240
  -- CREATE TABLE (fresh-db only); their indexes must live in the migration.
@@ -1888,10 +2345,35 @@ class Store {
1888
2345
  CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
1889
2346
  `);
1890
2347
  }
1891
- addColumnIfMissing(table, column, definition) {
1892
- const columns = this.db.query(`PRAGMA table_info(${table})`).all();
1893
- if (columns.some((c) => c.name === column))
1894
- return;
2348
+ createRunReceiptsSchema() {
2349
+ this.db.exec(`
2350
+ CREATE TABLE IF NOT EXISTS run_receipts (
2351
+ run_id TEXT PRIMARY KEY,
2352
+ loop_id TEXT NOT NULL,
2353
+ machine_json TEXT NOT NULL,
2354
+ repo TEXT NOT NULL,
2355
+ task_ids_json TEXT NOT NULL,
2356
+ knowledge_ids_json TEXT NOT NULL,
2357
+ digest_id TEXT NOT NULL,
2358
+ started_at TEXT,
2359
+ finished_at TEXT,
2360
+ status TEXT NOT NULL,
2361
+ exit_code INTEGER,
2362
+ summary_json TEXT NOT NULL,
2363
+ evidence_paths_json TEXT NOT NULL,
2364
+ created_at TEXT NOT NULL,
2365
+ updated_at TEXT NOT NULL
2366
+ );
2367
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_loop ON run_receipts(loop_id, created_at DESC);
2368
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_repo ON run_receipts(repo, created_at DESC);
2369
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_digest ON run_receipts(digest_id);
2370
+ CREATE INDEX IF NOT EXISTS idx_run_receipts_status ON run_receipts(status, created_at DESC);
2371
+ `);
2372
+ }
2373
+ addColumnIfMissing(table, column, definition) {
2374
+ const columns = this.db.query(`PRAGMA table_info(${table})`).all();
2375
+ if (columns.some((c) => c.name === column))
2376
+ return;
1895
2377
  this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
1896
2378
  }
1897
2379
  createWorkflowRunBackfillIndexes() {
@@ -1999,21 +2481,51 @@ class Store {
1999
2481
  }
2000
2482
  listLoops(opts = {}) {
2001
2483
  const limit = opts.limit ?? 200;
2484
+ const offset = Math.max(0, Math.floor(opts.offset ?? 0));
2485
+ if (opts.name != null) {
2486
+ const rows2 = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.name, limit, offset);
2487
+ return this.withLatestRunSummaries(rows2.map(rowToLoop));
2488
+ }
2002
2489
  let rows;
2003
2490
  if (opts.status && opts.archived) {
2004
- rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NOT NULL ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
2491
+ rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NOT NULL ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
2005
2492
  } else if (opts.status && opts.includeArchived) {
2006
- rows = this.db.query("SELECT * FROM loops WHERE status = ? ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
2493
+ rows = this.db.query("SELECT * FROM loops WHERE status = ? ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
2007
2494
  } else if (opts.status) {
2008
- rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NULL ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
2495
+ rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NULL ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
2009
2496
  } else if (opts.archived) {
2010
- rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NOT NULL ORDER BY archived_at DESC LIMIT ?").all(limit);
2497
+ rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NOT NULL ORDER BY archived_at DESC LIMIT ? OFFSET ?").all(limit, offset);
2011
2498
  } else if (opts.includeArchived) {
2012
- rows = this.db.query("SELECT * FROM loops ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
2499
+ rows = this.db.query("SELECT * FROM loops ORDER BY status ASC, next_run_at ASC LIMIT ? OFFSET ?").all(limit, offset);
2013
2500
  } else {
2014
- rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
2015
- }
2016
- return rows.map(rowToLoop);
2501
+ rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ? OFFSET ?").all(limit, offset);
2502
+ }
2503
+ return this.withLatestRunSummaries(rows.map(rowToLoop));
2504
+ }
2505
+ withLatestRunSummaries(loops) {
2506
+ if (loops.length === 0)
2507
+ return loops;
2508
+ const placeholders = loops.map(() => "?").join(",");
2509
+ const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
2510
+ FROM (
2511
+ SELECT loop_id, id, status, started_at, finished_at, created_at,
2512
+ ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
2513
+ FROM loop_runs
2514
+ WHERE loop_id IN (${placeholders})
2515
+ )
2516
+ WHERE rn = 1`).all(...loops.map((loop) => loop.id));
2517
+ const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
2518
+ return loops.map((loop) => {
2519
+ const latest = latestByLoopId.get(loop.id);
2520
+ if (!latest)
2521
+ return loop;
2522
+ return {
2523
+ ...loop,
2524
+ latestRunId: latest.id,
2525
+ latestRunStatus: latest.status,
2526
+ lastRunAt: latestRunTime(latest)
2527
+ };
2528
+ });
2017
2529
  }
2018
2530
  dueLoops(now, limit = 500) {
2019
2531
  const rows = this.db.query(`SELECT * FROM loops
@@ -2132,6 +2644,45 @@ class Store {
2132
2644
  throw error;
2133
2645
  }
2134
2646
  }
2647
+ updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
2648
+ const updated = (opts.now ?? new Date).toISOString();
2649
+ this.db.exec("BEGIN IMMEDIATE");
2650
+ try {
2651
+ const current = this.requireUniqueLoop(idOrName);
2652
+ if (current.archivedAt)
2653
+ throw new LoopArchivedError(current.name || current.id);
2654
+ if (current.target.type !== "agent")
2655
+ throw new Error(`loop is not an agent loop: ${idOrName}`);
2656
+ if (this.hasRunningRun(current.id))
2657
+ throw new Error(`refusing to update running loop: ${current.id}`);
2658
+ const target = { ...current.target, timeoutMs };
2659
+ if (timeoutMs === null && target.idleTimeoutMs !== undefined)
2660
+ delete target.idleTimeoutMs;
2661
+ const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
2662
+ WHERE id=$id
2663
+ AND ($daemonLeaseId IS NULL OR EXISTS (
2664
+ SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
2665
+ ))`).run({
2666
+ $id: current.id,
2667
+ $target: JSON.stringify(target),
2668
+ $updated: updated,
2669
+ $daemonLeaseId: opts.daemonLeaseId ?? null,
2670
+ $now: updated
2671
+ });
2672
+ if (res.changes !== 1)
2673
+ throw new Error("daemon lease lost");
2674
+ this.db.exec("COMMIT");
2675
+ const after = this.getLoop(current.id);
2676
+ if (!after)
2677
+ throw new Error(`loop not found after timeout update: ${current.id}`);
2678
+ return after;
2679
+ } catch (error) {
2680
+ try {
2681
+ this.db.exec("ROLLBACK");
2682
+ } catch {}
2683
+ throw error;
2684
+ }
2685
+ }
2135
2686
  createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
2136
2687
  const normalized = normalizeCreateWorkflowInput(workflowInput);
2137
2688
  const updated = (opts.now ?? new Date).toISOString();
@@ -2462,6 +3013,59 @@ class Store {
2462
3013
  updated
2463
3014
  });
2464
3015
  }
3016
+ taskLifecycleTodosPointerContext(workflowRunId) {
3017
+ const run = this.getWorkflowRun(workflowRunId);
3018
+ if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
3019
+ return;
3020
+ const workItem = this.getWorkflowWorkItem(run.workItemId);
3021
+ if (!workItem || workItem.routeKey !== "todos-task")
3022
+ return;
3023
+ const invocation = this.getWorkflowInvocation(run.invocationId);
3024
+ if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
3025
+ return;
3026
+ const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
3027
+ const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
3028
+ if (!projectPath || !taskId)
3029
+ return;
3030
+ return {
3031
+ projectPath,
3032
+ taskId,
3033
+ invocationId: invocation.id,
3034
+ workflowRunId: run.id,
3035
+ manifestPath: run.manifestPath
3036
+ };
3037
+ }
3038
+ syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
3039
+ const context = this.taskLifecycleTodosPointerContext(workflowRunId);
3040
+ if (!context)
3041
+ return;
3042
+ const result = runLocalCommand("todos", [
3043
+ "--project",
3044
+ context.projectPath,
3045
+ "task",
3046
+ "workflow-pointers",
3047
+ context.taskId,
3048
+ "--clear",
3049
+ "--invocation",
3050
+ context.invocationId,
3051
+ "--run",
3052
+ context.workflowRunId,
3053
+ "--manifest",
3054
+ context.manifestPath,
3055
+ "--state",
3056
+ "succeeded",
3057
+ "--actor",
3058
+ "openloops:task-lifecycle"
3059
+ ]);
3060
+ this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
3061
+ projectPath: context.projectPath,
3062
+ taskId: context.taskId,
3063
+ invocationId: context.invocationId,
3064
+ workflowRunId: context.workflowRunId,
3065
+ manifestPath: context.manifestPath,
3066
+ mutation: todosMutationSummary(result)
3067
+ });
3068
+ }
2465
3069
  createWorkflowInvocation(input) {
2466
3070
  const now = nowIso();
2467
3071
  const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
@@ -2572,10 +3176,10 @@ class Store {
2572
3176
  const id = genId();
2573
3177
  const status = input.status ?? "queued";
2574
3178
  this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
2575
- subject_ref, project_key, project_group, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
3179
+ subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
2576
3180
  workflow_id, loop_id, workflow_run_id, last_reason, created_at, updated_at)
2577
3181
  VALUES ($id, $routeKey, $idempotencyKey, $invocationId, $sourceType, $sourceRef, $subjectRef,
2578
- $projectKey, $projectGroup, $routeScope, $priority, $status, 0, $nextAttemptAt, NULL, NULL, NULL, NULL,
3182
+ $projectKey, $projectGroup, $machineId, $routeScope, $priority, $status, 0, $nextAttemptAt, NULL, NULL, NULL, NULL,
2579
3183
  $lastReason, $created, $updated)
2580
3184
  ON CONFLICT(route_key, idempotency_key) DO UPDATE SET
2581
3185
  invocation_id=excluded.invocation_id,
@@ -2584,6 +3188,10 @@ class Store {
2584
3188
  subject_ref=excluded.subject_ref,
2585
3189
  project_key=excluded.project_key,
2586
3190
  project_group=excluded.project_group,
3191
+ machine_id=CASE
3192
+ WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.machine_id
3193
+ ELSE excluded.machine_id
3194
+ END,
2587
3195
  route_scope=excluded.route_scope,
2588
3196
  priority=excluded.priority,
2589
3197
  status=CASE
@@ -2626,6 +3234,7 @@ class Store {
2626
3234
  $subjectRef: input.subjectRef,
2627
3235
  $projectKey: input.projectKey ?? null,
2628
3236
  $projectGroup: input.projectGroup ?? null,
3237
+ $machineId: input.machineId ?? null,
2629
3238
  $routeScope: input.routeScope ?? null,
2630
3239
  $priority: input.priority ?? 0,
2631
3240
  $status: status,
@@ -3135,7 +3744,7 @@ class Store {
3135
3744
  VALUES ($id, $workflowRunId, 1, 'created', NULL, $payload, $created)`).run({
3136
3745
  $id: genId(),
3137
3746
  $workflowRunId: runId,
3138
- $payload: JSON.stringify({
3747
+ $payload: persistedWorkflowEventPayload({
3139
3748
  workflowId: input.workflow.id,
3140
3749
  workflowName: input.workflow.name,
3141
3750
  stepCount: input.workflow.steps.length,
@@ -3424,6 +4033,8 @@ class Store {
3424
4033
  const run = this.getWorkflowRun(workflowRunId);
3425
4034
  if (!run)
3426
4035
  throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
4036
+ if (changed && status === "succeeded")
4037
+ this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
3427
4038
  return run;
3428
4039
  }
3429
4040
  cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
@@ -3464,7 +4075,7 @@ class Store {
3464
4075
  $sequence: sequence,
3465
4076
  $eventType: eventType,
3466
4077
  $stepId: stepId ?? null,
3467
- $payload: payload ? JSON.stringify(payload) : null,
4078
+ $payload: persistedWorkflowEventPayload(payload),
3468
4079
  $created: now
3469
4080
  });
3470
4081
  const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
@@ -3485,6 +4096,17 @@ class Store {
3485
4096
  const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
3486
4097
  return (row?.count ?? 0) > 0;
3487
4098
  }
4099
+ hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
4100
+ const rows = this.db.query(`SELECT * FROM loop_runs
4101
+ WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
4102
+ return rows.some((row) => {
4103
+ if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
4104
+ return true;
4105
+ if (isRecordedProcessAlive(row.pid, row.process_started_at))
4106
+ return true;
4107
+ return this.hasLiveWorkflowStepProcesses(row.id);
4108
+ });
4109
+ }
3488
4110
  markRunPid(id, pid, claimedBy, opts = {}) {
3489
4111
  const now = (opts.now ?? new Date).toISOString();
3490
4112
  const startedMs = processStartTimeMs(pid);
@@ -3617,6 +4239,10 @@ class Store {
3617
4239
  loop = currentLoop;
3618
4240
  const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
3619
4241
  const existing = this.getRunBySlot(loop.id, scheduledFor);
4242
+ if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
4243
+ this.db.exec("COMMIT");
4244
+ return;
4245
+ }
3620
4246
  if (existing) {
3621
4247
  if (existing.status === "running") {
3622
4248
  if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
@@ -3772,18 +4398,93 @@ class Store {
3772
4398
  }
3773
4399
  listRuns(opts = {}) {
3774
4400
  const limit = opts.limit ?? 100;
4401
+ const offset = Math.max(0, Math.floor(opts.offset ?? 0));
3775
4402
  let rows;
3776
4403
  if (opts.loopId && opts.status) {
3777
- rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND status = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopId, opts.status, limit);
4404
+ rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.loopId, opts.status, limit, offset);
3778
4405
  } else if (opts.loopId) {
3779
- rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopId, limit);
4406
+ rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.loopId, limit, offset);
3780
4407
  } else if (opts.status) {
3781
- rows = this.db.query("SELECT * FROM loop_runs WHERE status = ? ORDER BY created_at DESC LIMIT ?").all(opts.status, limit);
4408
+ rows = this.db.query("SELECT * FROM loop_runs WHERE status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
3782
4409
  } else {
3783
- rows = this.db.query("SELECT * FROM loop_runs ORDER BY created_at DESC LIMIT ?").all(limit);
4410
+ rows = this.db.query("SELECT * FROM loop_runs ORDER BY created_at DESC LIMIT ? OFFSET ?").all(limit, offset);
3784
4411
  }
3785
4412
  return rows.map(rowToRun);
3786
4413
  }
4414
+ writeRunReceipt(input, opts = {}) {
4415
+ const inputRunId = typeof input.run_id === "string" && input.run_id.trim() ? input.run_id : undefined;
4416
+ const existing = inputRunId ? this.getRunReceipt(inputRunId) : undefined;
4417
+ const run = inputRunId ? this.getRun(inputRunId) : undefined;
4418
+ const loop = input.loop_id ? this.getLoop(input.loop_id) : run ? this.getLoop(run.loopId) : undefined;
4419
+ const receipt = normalizeRunReceipt(input, { now: opts.now, run, loop, existing });
4420
+ this.db.query(`INSERT INTO run_receipts (run_id, loop_id, machine_json, repo, task_ids_json, knowledge_ids_json, digest_id,
4421
+ started_at, finished_at, status, exit_code, summary_json, evidence_paths_json, created_at, updated_at)
4422
+ VALUES ($runId, $loopId, $machineJson, $repo, $taskIdsJson, $knowledgeIdsJson, $digestId,
4423
+ $startedAt, $finishedAt, $status, $exitCode, $summaryJson, $evidencePathsJson, $createdAt, $updatedAt)
4424
+ ON CONFLICT(run_id) DO UPDATE SET
4425
+ loop_id=excluded.loop_id,
4426
+ machine_json=excluded.machine_json,
4427
+ repo=excluded.repo,
4428
+ task_ids_json=excluded.task_ids_json,
4429
+ knowledge_ids_json=excluded.knowledge_ids_json,
4430
+ digest_id=excluded.digest_id,
4431
+ started_at=excluded.started_at,
4432
+ finished_at=excluded.finished_at,
4433
+ status=excluded.status,
4434
+ exit_code=excluded.exit_code,
4435
+ summary_json=excluded.summary_json,
4436
+ evidence_paths_json=excluded.evidence_paths_json,
4437
+ updated_at=excluded.updated_at`).run({
4438
+ $runId: receipt.run_id,
4439
+ $loopId: receipt.loop_id,
4440
+ $machineJson: JSON.stringify(receipt.machine),
4441
+ $repo: receipt.repo,
4442
+ $taskIdsJson: JSON.stringify(receipt.task_ids),
4443
+ $knowledgeIdsJson: JSON.stringify(receipt.knowledge_ids),
4444
+ $digestId: receipt.digest_id,
4445
+ $startedAt: receipt.started_at,
4446
+ $finishedAt: receipt.finished_at,
4447
+ $status: receipt.status,
4448
+ $exitCode: receipt.exit_code,
4449
+ $summaryJson: JSON.stringify(receipt.summary),
4450
+ $evidencePathsJson: JSON.stringify(receipt.evidence_paths),
4451
+ $createdAt: receipt.created_at,
4452
+ $updatedAt: receipt.updated_at
4453
+ });
4454
+ return this.getRunReceipt(receipt.run_id) ?? receipt;
4455
+ }
4456
+ getRunReceipt(runId) {
4457
+ const row = this.db.query("SELECT * FROM run_receipts WHERE run_id = ?").get(runId);
4458
+ return row ? rowToRunReceipt(row) : undefined;
4459
+ }
4460
+ listRunReceipts(opts = {}) {
4461
+ const limit = opts.limit ?? 100;
4462
+ const filters = [];
4463
+ const params = [];
4464
+ if (opts.loopId) {
4465
+ filters.push("loop_id = ?");
4466
+ params.push(opts.loopId);
4467
+ }
4468
+ if (opts.repo) {
4469
+ filters.push("repo = ?");
4470
+ params.push(opts.repo);
4471
+ }
4472
+ if (opts.status) {
4473
+ filters.push("status = ?");
4474
+ params.push(opts.status);
4475
+ }
4476
+ if (opts.taskId) {
4477
+ filters.push("EXISTS (SELECT 1 FROM json_each(run_receipts.task_ids_json) WHERE value = ?)");
4478
+ params.push(opts.taskId);
4479
+ }
4480
+ if (opts.knowledgeId) {
4481
+ filters.push("EXISTS (SELECT 1 FROM json_each(run_receipts.knowledge_ids_json) WHERE value = ?)");
4482
+ params.push(opts.knowledgeId);
4483
+ }
4484
+ const where = filters.length ? `WHERE ${filters.join(" AND ")}` : "";
4485
+ const rows = this.db.query(`SELECT * FROM run_receipts ${where} ORDER BY created_at DESC LIMIT ?`).all(...params, limit);
4486
+ return rows.map(rowToRunReceipt);
4487
+ }
3787
4488
  deferLiveExpiredRun(id, now, opts = {}) {
3788
4489
  const updated = now.toISOString();
3789
4490
  const deferredUntil = new Date(now.getTime() + LIVE_EXPIRED_RUN_GRACE_MS).toISOString();
@@ -3942,6 +4643,9 @@ class Store {
3942
4643
  const runs = includeRuns ? this.db.query("SELECT * FROM loop_runs ORDER BY created_at ASC, id ASC").all().map(rowToRun) : [];
3943
4644
  return { schemaVersion: SCHEMA_USER_VERSION, workflows, loops, runs, checks: this.migrationChecks() };
3944
4645
  }
4646
+ exportMigrationRunPage(opts) {
4647
+ return this.db.query("SELECT * FROM loop_runs ORDER BY created_at ASC, id ASC LIMIT ? OFFSET ?").all(opts.limit, opts.offset).map(rowToRun);
4648
+ }
3945
4649
  countTable(table) {
3946
4650
  const row = this.db.query(`SELECT COUNT(*) AS count FROM ${table}`).get();
3947
4651
  return row?.count ?? 0;
@@ -4188,9 +4892,9 @@ class Store {
4188
4892
  const runDir = dirname2(manifestPath);
4189
4893
  try {
4190
4894
  if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
4191
- rmSync2(runDir, { recursive: true, force: true });
4895
+ rmSync3(runDir, { recursive: true, force: true });
4192
4896
  } else {
4193
- rmSync2(manifestPath, { force: true });
4897
+ rmSync3(manifestPath, { force: true });
4194
4898
  }
4195
4899
  } catch {}
4196
4900
  }
@@ -4257,7 +4961,7 @@ class Store {
4257
4961
  this.db.close();
4258
4962
  if (this.memoryRootDir) {
4259
4963
  try {
4260
- rmSync2(this.memoryRootDir, { recursive: true, force: true });
4964
+ rmSync3(this.memoryRootDir, { recursive: true, force: true });
4261
4965
  } catch {}
4262
4966
  this.memoryRootDir = undefined;
4263
4967
  }
@@ -4266,7 +4970,7 @@ class Store {
4266
4970
  // package.json
4267
4971
  var package_default = {
4268
4972
  name: "@hasna/loops",
4269
- version: "0.4.13",
4973
+ version: "0.4.22",
4270
4974
  description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
4271
4975
  type: "module",
4272
4976
  main: "dist/index.js",
@@ -4275,6 +4979,7 @@ var package_default = {
4275
4979
  loops: "dist/cli/index.js",
4276
4980
  "loops-daemon": "dist/daemon/index.js",
4277
4981
  "loops-api": "dist/api/index.js",
4982
+ "loops-serve": "dist/serve/index.js",
4278
4983
  "loops-runner": "dist/runner/index.js",
4279
4984
  "loops-mcp": "dist/mcp/index.js"
4280
4985
  },
@@ -4287,6 +4992,14 @@ var package_default = {
4287
4992
  types: "./dist/sdk/index.d.ts",
4288
4993
  import: "./dist/sdk/index.js"
4289
4994
  },
4995
+ "./sdk/http": {
4996
+ types: "./dist/sdk/http.d.ts",
4997
+ import: "./dist/sdk/http.js"
4998
+ },
4999
+ "./serve": {
5000
+ types: "./dist/serve/index.d.ts",
5001
+ import: "./dist/serve/index.js"
5002
+ },
4290
5003
  "./mcp": {
4291
5004
  types: "./dist/mcp/index.d.ts",
4292
5005
  import: "./dist/mcp/index.js"
@@ -4332,7 +5045,7 @@ var package_default = {
4332
5045
  "LICENSE"
4333
5046
  ],
4334
5047
  scripts: {
4335
- build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
5048
+ build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
4336
5049
  "build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
4337
5050
  typecheck: "tsc --noEmit",
4338
5051
  test: "bun test",
@@ -4372,13 +5085,16 @@ var package_default = {
4372
5085
  "@openrouter/ai-sdk-provider": "2.9.1",
4373
5086
  ai: "6.0.204",
4374
5087
  commander: "^13.1.0",
5088
+ pg: "^8.13.1",
4375
5089
  zod: "4.4.3"
4376
5090
  },
4377
5091
  optionalDependencies: {
4378
- "@hasna/machines": "0.0.49"
5092
+ "@hasna/machines": "0.0.49",
5093
+ "@hasna/contracts": "^0.4.2"
4379
5094
  },
4380
5095
  devDependencies: {
4381
5096
  "@types/bun": "latest",
5097
+ "@types/pg": "^8.11.10",
4382
5098
  typescript: "^5.7.3"
4383
5099
  },
4384
5100
  publishConfig: {
@@ -4586,15 +5302,11 @@ function deploymentStatusLine(status) {
4586
5302
  ].join(" ");
4587
5303
  }
4588
5304
 
4589
- // src/lib/doctor.ts
4590
- import { spawnSync as spawnSync5 } from "child_process";
4591
- import { accessSync as accessSync2, constants as constants2 } from "fs";
4592
-
4593
5305
  // src/daemon/control.ts
4594
- import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync3, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
4595
- import { spawnSync as spawnSync2 } from "child_process";
4596
- import { hostname } from "os";
4597
- import { dirname as dirname3, join as join4 } from "path";
5306
+ import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
5307
+ import { spawnSync as spawnSync3 } from "child_process";
5308
+ import { hostname as hostname2 } from "os";
5309
+ import { dirname as dirname3, join as join5 } from "path";
4598
5310
 
4599
5311
  // src/daemon/loop.ts
4600
5312
  function realSleep(ms) {
@@ -4624,7 +5336,7 @@ function readPidRecord(path = pidFilePath()) {
4624
5336
  return;
4625
5337
  let raw;
4626
5338
  try {
4627
- raw = readFileSync3(path, "utf8").trim();
5339
+ raw = readFileSync4(path, "utf8").trim();
4628
5340
  } catch {
4629
5341
  return;
4630
5342
  }
@@ -4650,7 +5362,7 @@ function writePid(pid = process.pid, path = pidFilePath()) {
4650
5362
  writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
4651
5363
  }
4652
5364
  function removePid(path = pidFilePath()) {
4653
- rmSync3(path, { force: true });
5365
+ rmSync4(path, { force: true });
4654
5366
  }
4655
5367
  function isAlive(pid, startedAt) {
4656
5368
  try {
@@ -4680,7 +5392,7 @@ function ownProcessGroupId() {
4680
5392
  return pgrp;
4681
5393
  }
4682
5394
  try {
4683
- const run = spawnSync2("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
5395
+ const run = spawnSync3("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
4684
5396
  const pgid = Number(run.stdout.trim());
4685
5397
  if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
4686
5398
  return pgid;
@@ -4746,7 +5458,7 @@ function daemonStatus(store, path = pidFilePath()) {
4746
5458
  return {
4747
5459
  ...isDaemonRunning(path),
4748
5460
  lease: store.getDaemonLease(),
4749
- host: hostname(),
5461
+ host: hostname2(),
4750
5462
  loops: {
4751
5463
  total: store.countLoops(),
4752
5464
  active: store.countLoops("active"),
@@ -4776,7 +5488,7 @@ async function stopDaemon(opts = {}) {
4776
5488
  if (!state.running || !state.pid)
4777
5489
  return { wasRunning: false, stopped: false, forced: false };
4778
5490
  const sleep = opts.sleep ?? realSleep;
4779
- const store = new Store(join4(dirname3(path), "loops.db"));
5491
+ const store = new Store(join5(dirname3(path), "loops.db"));
4780
5492
  try {
4781
5493
  const lease = store.getDaemonLease();
4782
5494
  const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
@@ -4813,15 +5525,19 @@ async function stopDaemon(opts = {}) {
4813
5525
  }
4814
5526
  }
4815
5527
 
5528
+ // src/lib/doctor.ts
5529
+ import { spawnSync as spawnSync6 } from "child_process";
5530
+ import { accessSync as accessSync2, constants as constants2 } from "fs";
5531
+
4816
5532
  // src/lib/executor.ts
4817
- import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
5533
+ import { spawn as spawn2, spawnSync as spawnSync5 } from "child_process";
4818
5534
  import { randomBytes as randomBytes2 } from "crypto";
4819
5535
  import { once } from "events";
4820
5536
  import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
4821
5537
  import { dirname as dirname4, resolve as resolve2 } from "path";
4822
5538
 
4823
5539
  // src/lib/accounts.ts
4824
- import { spawnSync as spawnSync3 } from "child_process";
5540
+ import { spawnSync as spawnSync4 } from "child_process";
4825
5541
  import { existsSync as existsSync3 } from "fs";
4826
5542
  var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
4827
5543
  var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
@@ -4926,7 +5642,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
4926
5642
  if (!account)
4927
5643
  return {};
4928
5644
  const tool = requiredAccountTool(account, toolHint);
4929
- const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
5645
+ const result = spawnSync4("accounts", ["env", account.profile, "--tool", tool], {
4930
5646
  encoding: "utf8",
4931
5647
  env,
4932
5648
  stdio: ["ignore", "pipe", "pipe"],
@@ -4942,8 +5658,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
4942
5658
 
4943
5659
  // src/lib/env.ts
4944
5660
  import { accessSync, constants } from "fs";
4945
- import { homedir as homedir2 } from "os";
4946
- import { delimiter, join as join5 } from "path";
5661
+ import { homedir as homedir3 } from "os";
5662
+ import { delimiter, join as join6 } from "path";
4947
5663
  function compactPathParts(parts) {
4948
5664
  const seen = new Set;
4949
5665
  const result = [];
@@ -4957,16 +5673,16 @@ function compactPathParts(parts) {
4957
5673
  return result;
4958
5674
  }
4959
5675
  function commonExecutableDirs(env = process.env) {
4960
- const home = env.HOME || homedir2();
5676
+ const home = env.HOME || homedir3();
4961
5677
  return compactPathParts([
4962
- join5(home, ".local", "bin"),
4963
- join5(home, ".bun", "bin"),
4964
- join5(home, ".cargo", "bin"),
4965
- join5(home, ".npm-global", "bin"),
4966
- join5(home, "bin"),
4967
- env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
5678
+ join6(home, ".local", "bin"),
5679
+ join6(home, ".bun", "bin"),
5680
+ join6(home, ".cargo", "bin"),
5681
+ join6(home, ".npm-global", "bin"),
5682
+ join6(home, "bin"),
5683
+ env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
4968
5684
  env.PNPM_HOME,
4969
- env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
5685
+ env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
4970
5686
  "/opt/homebrew/bin",
4971
5687
  "/usr/local/bin",
4972
5688
  "/usr/bin",
@@ -4990,7 +5706,7 @@ function executableExists(command, env = process.env) {
4990
5706
  if (command.includes("/"))
4991
5707
  return isExecutable(command);
4992
5708
  for (const dir of (env.PATH ?? "").split(delimiter)) {
4993
- if (dir && isExecutable(join5(dir, command)))
5709
+ if (dir && isExecutable(join6(dir, command)))
4994
5710
  return true;
4995
5711
  }
4996
5712
  return false;
@@ -5099,6 +5815,7 @@ var TRANSPORT_ENV_KEYS = new Set([
5099
5815
  "LOGNAME",
5100
5816
  "PATH",
5101
5817
  "SHELL",
5818
+ "SHLVL",
5102
5819
  "SSH_AGENT_PID",
5103
5820
  "SSH_AUTH_SOCK",
5104
5821
  "TERM",
@@ -5131,6 +5848,32 @@ function timeoutResult(startedAt, error, fields = {}) {
5131
5848
  function successResult(startedAt, fields = {}) {
5132
5849
  return buildResult("succeeded", startedAt, fields);
5133
5850
  }
5851
+ function codewithJsonlHasTerminalSuccess(stdout) {
5852
+ for (const line of stdout.split(/\r?\n/)) {
5853
+ const trimmed = line.trim();
5854
+ if (!trimmed || !trimmed.startsWith("{"))
5855
+ continue;
5856
+ let event;
5857
+ try {
5858
+ event = JSON.parse(trimmed);
5859
+ } catch {
5860
+ continue;
5861
+ }
5862
+ if (!event || typeof event !== "object")
5863
+ continue;
5864
+ const record = event;
5865
+ if (record.type === "task_complete")
5866
+ return true;
5867
+ const payload = record.payload;
5868
+ if (payload && typeof payload === "object" && payload.type === "task_complete") {
5869
+ return true;
5870
+ }
5871
+ }
5872
+ return false;
5873
+ }
5874
+ function codewithJsonlReconciledSuccess(spec, fields) {
5875
+ return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
5876
+ }
5134
5877
  function notifySpawn(pid, opts) {
5135
5878
  if (!pid)
5136
5879
  return;
@@ -5146,10 +5889,48 @@ function shellQuote(value) {
5146
5889
  return `'${value.replace(/'/g, `'\\''`)}'`;
5147
5890
  }
5148
5891
  function codewithProfileCandidateFromLine(line) {
5149
- const cols = line.trim().split(/\s+/);
5150
- if (cols[0] === "*")
5151
- return cols[1];
5152
- return cols[0];
5892
+ const trimmed = line.trim();
5893
+ if (!trimmed || trimmed === "No auth profiles saved.")
5894
+ return;
5895
+ const cols = trimmed.split(/\s+/);
5896
+ const candidate = cols[0] === "*" ? cols[1] : cols[0];
5897
+ if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
5898
+ return;
5899
+ if (candidate === '"name":') {
5900
+ const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
5901
+ if (jsonName)
5902
+ return jsonName;
5903
+ }
5904
+ return candidate;
5905
+ }
5906
+ function codewithProfileCandidatesFromJson(value) {
5907
+ const candidates = [];
5908
+ const root = value && typeof value === "object" ? value : undefined;
5909
+ const addProfile = (entry) => {
5910
+ if (!entry || typeof entry !== "object")
5911
+ return;
5912
+ const name = entry.name;
5913
+ if (typeof name === "string" && name)
5914
+ candidates.push(name);
5915
+ };
5916
+ const data = root?.data;
5917
+ if (Array.isArray(data))
5918
+ data.forEach(addProfile);
5919
+ const profiles = root?.profiles;
5920
+ if (Array.isArray(profiles))
5921
+ profiles.forEach(addProfile);
5922
+ return candidates;
5923
+ }
5924
+ function codewithProfileCandidatesFromOutput(output) {
5925
+ const trimmed = output.trim();
5926
+ const jsonStart = trimmed.indexOf("{");
5927
+ const jsonEnd = trimmed.lastIndexOf("}");
5928
+ if (jsonStart >= 0 && jsonEnd >= jsonStart) {
5929
+ try {
5930
+ return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
5931
+ } catch {}
5932
+ }
5933
+ return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
5153
5934
  }
5154
5935
  function codewithProfileForError(profile) {
5155
5936
  return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
@@ -5242,6 +6023,7 @@ function commandSpec(target, opts) {
5242
6023
  preflightAnyOf: invocation.preflightAnyOf,
5243
6024
  stdin: invocation.stdin,
5244
6025
  allowlist: agentTarget.allowlist,
6026
+ agentProvider: agentTarget.provider,
5245
6027
  worktree: agentTarget.worktree
5246
6028
  };
5247
6029
  }
@@ -5255,6 +6037,7 @@ function composeExecutionEnv(spec, metadata, opts, accountEnv) {
5255
6037
  Object.assign(env, spec.env ?? {});
5256
6038
  Object.assign(env, allowlistEnv(spec.allowlist));
5257
6039
  env.PATH = normalizeExecutionPath(env);
6040
+ env.SHLVL ||= "1";
5258
6041
  Object.assign(env, metadataEnv(metadata));
5259
6042
  return env;
5260
6043
  }
@@ -5326,7 +6109,7 @@ function remoteWorktreePrepareLines(worktree) {
5326
6109
  return [
5327
6110
  "__openloops_prepare_worktree() {",
5328
6111
  ` local repo=${shellQuote(repoRoot)} path=${shellQuote(path)} branch=${shellQuote(branch)}`,
5329
- " local top expected_common actual_common current",
6112
+ " local top expected_common actual_common current status recovered",
5330
6113
  ' if [ -L "$path" ]; then echo "refusing symlinked worktree path $path" >&2; return 1; fi',
5331
6114
  ' if [ -e "$path" ]; then',
5332
6115
  ' top="$(git -C "$path" rev-parse --show-toplevel 2>/dev/null)" || { echo "refusing to reuse non-worktree path: $path" >&2; return 1; }',
@@ -5335,7 +6118,19 @@ function remoteWorktreePrepareLines(worktree) {
5335
6118
  ' actual_common="$(cd "$path" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
5336
6119
  ' if [ -z "$expected_common" ] || [ "$expected_common" != "$actual_common" ]; then echo "existing worktree $path belongs to a different git common dir" >&2; return 1; fi',
5337
6120
  ' current="$(git -C "$path" branch --show-current 2>/dev/null)"',
5338
- ' if [ "$current" != "$branch" ]; then echo "existing worktree $path is on branch ${current:-unknown}, expected $branch" >&2; return 1; fi',
6121
+ ' if [ "$current" != "$branch" ]; then',
6122
+ ' status="$(git -C "$path" status --porcelain=v1 --untracked-files=all 2>/dev/null)" || { echo "existing worktree $path is on branch ${current:-unknown}, expected $branch, and cleanliness could not be checked; inspect or remove the stale worktree" >&2; return 1; }',
6123
+ ' if [ -n "$status" ]; then echo "existing worktree $path is on branch ${current:-unknown}, expected $branch, and has local changes; commit/stash them or remove the stale worktree before retrying" >&2; return 1; fi',
6124
+ ' if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
6125
+ ' git -C "$path" checkout "$branch" 1>&2 || { echo "existing worktree $path is clean but could not switch from branch ${current:-unknown} to expected $branch; remove/prune the stale worktree or free the branch before retrying" >&2; return 1; }',
6126
+ ' elif [ -z "$current" ]; then',
6127
+ ' git -C "$path" checkout -b "$branch" 1>&2 || { echo "existing worktree $path is clean but could not recreate expected branch $branch at the current detached HEAD; remove/prune the stale worktree before retrying" >&2; return 1; }',
6128
+ " else",
6129
+ ' echo "existing worktree $path is on branch $current, expected $branch, but expected branch does not exist; remove/prune the stale worktree or recreate the expected branch before retrying" >&2; return 1',
6130
+ " fi",
6131
+ ' recovered="$(git -C "$path" branch --show-current 2>/dev/null)"',
6132
+ ' if [ "$recovered" != "$branch" ]; then echo "existing worktree $path branch recovery ended on ${recovered:-unknown}, expected $branch; remove/prune the stale worktree before retrying" >&2; return 1; fi',
6133
+ " fi",
5339
6134
  " return 0",
5340
6135
  " fi",
5341
6136
  ' git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
@@ -5401,7 +6196,7 @@ function remotePreflightScript(spec, metadata) {
5401
6196
  }
5402
6197
  if (spec.nativeAuthProfile?.provider === "codewith") {
5403
6198
  const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
5404
- lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", `__OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", "}", `if ! printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk 'NR > 1 { candidate = ($1 == "*" ? $2 : $1); if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'; then`, ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", "fi");
6199
+ lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
5405
6200
  }
5406
6201
  return lines.join(`
5407
6202
  `);
@@ -5416,9 +6211,28 @@ function transportEnv(opts) {
5416
6211
  env[key] = value;
5417
6212
  }
5418
6213
  env.PATH = normalizeExecutionPath(env);
6214
+ env.SHLVL ||= "1";
5419
6215
  return env;
5420
6216
  }
6217
+ function remotePreflightFailureMessage(machineId, detail) {
6218
+ const normalized = detail.trim();
6219
+ if (/Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED|Offending .*key in .*known_hosts/i.test(normalized)) {
6220
+ return [
6221
+ `remote preflight failed on ${machineId}: SSH host key verification failed.`,
6222
+ `Verify ${machineId}'s host identity and repair SSH known_hosts/trust material outside OpenLoops;`,
6223
+ "OpenLoops will not disable host-key checking or modify known_hosts automatically.",
6224
+ normalized ? `Transport detail: ${normalized}` : ""
6225
+ ].filter(Boolean).join(" ");
6226
+ }
6227
+ return `remote preflight failed on ${machineId}${normalized ? `: ${normalized}` : ""}`;
6228
+ }
5421
6229
  function assertCodewithProfileListed(profile, result) {
6230
+ const profiles = codewithProfileSetFromResult(result);
6231
+ if (!profiles.has(profile)) {
6232
+ throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
6233
+ }
6234
+ }
6235
+ function codewithProfileSetFromResult(result) {
5422
6236
  if (result.error) {
5423
6237
  throw new Error(`codewith auth profile preflight failed: ${result.error}`);
5424
6238
  }
@@ -5426,32 +6240,45 @@ function assertCodewithProfileListed(profile, result) {
5426
6240
  const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
5427
6241
  throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
5428
6242
  }
5429
- const profiles = new Set((result.stdout || "").split(/\r?\n/).slice(1).map(codewithProfileCandidateFromLine).filter(Boolean));
5430
- if (!profiles.has(profile)) {
5431
- throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
5432
- }
6243
+ return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
5433
6244
  }
5434
6245
  function preflightNativeAuthProfileSync(spec, env) {
5435
6246
  if (spec.nativeAuthProfile?.provider !== "codewith")
5436
6247
  return;
5437
- const result = spawnSync4(spec.command, ["profile", "list"], {
5438
- encoding: "utf8",
5439
- env,
5440
- stdio: ["ignore", "pipe", "pipe"],
5441
- timeout: 15000
5442
- });
5443
- assertCodewithProfileListed(spec.nativeAuthProfile.profile, {
5444
- status: result.status,
5445
- stdout: result.stdout ?? "",
5446
- stderr: result.stderr ?? "",
5447
- error: result.error?.message
5448
- });
6248
+ const runProfileList = (args) => {
6249
+ const result = spawnSync5(spec.command, args, {
6250
+ encoding: "utf8",
6251
+ env,
6252
+ stdio: ["ignore", "pipe", "pipe"],
6253
+ timeout: 15000
6254
+ });
6255
+ return {
6256
+ status: result.status,
6257
+ stdout: result.stdout ?? "",
6258
+ stderr: result.stderr ?? "",
6259
+ error: result.error?.message
6260
+ };
6261
+ };
6262
+ const jsonResult = runProfileList(["profile", "list", "--json"]);
6263
+ try {
6264
+ if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
6265
+ return;
6266
+ } catch {}
6267
+ assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
5449
6268
  }
5450
6269
  async function preflightNativeAuthProfile(spec, env) {
5451
6270
  if (spec.nativeAuthProfile?.provider !== "codewith")
5452
6271
  return;
5453
- const result = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
5454
- assertCodewithProfileListed(spec.nativeAuthProfile.profile, result);
6272
+ const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
6273
+ try {
6274
+ if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
6275
+ return;
6276
+ } catch {}
6277
+ const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
6278
+ assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
6279
+ }
6280
+ function spawnDetail(result) {
6281
+ return (result.stderr || result.stdout || result.error || "").toString().trim();
5455
6282
  }
5456
6283
  function resolvedDirEquals(left, right) {
5457
6284
  try {
@@ -5502,8 +6329,45 @@ async function ensureLocalWorktree(worktree, env) {
5502
6329
  }
5503
6330
  const current = await git(["-C", path, "branch", "--show-current"]);
5504
6331
  const actualBranch = current.stdout.trim();
5505
- if (current.error || (current.status ?? 1) !== 0 || actualBranch !== branch) {
5506
- return { error: `existing worktree ${path} is on branch ${actualBranch || "unknown"}, expected ${branch}` };
6332
+ if (current.error || (current.status ?? 1) !== 0) {
6333
+ return { error: `existing worktree ${path} branch could not be inspected${spawnDetail(current) ? `: ${spawnDetail(current)}` : ""}` };
6334
+ }
6335
+ if (actualBranch !== branch) {
6336
+ const actualLabel = actualBranch || "unknown";
6337
+ const status = await git(["-C", path, "status", "--porcelain=v1", "--untracked-files=all"]);
6338
+ if (status.error || (status.status ?? 1) !== 0) {
6339
+ const detail = spawnDetail(status);
6340
+ return {
6341
+ error: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, and cleanliness could not be checked; inspect or remove the stale worktree${detail ? `: ${detail}` : ""}`
6342
+ };
6343
+ }
6344
+ if (status.stdout.trim()) {
6345
+ return {
6346
+ error: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, and has local changes; commit/stash them or remove the stale worktree before retrying`
6347
+ };
6348
+ }
6349
+ const hasBranch2 = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
6350
+ const checkout = hasBranch2.status === 0 ? await git(["-C", path, "checkout", branch]) : actualBranch ? {
6351
+ status: 1,
6352
+ stdout: "",
6353
+ stderr: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, but expected branch does not exist; remove/prune the stale worktree or recreate the expected branch before retrying`,
6354
+ signal: null,
6355
+ timedOut: false
6356
+ } : await git(["-C", path, "checkout", "-b", branch]);
6357
+ if (checkout.error || (checkout.status ?? 1) !== 0) {
6358
+ const detail = spawnDetail(checkout);
6359
+ return {
6360
+ error: `existing worktree ${path} is clean but could not recover branch ${branch} from ${actualLabel}; remove/prune the stale worktree before retrying${detail ? `: ${detail}` : ""}`
6361
+ };
6362
+ }
6363
+ const recovered = await git(["-C", path, "branch", "--show-current"]);
6364
+ if (recovered.error || (recovered.status ?? 1) !== 0 || recovered.stdout.trim() !== branch) {
6365
+ const recoveredBranch = recovered.stdout.trim() || "unknown";
6366
+ const detail = spawnDetail(recovered);
6367
+ return {
6368
+ error: `existing worktree ${path} branch recovery ended on ${recoveredBranch}, expected ${branch}; remove/prune the stale worktree before retrying${detail ? `: ${detail}` : ""}`
6369
+ };
6370
+ }
5507
6371
  }
5508
6372
  return { cwd: worktree.cwd };
5509
6373
  }
@@ -5519,7 +6383,7 @@ async function ensureLocalWorktree(worktree, env) {
5519
6383
  const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
5520
6384
  const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
5521
6385
  if (add.error || (add.status ?? 1) !== 0) {
5522
- const detail = (add.stderr || add.stdout || add.error || "").toString().trim();
6386
+ const detail = spawnDetail(add);
5523
6387
  return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
5524
6388
  }
5525
6389
  return { cwd: worktree.cwd };
@@ -5554,7 +6418,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
5554
6418
  }
5555
6419
  function preflightRemoteSpec(spec, machine, metadata, opts) {
5556
6420
  const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
5557
- const result = spawnSync4(plan.command, plan.args, {
6421
+ const result = spawnSync5(plan.command, plan.args, {
5558
6422
  encoding: "utf8",
5559
6423
  env: transportEnv(opts),
5560
6424
  input: remotePreflightScript(spec, metadata),
@@ -5565,7 +6429,7 @@ function preflightRemoteSpec(spec, machine, metadata, opts) {
5565
6429
  throw new Error(`remote preflight failed on ${machine.id}: ${result.error.message}`);
5566
6430
  if ((result.status ?? 1) !== 0) {
5567
6431
  const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
5568
- throw new Error(`remote preflight failed on ${machine.id}${detail ? `: ${detail}` : ""}`);
6432
+ throw new Error(remotePreflightFailureMessage(machine.id, detail));
5569
6433
  }
5570
6434
  }
5571
6435
  async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
@@ -5653,6 +6517,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
5653
6517
  if (timedOut || idleTimedOut) {
5654
6518
  return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
5655
6519
  }
6520
+ if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
6521
+ return successResult(startedAt, fields);
6522
+ }
5656
6523
  if (error || exitCode !== 0) {
5657
6524
  return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
5658
6525
  }
@@ -5788,6 +6655,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
5788
6655
  if (timedOut || idleTimedOut) {
5789
6656
  return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
5790
6657
  }
6658
+ if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
6659
+ return successResult(startedAt, fields);
6660
+ }
5791
6661
  if (error || exitCode !== 0) {
5792
6662
  return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
5793
6663
  }
@@ -5818,322 +6688,392 @@ async function executeLoop(loop, run, opts = {}) {
5818
6688
  }, { ...opts, machine: opts.machine ?? loop.machine });
5819
6689
  }
5820
6690
 
5821
- // src/lib/doctor.ts
5822
- var PROVIDER_COMMANDS = [
5823
- "claude",
5824
- "agent",
5825
- "codewith",
5826
- "aicopilot",
5827
- "opencode",
5828
- "codex"
6691
+ // src/lib/health.ts
6692
+ import { createHash as createHash3 } from "crypto";
6693
+ import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
6694
+ import { join as join7 } from "path";
6695
+ var EVIDENCE_CHARS = 2000;
6696
+ var FINGERPRINT_EVIDENCE_CHARS = 120;
6697
+ var DEFAULT_SCAN_LIMIT = 200;
6698
+ var DEFAULT_SCAN_STATUSES = ["active", "paused"];
6699
+ var DEFAULT_MAX_FINDINGS = 100;
6700
+ var MIN_STALE_RUNNING_MS = 10 * 60000;
6701
+ var CLASSIFICATIONS = [
6702
+ "rate_limit",
6703
+ "auth",
6704
+ "provider_unavailable",
6705
+ "model_not_found",
6706
+ "context_length",
6707
+ "schema_response_format",
6708
+ "node_init",
6709
+ "preflight",
6710
+ "route_functional",
6711
+ "timeout",
6712
+ "sigsegv",
6713
+ "restart_interrupted",
6714
+ "skipped_previous_active",
6715
+ "circuit_breaker",
6716
+ "unknown"
5829
6717
  ];
5830
- function hasCommand(command) {
5831
- const result = spawnSync5("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
5832
- return (result.status ?? 1) === 0;
6718
+ var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
6719
+ function bounded(value, limit = EVIDENCE_CHARS) {
6720
+ if (!value)
6721
+ return;
6722
+ if (value.length <= limit)
6723
+ return value;
6724
+ return `${value.slice(0, limit)}
6725
+ [truncated ${value.length - limit} chars]`;
5833
6726
  }
5834
- function commandVersion(command) {
5835
- const result = spawnSync5(command, ["--version"], {
5836
- encoding: "utf8",
5837
- stdio: ["ignore", "pipe", "pipe"]
5838
- });
5839
- if ((result.status ?? 1) !== 0)
6727
+ function redactedEvidence(value) {
6728
+ return redact(bounded(value));
6729
+ }
6730
+ function searchableText(run) {
6731
+ return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
6732
+ `);
6733
+ }
6734
+ function isRecord(value) {
6735
+ return Boolean(value && typeof value === "object" && !Array.isArray(value));
6736
+ }
6737
+ function stringValue(value) {
6738
+ return typeof value === "string" && value.trim() ? value : undefined;
6739
+ }
6740
+ function objectField(value, key) {
6741
+ if (!value)
5840
6742
  return;
5841
- return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
6743
+ const field = value[key];
6744
+ return isRecord(field) ? field : undefined;
5842
6745
  }
5843
- function runDoctor(store) {
5844
- const checks = [];
5845
- try {
5846
- const dir = ensureDataDir();
5847
- accessSync2(dir, constants2.R_OK | constants2.W_OK);
5848
- checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
5849
- } catch (error) {
5850
- checks.push({
5851
- id: "data-dir",
5852
- status: "fail",
5853
- message: "data directory is not writable",
5854
- detail: error instanceof Error ? error.message : String(error)
5855
- });
6746
+ function tagsFromValue(value) {
6747
+ if (Array.isArray(value))
6748
+ return value.map((entry) => String(entry).trim()).filter(Boolean);
6749
+ if (typeof value === "string")
6750
+ return value.split(",").map((entry) => entry.trim()).filter(Boolean);
6751
+ return [];
6752
+ }
6753
+ function stableFingerprint(parts) {
6754
+ return createHash3("sha256").update(parts.join(`
6755
+ `)).digest("hex").slice(0, 16);
6756
+ }
6757
+ function stableScanFingerprint(parts) {
6758
+ return `openloops:health-scan:${stableFingerprint(parts)}`;
6759
+ }
6760
+ function safeHost(value) {
6761
+ if (!value)
6762
+ return;
6763
+ let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
6764
+ host = host.replace(/^\[|\]$/g, "");
6765
+ return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
6766
+ }
6767
+ function isCursorHost(host) {
6768
+ return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
6769
+ }
6770
+ function providerUnavailableSummary(rawText) {
6771
+ const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
6772
+ if (dns) {
6773
+ const host = safeHost(dns[2]);
6774
+ if (isCursorHost(host))
6775
+ return `provider DNS lookup failed: ${dns[1]} ${host}`;
5856
6776
  }
5857
- const bunVersion = commandVersion("bun");
5858
- checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
5859
- const accountsVersion = commandVersion("accounts");
5860
- checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
5861
- try {
5862
- const machines = listOpenMachines();
5863
- const local = machines.find((machine) => machine.local);
5864
- checks.push({
5865
- id: "machines",
5866
- status: "ok",
5867
- message: `OpenMachines topology available (${machines.length} machine(s))`,
5868
- detail: local ? `local=${local.id}` : undefined
5869
- });
5870
- } catch (error) {
5871
- checks.push({
5872
- id: "machines",
5873
- status: "warn",
5874
- message: "OpenMachines topology is not available; machine-assigned loops will fail",
5875
- detail: error instanceof Error ? error.message : String(error)
5876
- });
5877
- }
5878
- for (const command of PROVIDER_COMMANDS) {
5879
- checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
5880
- }
5881
- const status = daemonStatus(store);
5882
- checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
5883
- const failedRuns = store.countRuns("failed");
5884
- checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
5885
- const deployment = buildDeploymentStatus();
5886
- const schedulerState = deployment.schedulerState;
5887
- checks.push({
5888
- id: "scheduler-state",
5889
- status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
5890
- message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
5891
- detail: [
5892
- `route_state=${schedulerState.routeAdmission.stateStore}`,
5893
- `active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
5894
- `gates=${schedulerState.routeAdmission.gates.join(",")}`,
5895
- `artifacts=${schedulerState.localStore.runArtifacts}`,
5896
- `remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
5897
- `remote_apply=${String(schedulerState.remoteStore.applySupported)}`
5898
- ].join(" ")
5899
- });
5900
- for (const loop of store.listLoops({ status: "active" })) {
5901
- try {
5902
- if (loop.target.type === "workflow") {
5903
- const workflow = store.requireWorkflow(loop.target.workflowId);
5904
- for (const step of workflowExecutionOrder(workflow)) {
5905
- preflightTarget({
5906
- ...step.target,
5907
- account: step.account ?? step.target.account,
5908
- timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
5909
- }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
5910
- }
5911
- } else {
5912
- preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
5913
- }
5914
- checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
5915
- } catch (error) {
5916
- checks.push({
5917
- id: `loop:${loop.id}:preflight`,
5918
- status: "fail",
5919
- message: `active loop target preflight failed: ${loop.name}`,
5920
- detail: error instanceof Error ? error.message : String(error)
5921
- });
5922
- }
5923
- }
5924
- return {
5925
- ok: checks.every((check) => check.status !== "fail"),
5926
- checks
5927
- };
6777
+ return;
5928
6778
  }
5929
-
5930
- // src/lib/health.ts
5931
- import { createHash as createHash2 } from "crypto";
5932
- import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
5933
-
5934
- // src/lib/format.ts
5935
- var TEXT_OUTPUT_LIMIT = 32 * 1024;
5936
- var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
5937
- function redact(value, visible = 0) {
5938
- if (!value)
5939
- return value;
5940
- if (value.length <= visible)
5941
- return value;
5942
- if (visible <= 0)
5943
- return `[redacted ${value.length} chars]`;
5944
- return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
6779
+ function stableFailureFingerprint(run, classification, summary) {
6780
+ const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
6781
+ return stableFingerprint([
6782
+ run.loopId,
6783
+ classification,
6784
+ String(run.status),
6785
+ String(run.exitCode ?? ""),
6786
+ evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6787
+ ]);
5945
6788
  }
5946
- function truncateTextOutput(value) {
5947
- if (value.length <= TEXT_OUTPUT_LIMIT)
5948
- return value;
5949
- return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
5950
- [truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
6789
+ function stableRouteFunctionalFingerprint(loop, reason) {
6790
+ return stableFingerprint([
6791
+ loop.id,
6792
+ "route_functional",
6793
+ reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6794
+ ]);
5951
6795
  }
5952
- function redactSensitivePayload(value, key) {
5953
- if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
5954
- if (typeof value === "string")
5955
- return redact(value);
5956
- if (value === undefined || value === null)
5957
- return value;
5958
- return "[redacted]";
5959
- }
5960
- if (Array.isArray(value))
5961
- return value.map((item) => redactSensitivePayload(item));
5962
- if (value && typeof value === "object") {
5963
- return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
5964
- }
5965
- return value;
6796
+ function healthRun(run) {
6797
+ return {
6798
+ ...run,
6799
+ error: redactedEvidence(run.error),
6800
+ stdout: redactedEvidence(run.stdout),
6801
+ stderr: redactedEvidence(run.stderr)
6802
+ };
5966
6803
  }
5967
- function textOutputBlocks(value, opts = {}) {
5968
- const indent = opts.indent ?? "";
5969
- const nested = `${indent} `;
5970
- const blocks = [];
5971
- for (const [label, output] of [
5972
- ["stdout", value.stdout],
5973
- ["stderr", value.stderr]
5974
- ]) {
5975
- if (!output)
5976
- continue;
5977
- blocks.push(`${indent}${label}:`);
5978
- for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
5979
- blocks.push(`${nested}${line}`);
5980
- }
5981
- }
5982
- return blocks;
6804
+ function compactText(value, limit = 500) {
6805
+ const text = redact(bounded(value, limit));
6806
+ return text?.replace(/\s+/g, " ").trim();
5983
6807
  }
5984
- function publicLoop(loop) {
5985
- const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
6808
+ function publicDoctorCheck(check) {
5986
6809
  return {
5987
- ...loop,
5988
- target
6810
+ id: check.id,
6811
+ status: check.status,
6812
+ message: redact(bounded(check.message, 500)) ?? "",
6813
+ detail: redact(bounded(check.detail, 800))
5989
6814
  };
5990
6815
  }
5991
- function publicRun(run, showOutput = false, opts = {}) {
6816
+ function publicDoctorReport(report) {
5992
6817
  return {
5993
- ...run,
5994
- stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
5995
- stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
5996
- error: opts.redactError ? redact(run.error) : run.error
6818
+ ok: report.ok,
6819
+ checks: report.checks.map(publicDoctorCheck)
5997
6820
  };
5998
6821
  }
5999
- function publicExecutorResult(result, showOutput = false) {
6822
+ function includedStatusSet(statuses) {
6823
+ const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
6824
+ return new Set(values);
6825
+ }
6826
+ function statusCounts(loops) {
6000
6827
  return {
6001
- ...result,
6002
- stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
6003
- stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
6004
- error: redact(result.error)
6828
+ loops: loops.length,
6829
+ active: loops.filter((loop) => loop.status === "active").length,
6830
+ paused: loops.filter((loop) => loop.status === "paused").length,
6831
+ stopped: loops.filter((loop) => loop.status === "stopped").length,
6832
+ expired: loops.filter((loop) => loop.status === "expired").length
6005
6833
  };
6006
6834
  }
6007
- function publicWorkflow(workflow) {
6835
+ function compareLoopsForScan(left, right) {
6836
+ const statusOrder = left.status.localeCompare(right.status);
6837
+ if (statusOrder !== 0)
6838
+ return statusOrder;
6839
+ return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
6840
+ }
6841
+ function scanLoops(store, statuses, opts) {
6842
+ const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
6843
+ return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
6844
+ }
6845
+ function healthReportForLoops(store, loops, generatedAt) {
6846
+ const expectations = loops.map((loop) => expectationForLoop(store, loop));
6847
+ const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
6848
+ for (const expectation of expectations) {
6849
+ if (expectation.failure)
6850
+ classifications[expectation.failure.classification] += 1;
6851
+ }
6852
+ const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
6853
+ const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
6008
6854
  return {
6009
- ...workflow,
6010
- steps: workflow.steps.map((step) => ({
6011
- ...step,
6012
- target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
6013
- }))
6855
+ ok: unhealthy === 0,
6856
+ generatedAt,
6857
+ summary: {
6858
+ loops: expectations.length,
6859
+ healthy: expectations.length - unhealthy,
6860
+ unhealthy,
6861
+ warnings
6862
+ },
6863
+ classifications,
6864
+ expectations
6014
6865
  };
6015
6866
  }
6016
- function publicWorkflowRun(run) {
6017
- return { ...run, error: redact(run.error) };
6867
+ function ageMs(run, now) {
6868
+ const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
6869
+ const time = Date.parse(stamp);
6870
+ return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
6018
6871
  }
6019
- function publicWorkflowInvocation(invocation) {
6020
- return redactSensitivePayload(invocation);
6872
+ function shortLoop(loop) {
6873
+ return {
6874
+ id: loop.id,
6875
+ name: loop.name,
6876
+ status: loop.status,
6877
+ nextRunAt: loop.nextRunAt,
6878
+ leaseMs: loop.leaseMs
6879
+ };
6021
6880
  }
6022
- function publicWorkflowWorkItem(item) {
6023
- return { ...item, lastReason: redact(item.lastReason, 240) };
6881
+ function priorityForSeverity(severity) {
6882
+ if (severity === "critical")
6883
+ return "critical";
6884
+ if (severity === "high")
6885
+ return "high";
6886
+ if (severity === "low")
6887
+ return "low";
6888
+ return "medium";
6889
+ }
6890
+ function recommendedFindingTask(finding, route) {
6891
+ const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
6892
+ if (finding.classification)
6893
+ tags.push(finding.classification);
6894
+ const description = [
6895
+ `OpenLoops health scan found a ${finding.kind} issue.`,
6896
+ finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
6897
+ finding.run ? `Run: ${finding.run.id}` : undefined,
6898
+ finding.classification ? `Classification: ${finding.classification}` : undefined,
6899
+ finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
6900
+ finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
6901
+ `Fingerprint: ${finding.fingerprint}`,
6902
+ `Severity: ${finding.severity}`,
6903
+ `No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
6904
+ route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
6905
+ route?.provider ? `Provider: ${route.provider}` : undefined,
6906
+ "",
6907
+ finding.message,
6908
+ finding.run?.error ? `Error:
6909
+ ${finding.run.error}` : undefined,
6910
+ finding.run?.stderr ? `Stderr:
6911
+ ${finding.run.stderr}` : undefined
6912
+ ].filter(Boolean).join(`
6913
+
6914
+ `);
6915
+ return {
6916
+ title: finding.title,
6917
+ description,
6918
+ priority: priorityForSeverity(finding.severity),
6919
+ tags,
6920
+ dedupeKey: finding.fingerprint,
6921
+ search: { query: finding.fingerprint },
6922
+ compatibilityFallback: {
6923
+ search: ["todos", "search", finding.fingerprint, "--json"],
6924
+ add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
6925
+ comment: ["todos", "comment", "<task-id>", description]
6926
+ },
6927
+ futureNativeUpsert: {
6928
+ command: "todos task upsert",
6929
+ fields: {
6930
+ title: finding.title,
6931
+ description,
6932
+ priority: priorityForSeverity(finding.severity),
6933
+ tags,
6934
+ dedupeKey: finding.fingerprint,
6935
+ routeSource: route?.source ?? "openloops",
6936
+ routeKind: route?.kind ?? "health_scan",
6937
+ routeLoopId: finding.loop?.id ?? "",
6938
+ routeLoopName: finding.loop?.name ?? ""
6939
+ }
6940
+ }
6941
+ };
6024
6942
  }
6025
- function publicWorkflowStepRun(run, showOutput = false) {
6943
+ function daemonFinding(daemon) {
6944
+ if (daemon.running && !daemon.stale)
6945
+ return;
6946
+ const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
6947
+ const severity = "critical";
6948
+ const finding = {
6949
+ kind: "daemon",
6950
+ severity,
6951
+ fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
6952
+ title: "OpenLoops daemon health issue",
6953
+ message: reason
6954
+ };
6026
6955
  return {
6027
- ...run,
6028
- stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
6029
- stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
6030
- error: redact(run.error)
6956
+ ...finding,
6957
+ recommendedTask: recommendedFindingTask(finding, undefined)
6031
6958
  };
6032
6959
  }
6033
- function publicWorkflowEvent(event) {
6034
- return { ...event, payload: redactSensitivePayload(event.payload) };
6960
+ function doctorSeverity(check) {
6961
+ if (check.status === "fail" && check.id === "data-dir")
6962
+ return "critical";
6963
+ if (check.status === "fail")
6964
+ return "high";
6965
+ return "medium";
6035
6966
  }
6036
- function publicGoal(goal) {
6967
+ function doctorFinding(check, loop, route) {
6968
+ if (check.status === "ok" || check.id === "loop-runs")
6969
+ return;
6970
+ const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
6971
+ const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
6972
+ const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
6973
+ const finding = {
6974
+ kind,
6975
+ severity,
6976
+ fingerprint,
6977
+ title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
6978
+ message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
6979
+ loop: loop ? shortLoop(loop) : undefined,
6980
+ route,
6981
+ doctorCheck: publicDoctorCheck(check)
6982
+ };
6037
6983
  return {
6038
- ...goal,
6039
- objective: redact(goal.objective, 120)
6984
+ ...finding,
6985
+ recommendedTask: recommendedFindingTask(finding, route)
6040
6986
  };
6041
6987
  }
6042
- function publicGoalRun(run) {
6988
+ function latestRunFinding(expectation) {
6989
+ if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
6990
+ return;
6991
+ const failure = expectation.failure;
6992
+ if (!failure)
6993
+ return;
6994
+ const severity = expectation.loop.status === "active" ? "high" : "medium";
6043
6995
  return {
6044
- ...run,
6045
- evidence: redactSensitivePayload(run.evidence),
6046
- rawResponse: redactSensitivePayload(run.rawResponse)
6996
+ kind: "latest-run",
6997
+ severity,
6998
+ fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
6999
+ title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
7000
+ message: expectation.check.message,
7001
+ loop: expectation.loop,
7002
+ run: expectation.latestRun,
7003
+ route: expectation.route,
7004
+ classification: failure.classification,
7005
+ doctorCheck: undefined,
7006
+ recommendedTask: expectation.recommendedTask
6047
7007
  };
6048
7008
  }
6049
-
6050
- // src/lib/health.ts
6051
- var EVIDENCE_CHARS = 2000;
6052
- var FINGERPRINT_EVIDENCE_CHARS = 120;
6053
- var CLASSIFICATIONS = [
6054
- "rate_limit",
6055
- "auth",
6056
- "model_not_found",
6057
- "context_length",
6058
- "schema_response_format",
6059
- "node_init",
6060
- "preflight",
6061
- "route_functional",
6062
- "timeout",
6063
- "sigsegv",
6064
- "skipped_previous_active",
6065
- "circuit_breaker",
6066
- "unknown"
6067
- ];
6068
- function bounded(value, limit = EVIDENCE_CHARS) {
6069
- if (!value)
7009
+ function staleRunningFinding(loop, expectation, now, staleRunningMs) {
7010
+ const run = expectation.latestRun;
7011
+ if (loop.status !== "active" || run?.status !== "running")
6070
7012
  return;
6071
- if (value.length <= limit)
6072
- return value;
6073
- return `${value.slice(0, limit)}
6074
- [truncated ${value.length - limit} chars]`;
6075
- }
6076
- function redactedEvidence(value) {
6077
- return redact(bounded(value));
7013
+ const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
7014
+ const age = ageMs(run, now);
7015
+ if (age <= threshold)
7016
+ return;
7017
+ const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
7018
+ const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
7019
+ const finding = {
7020
+ kind: "stale-running",
7021
+ severity: "critical",
7022
+ fingerprint,
7023
+ title: `OpenLoops stale running run - ${loop.name}`,
7024
+ message,
7025
+ loop: shortLoop(loop),
7026
+ run,
7027
+ route: expectation.route,
7028
+ ageMs: age,
7029
+ staleThresholdMs: threshold
7030
+ };
7031
+ return {
7032
+ ...finding,
7033
+ recommendedTask: recommendedFindingTask(finding, expectation.route)
7034
+ };
6078
7035
  }
6079
- function searchableText(run) {
6080
- return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
6081
- `).toLowerCase();
7036
+ function scanStatus(findings) {
7037
+ if (findings.some((finding) => finding.severity === "critical"))
7038
+ return "critical";
7039
+ if (findings.length > 0)
7040
+ return "degraded";
7041
+ return "ok";
6082
7042
  }
6083
- function isRecord(value) {
6084
- return Boolean(value && typeof value === "object" && !Array.isArray(value));
7043
+ function timestampDir(root, generatedAt) {
7044
+ const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
7045
+ return join7(root, stamp);
6085
7046
  }
6086
- function stringValue(value) {
6087
- return typeof value === "string" && value.trim() ? value : undefined;
6088
- }
6089
- function objectField(value, key) {
6090
- if (!value)
6091
- return;
6092
- const field = value[key];
6093
- return isRecord(field) ? field : undefined;
6094
- }
6095
- function tagsFromValue(value) {
6096
- if (Array.isArray(value))
6097
- return value.map((entry) => String(entry).trim()).filter(Boolean);
6098
- if (typeof value === "string")
6099
- return value.split(",").map((entry) => entry.trim()).filter(Boolean);
6100
- return [];
6101
- }
6102
- function stableFingerprint(parts) {
6103
- return createHash2("sha256").update(parts.join(`
6104
- `)).digest("hex").slice(0, 16);
6105
- }
6106
- function stableFailureFingerprint(run, classification) {
6107
- return stableFingerprint([
6108
- run.loopId,
6109
- classification,
6110
- String(run.status),
6111
- String(run.exitCode ?? ""),
6112
- (run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6113
- ]);
6114
- }
6115
- function stableRouteFunctionalFingerprint(loop, reason) {
6116
- return stableFingerprint([
6117
- loop.id,
6118
- "route_functional",
6119
- reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
6120
- ]);
6121
- }
6122
- function healthRun(run) {
6123
- return {
6124
- ...run,
6125
- error: redactedEvidence(run.error),
6126
- stdout: redactedEvidence(run.stdout),
6127
- stderr: redactedEvidence(run.stderr)
6128
- };
7047
+ function healthScanMarkdown(scan) {
7048
+ return [
7049
+ "# OpenLoops Health Scan",
7050
+ "",
7051
+ `- status: ${scan.status}`,
7052
+ `- generated_at: ${scan.generatedAt}`,
7053
+ `- included_statuses: ${scan.includedStatuses.join(",")}`,
7054
+ `- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
7055
+ `- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
7056
+ scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
7057
+ scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
7058
+ scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
7059
+ "",
7060
+ "## Findings",
7061
+ scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
7062
+ `) : "None."
7063
+ ].join(`
7064
+ `);
6129
7065
  }
6130
7066
  function classifyRunFailure(run) {
6131
7067
  if (run.status === "succeeded" || run.status === "running")
6132
7068
  return;
6133
- const text = searchableText(run);
7069
+ const rawText = searchableText(run);
7070
+ const text = rawText.toLowerCase();
6134
7071
  let classification = "unknown";
7072
+ let summary;
6135
7073
  if (run.status === "timed_out")
6136
7074
  classification = "timeout";
7075
+ else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
7076
+ classification = "restart_interrupted";
6137
7077
  else if (run.status === "skipped" && /circuit breaker open/.test(text))
6138
7078
  classification = "circuit_breaker";
6139
7079
  else if (run.status === "skipped" && /previous run still active/.test(text))
@@ -6142,8 +7082,10 @@ function classifyRunFailure(run) {
6142
7082
  classification = "preflight";
6143
7083
  else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
6144
7084
  classification = "rate_limit";
6145
- else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
7085
+ else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
6146
7086
  classification = "auth";
7087
+ else if (summary = providerUnavailableSummary(rawText))
7088
+ classification = "provider_unavailable";
6147
7089
  else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
6148
7090
  classification = "model_not_found";
6149
7091
  else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
@@ -6156,8 +7098,9 @@ function classifyRunFailure(run) {
6156
7098
  classification = "sigsegv";
6157
7099
  return {
6158
7100
  classification,
6159
- fingerprint: stableFailureFingerprint(run, classification),
7101
+ fingerprint: stableFailureFingerprint(run, classification, summary),
6160
7102
  evidence: {
7103
+ summary,
6161
7104
  error: redactedEvidence(run.error),
6162
7105
  stdout: redactedEvidence(run.stdout),
6163
7106
  stderr: redactedEvidence(run.stderr),
@@ -6202,7 +7145,7 @@ function routeEvidenceReport(run) {
6202
7145
  const evidencePath = stringValue(stdoutReport?.evidencePath);
6203
7146
  if (evidencePath && existsSync4(evidencePath)) {
6204
7147
  try {
6205
- return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
7148
+ return parseJsonObject(readFileSync5(evidencePath, "utf8")) ?? stdoutReport;
6206
7149
  } catch {
6207
7150
  return stdoutReport;
6208
7151
  }
@@ -6334,6 +7277,12 @@ function targetRoute(loop) {
6334
7277
  loopName: loop.name
6335
7278
  };
6336
7279
  }
7280
+ function expectationLoop(loop) {
7281
+ return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
7282
+ }
7283
+ function hasPendingRetry(loop, run) {
7284
+ return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
7285
+ }
6337
7286
  function recommendedTask(loop, run, failure, route) {
6338
7287
  const title = `BUG: open-loops loop failure - ${loop.name}`;
6339
7288
  const description = [
@@ -6345,6 +7294,7 @@ function recommendedTask(loop, run, failure, route) {
6345
7294
  `No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
6346
7295
  route.cwd ? `Route cwd: ${route.cwd}` : undefined,
6347
7296
  route.provider ? `Provider: ${route.provider}` : undefined,
7297
+ failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
6348
7298
  failure.evidence.error ? `Error:
6349
7299
  ${failure.evidence.error}` : undefined,
6350
7300
  failure.evidence.stderr ? `Stderr:
@@ -6354,7 +7304,7 @@ ${failure.evidence.stderr}` : undefined
6354
7304
  `);
6355
7305
  const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
6356
7306
  const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
6357
- const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
7307
+ const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
6358
7308
  return {
6359
7309
  title,
6360
7310
  description,
@@ -6388,7 +7338,7 @@ function expectationForLoop(store, loop) {
6388
7338
  const route = targetRoute(loop);
6389
7339
  if (!latestRun) {
6390
7340
  return {
6391
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
7341
+ loop: expectationLoop(loop),
6392
7342
  ok: true,
6393
7343
  check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
6394
7344
  route
@@ -6397,7 +7347,7 @@ function expectationForLoop(store, loop) {
6397
7347
  const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
6398
7348
  if (routeFailure) {
6399
7349
  return {
6400
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
7350
+ loop: expectationLoop(loop),
6401
7351
  ok: false,
6402
7352
  check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
6403
7353
  latestRun: healthRun(latestRun),
@@ -6408,7 +7358,7 @@ function expectationForLoop(store, loop) {
6408
7358
  }
6409
7359
  if (latestRun.status === "succeeded") {
6410
7360
  return {
6411
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
7361
+ loop: expectationLoop(loop),
6412
7362
  ok: true,
6413
7363
  check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
6414
7364
  latestRun: healthRun(latestRun),
@@ -6419,7 +7369,7 @@ function expectationForLoop(store, loop) {
6419
7369
  if (failure?.classification === "circuit_breaker") {
6420
7370
  if (loop.status !== "paused") {
6421
7371
  return {
6422
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
7372
+ loop: expectationLoop(loop),
6423
7373
  ok: true,
6424
7374
  check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
6425
7375
  latestRun: healthRun(latestRun),
@@ -6427,7 +7377,7 @@ function expectationForLoop(store, loop) {
6427
7377
  };
6428
7378
  }
6429
7379
  return {
6430
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
7380
+ loop: expectationLoop(loop),
6431
7381
  ok: false,
6432
7382
  check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
6433
7383
  latestRun: healthRun(latestRun),
@@ -6436,8 +7386,33 @@ function expectationForLoop(store, loop) {
6436
7386
  recommendedTask: recommendedTask(loop, latestRun, failure, route)
6437
7387
  };
6438
7388
  }
7389
+ if (failure?.classification === "restart_interrupted") {
7390
+ return {
7391
+ loop: expectationLoop(loop),
7392
+ ok: true,
7393
+ check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
7394
+ latestRun: healthRun(latestRun),
7395
+ failure,
7396
+ route
7397
+ };
7398
+ }
7399
+ if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
7400
+ const message = [
7401
+ "provider unavailable/network failure; retry is scheduled",
7402
+ loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
7403
+ failure.evidence.summary
7404
+ ].filter(Boolean).join("; ");
7405
+ return {
7406
+ loop: expectationLoop(loop),
7407
+ ok: true,
7408
+ check: { id: "latest-run-succeeded", status: "warn", message },
7409
+ latestRun: healthRun(latestRun),
7410
+ failure,
7411
+ route
7412
+ };
7413
+ }
6439
7414
  return {
6440
- loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
7415
+ loop: expectationLoop(loop),
6441
7416
  ok: false,
6442
7417
  check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
6443
7418
  latestRun: healthRun(latestRun),
@@ -6469,10 +7444,210 @@ function buildHealthReport(store, opts = {}) {
6469
7444
  expectations
6470
7445
  };
6471
7446
  }
7447
+ function buildHealthScan(store, opts = {}) {
7448
+ const generatedAt = (opts.now ?? new Date).toISOString();
7449
+ const now = opts.now ?? new Date(generatedAt);
7450
+ const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
7451
+ const loops = scanLoops(store, includeStatuses, opts);
7452
+ const health = healthReportForLoops(store, loops, generatedAt);
7453
+ const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
7454
+ const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
7455
+ const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
7456
+ const allFindings = [];
7457
+ const pushFinding = (finding) => {
7458
+ if (!finding)
7459
+ return;
7460
+ allFindings.push(finding);
7461
+ };
7462
+ if (opts.daemon)
7463
+ pushFinding(daemonFinding(opts.daemon));
7464
+ if (opts.latestRun !== false) {
7465
+ for (const loop of loops) {
7466
+ const expectation = expectationsByLoopId.get(loop.id);
7467
+ if (!expectation)
7468
+ continue;
7469
+ pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
7470
+ pushFinding(latestRunFinding(expectation));
7471
+ }
7472
+ }
7473
+ const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
7474
+ if (doctor) {
7475
+ for (const check of doctor.checks) {
7476
+ const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
7477
+ const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
7478
+ const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
7479
+ pushFinding(doctorFinding(check, loop, route));
7480
+ }
7481
+ }
7482
+ const findings = allFindings.slice(0, maxFindings);
7483
+ const status = scanStatus(allFindings);
7484
+ const baseCounts = statusCounts(loops);
7485
+ return {
7486
+ ok: status === "ok",
7487
+ status,
7488
+ generatedAt,
7489
+ includedStatuses: includeStatuses,
7490
+ counts: {
7491
+ ...baseCounts,
7492
+ latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
7493
+ staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
7494
+ daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
7495
+ doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
7496
+ preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
7497
+ findings: allFindings.length,
7498
+ reportedFindings: findings.length,
7499
+ truncatedFindings: Math.max(0, allFindings.length - findings.length)
7500
+ },
7501
+ daemon: opts.daemon ? {
7502
+ running: opts.daemon.running,
7503
+ stale: opts.daemon.stale,
7504
+ pid: opts.daemon.pid,
7505
+ host: opts.daemon.host,
7506
+ loops: opts.daemon.loops,
7507
+ runs: opts.daemon.runs,
7508
+ logPath: opts.daemon.logPath
7509
+ } : undefined,
7510
+ doctor,
7511
+ health,
7512
+ selfHeals: opts.selfHeals ?? [],
7513
+ findings
7514
+ };
7515
+ }
7516
+ function writeHealthScanReports(scan, opts = {}) {
7517
+ const root = opts.reportDir ?? join7(dataDir(), "reports", "health-scan");
7518
+ const dir = timestampDir(root, scan.generatedAt);
7519
+ mkdirSync6(dir, { recursive: true, mode: 448 });
7520
+ const json = join7(dir, "summary.json");
7521
+ const markdown = join7(dir, "report.md");
7522
+ const withReports = {
7523
+ ...scan,
7524
+ reports: { dir, json, markdown }
7525
+ };
7526
+ writeFileSync3(json, JSON.stringify(withReports, null, 2), { mode: 384 });
7527
+ writeFileSync3(markdown, healthScanMarkdown(withReports), { mode: 384 });
7528
+ return withReports;
7529
+ }
7530
+
7531
+ // src/lib/doctor.ts
7532
+ var PROVIDER_COMMANDS = [
7533
+ "claude",
7534
+ "agent",
7535
+ "codewith",
7536
+ "aicopilot",
7537
+ "opencode",
7538
+ "codex"
7539
+ ];
7540
+ function hasCommand(command) {
7541
+ const result = spawnSync6("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
7542
+ return (result.status ?? 1) === 0;
7543
+ }
7544
+ function commandVersion(command) {
7545
+ const result = spawnSync6(command, ["--version"], {
7546
+ encoding: "utf8",
7547
+ stdio: ["ignore", "pipe", "pipe"]
7548
+ });
7549
+ if ((result.status ?? 1) !== 0)
7550
+ return;
7551
+ return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
7552
+ }
7553
+ function runDoctor(store) {
7554
+ const checks = [];
7555
+ try {
7556
+ const dir = ensureDataDir();
7557
+ accessSync2(dir, constants2.R_OK | constants2.W_OK);
7558
+ checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
7559
+ } catch (error) {
7560
+ checks.push({
7561
+ id: "data-dir",
7562
+ status: "fail",
7563
+ message: "data directory is not writable",
7564
+ detail: error instanceof Error ? error.message : String(error)
7565
+ });
7566
+ }
7567
+ const bunVersion = commandVersion("bun");
7568
+ checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
7569
+ const accountsVersion = commandVersion("accounts");
7570
+ checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
7571
+ try {
7572
+ const machines = listOpenMachines();
7573
+ const local = machines.find((machine) => machine.local);
7574
+ checks.push({
7575
+ id: "machines",
7576
+ status: "ok",
7577
+ message: `OpenMachines topology available (${machines.length} machine(s))`,
7578
+ detail: local ? `local=${local.id}` : undefined
7579
+ });
7580
+ } catch (error) {
7581
+ checks.push({
7582
+ id: "machines",
7583
+ status: "warn",
7584
+ message: "OpenMachines topology is not available; machine-assigned loops will fail",
7585
+ detail: error instanceof Error ? error.message : String(error)
7586
+ });
7587
+ }
7588
+ for (const command of PROVIDER_COMMANDS) {
7589
+ checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
7590
+ }
7591
+ const status = daemonStatus(store);
7592
+ checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
7593
+ const failedRuns = store.countRuns("failed");
7594
+ const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
7595
+ checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
7596
+ if (restartInterruptedRuns > 0) {
7597
+ checks.push({
7598
+ id: "loop-runs:restart-interrupted",
7599
+ status: "warn",
7600
+ message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
7601
+ });
7602
+ }
7603
+ const deployment = buildDeploymentStatus();
7604
+ const schedulerState = deployment.schedulerState;
7605
+ checks.push({
7606
+ id: "scheduler-state",
7607
+ status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
7608
+ message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
7609
+ detail: [
7610
+ `route_state=${schedulerState.routeAdmission.stateStore}`,
7611
+ `active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
7612
+ `gates=${schedulerState.routeAdmission.gates.join(",")}`,
7613
+ `artifacts=${schedulerState.localStore.runArtifacts}`,
7614
+ `remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
7615
+ `remote_apply=${String(schedulerState.remoteStore.applySupported)}`
7616
+ ].join(" ")
7617
+ });
7618
+ for (const loop of store.listLoops({ status: "active" })) {
7619
+ try {
7620
+ if (loop.target.type === "workflow") {
7621
+ const workflow = store.requireWorkflow(loop.target.workflowId);
7622
+ for (const step of workflowExecutionOrder(workflow)) {
7623
+ preflightTarget({
7624
+ ...step.target,
7625
+ account: step.account ?? step.target.account,
7626
+ timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
7627
+ }, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
7628
+ }
7629
+ } else {
7630
+ preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
7631
+ }
7632
+ checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
7633
+ } catch (error) {
7634
+ checks.push({
7635
+ id: `loop:${loop.id}:preflight`,
7636
+ status: "fail",
7637
+ message: `active loop target preflight failed: ${loop.name}`,
7638
+ detail: error instanceof Error ? error.message : String(error)
7639
+ });
7640
+ }
7641
+ }
7642
+ return {
7643
+ ok: checks.every((check) => check.status !== "fail"),
7644
+ checks
7645
+ };
7646
+ }
6472
7647
 
6473
7648
  // src/lib/migration.ts
6474
- import { createHash as createHash3 } from "crypto";
6475
- import { hostname as hostname2 } from "os";
7649
+ import { createHash as createHash4 } from "crypto";
7650
+ import { hostname as hostname3 } from "os";
6476
7651
  var LOOPS_MIGRATION_SCHEMA = "open-loops.migration/v1";
6477
7652
  function canonicalize(value) {
6478
7653
  if (Array.isArray(value))
@@ -6483,7 +7658,7 @@ function canonicalize(value) {
6483
7658
  return value;
6484
7659
  }
6485
7660
  function migrationHash(value) {
6486
- return createHash3("sha256").update(JSON.stringify(canonicalize(value))).digest("hex");
7661
+ return createHash4("sha256").update(JSON.stringify(canonicalize(value))).digest("hex");
6487
7662
  }
6488
7663
  function pushBlocker(rows, resource, id, reason, name) {
6489
7664
  rows.push({ resource, id, name, action: "blocked", reason });
@@ -6559,7 +7734,7 @@ function exportLoopsMigrationBundle(store, opts = {}) {
6559
7734
  source: {
6560
7735
  backend: "sqlite",
6561
7736
  schemaVersion: rows.schemaVersion,
6562
- hostname: hostname2()
7737
+ hostname: hostname3()
6563
7738
  },
6564
7739
  checks: rows.checks,
6565
7740
  importable: sanitized.blockers.length === 0,
@@ -6940,6 +8115,85 @@ async function buildSelfHostedMigrationPlan(store, opts) {
6940
8115
  });
6941
8116
  return { ...plan, importable: false };
6942
8117
  }
8118
+ async function postImportBatch(fetchImpl, config, payload) {
8119
+ const body = await requestJson(fetchImpl, config, "/v1/import", { method: "POST", body: JSON.stringify(payload) });
8120
+ const imported = body.imported ?? {};
8121
+ return {
8122
+ imported: { workflows: imported.workflows ?? 0, loops: imported.loops ?? 0, runs: imported.runs ?? 0 },
8123
+ skippedRunning: typeof body.skippedRunning === "number" ? body.skippedRunning : 0
8124
+ };
8125
+ }
8126
+ async function applySelfHostedPush(store, opts) {
8127
+ const resolved = resolveApiConfig(opts);
8128
+ if (!resolved.apiUrl)
8129
+ throw new ValidationError("LOOPS_API_URL or --api-url is required for self-hosted push");
8130
+ if (!isLocalApiUrl(resolved.apiUrl) && !resolved.token) {
8131
+ throw new ValidationError("non-local self-hosted APIs require LOOPS_API_TOKEN or HASNA_LOOPS_API_TOKEN");
8132
+ }
8133
+ const config = { apiUrl: resolved.apiUrl, token: resolved.token };
8134
+ const fetchImpl = opts.fetchImpl ?? fetch;
8135
+ const includeRuns = opts.includeRuns ?? true;
8136
+ const replace = opts.replace ?? false;
8137
+ const batchRows = Math.max(1, opts.batchRows ?? 200);
8138
+ const runBatchBytes = Math.max(64 * 1024, opts.runBatchBytes ?? 4 * 1024 * 1024);
8139
+ const applied = { workflows: 0, loops: 0, runs: 0 };
8140
+ const skipped = { runningRuns: 0, orphanRuns: 0 };
8141
+ let requests = 0;
8142
+ const base = store.exportMigrationRows({ includeRuns: false });
8143
+ const loopIds = new Set(base.loops.map((loop) => loop.id));
8144
+ for (let i = 0;i < base.workflows.length; i += batchRows) {
8145
+ const batch = base.workflows.slice(i, i + batchRows);
8146
+ const result = await postImportBatch(fetchImpl, config, { workflows: batch, replace });
8147
+ applied.workflows += result.imported.workflows;
8148
+ requests += 1;
8149
+ opts.onProgress?.({ phase: "workflows", sent: applied.workflows, requests });
8150
+ }
8151
+ for (let i = 0;i < base.loops.length; i += batchRows) {
8152
+ const batch = base.loops.slice(i, i + batchRows);
8153
+ const result = await postImportBatch(fetchImpl, config, { loops: batch, replace });
8154
+ applied.loops += result.imported.loops;
8155
+ requests += 1;
8156
+ opts.onProgress?.({ phase: "loops", sent: applied.loops, requests });
8157
+ }
8158
+ if (includeRuns) {
8159
+ const pageSize = 500;
8160
+ let pending = [];
8161
+ let pendingBytes = 0;
8162
+ const flush = async () => {
8163
+ if (pending.length === 0)
8164
+ return;
8165
+ const result = await postImportBatch(fetchImpl, config, { runs: pending, replace });
8166
+ applied.runs += result.imported.runs;
8167
+ skipped.runningRuns += result.skippedRunning;
8168
+ requests += 1;
8169
+ opts.onProgress?.({ phase: "runs", sent: applied.runs, requests });
8170
+ pending = [];
8171
+ pendingBytes = 0;
8172
+ };
8173
+ for (let offset = 0;; offset += pageSize) {
8174
+ const page = store.exportMigrationRunPage({ limit: pageSize, offset });
8175
+ if (page.length === 0)
8176
+ break;
8177
+ for (const run of page) {
8178
+ if (!loopIds.has(run.loopId)) {
8179
+ skipped.orphanRuns += 1;
8180
+ continue;
8181
+ }
8182
+ const encoded = JSON.stringify(run).length;
8183
+ if (pending.length > 0 && pendingBytes + encoded > runBatchBytes)
8184
+ await flush();
8185
+ pending.push(run);
8186
+ pendingBytes += encoded;
8187
+ if (pending.length >= batchRows * 4)
8188
+ await flush();
8189
+ }
8190
+ if (page.length < pageSize)
8191
+ break;
8192
+ }
8193
+ await flush();
8194
+ }
8195
+ return { ok: true, apiUrl: config.apiUrl, applied, skipped, requests };
8196
+ }
6943
8197
  async function registerSelfHostedRunner(opts) {
6944
8198
  const config = resolveApiConfig(opts);
6945
8199
  if (!config.apiUrl)
@@ -7064,55 +8318,6 @@ ${evidence.join(`
7064
8318
  import { generateObject } from "ai";
7065
8319
  import { z } from "zod";
7066
8320
 
7067
- // src/lib/run-envelope.ts
7068
- var ENVELOPE_EXCERPT_CHARS = 2048;
7069
- var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
7070
- function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
7071
- if (!text)
7072
- return;
7073
- const scrubbed = scrubSecrets(text);
7074
- if (scrubbed.length <= limit * 2)
7075
- return scrubbed;
7076
- const omitted = scrubbed.length - limit * 2;
7077
- return `${scrubbed.slice(0, limit)}
7078
- [... ${omitted} chars omitted ...]
7079
- ${scrubbed.slice(-limit)}`;
7080
- }
7081
- function summarizeOutput(stdout, stderr) {
7082
- return {
7083
- stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
7084
- stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
7085
- stdoutExcerpt: boundedExcerpt(stdout),
7086
- stderrExcerpt: boundedExcerpt(stderr)
7087
- };
7088
- }
7089
- function summarizeExecutorResult(result) {
7090
- return {
7091
- ...summarizeOutput(result.stdout, result.stderr),
7092
- status: result.status,
7093
- exitCode: result.exitCode,
7094
- durationMs: result.durationMs,
7095
- error: boundedExcerpt(result.error)
7096
- };
7097
- }
7098
- function isBlockedStepRun(step) {
7099
- return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
7100
- }
7101
- function summarizeWorkflowStepRun(step) {
7102
- return {
7103
- ...summarizeOutput(step.stdout, step.stderr),
7104
- stepId: step.stepId,
7105
- status: step.status,
7106
- exitCode: step.exitCode,
7107
- durationMs: step.durationMs,
7108
- error: boundedExcerpt(step.error),
7109
- blocked: isBlockedStepRun(step)
7110
- };
7111
- }
7112
- function workflowRunEnvelope(run, steps) {
7113
- return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
7114
- }
7115
-
7116
8321
  // src/lib/goal/model-factory.ts
7117
8322
  import { createOpenRouter } from "@openrouter/ai-sdk-provider";
7118
8323
  var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
@@ -7977,7 +9182,7 @@ var MAX_RETRY_EXPONENT = 20;
7977
9182
  function retryBackoffDelayMs(loop, run, random = Math.random) {
7978
9183
  const attempt = Math.max(1, run.attempt);
7979
9184
  const failure = classifyRunFailure(run);
7980
- const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
9185
+ const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
7981
9186
  const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
7982
9187
  const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
7983
9188
  const jitter = 0.5 + random();
@@ -8104,20 +9309,21 @@ async function executeClaimedRun(deps) {
8104
9309
  daemonLeaseId: deps.daemonLeaseId,
8105
9310
  onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
8106
9311
  })))(deps.loop, deps.run);
9312
+ const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
8107
9313
  deps.beforeFinalize?.(deps.loop, deps.run);
8108
9314
  return deps.store.finalizeRun(deps.run.id, {
8109
- status: result.status,
8110
- finishedAt: result.finishedAt,
8111
- durationMs: result.durationMs,
8112
- stdout: result.stdout,
8113
- stderr: result.stderr,
8114
- exitCode: result.exitCode,
8115
- error: result.error,
8116
- pid: result.pid
9315
+ status: finalResult.status,
9316
+ finishedAt: finalResult.finishedAt,
9317
+ durationMs: finalResult.durationMs,
9318
+ stdout: finalResult.stdout,
9319
+ stderr: finalResult.stderr,
9320
+ exitCode: finalResult.exitCode,
9321
+ error: finalResult.error,
9322
+ pid: finalResult.pid
8117
9323
  }, {
8118
9324
  claimedBy: deps.runnerId,
8119
9325
  daemonLeaseId: deps.daemonLeaseId,
8120
- now: deps.now?.() ?? new Date(result.finishedAt)
9326
+ now: deps.now?.() ?? new Date(finalResult.finishedAt)
8121
9327
  });
8122
9328
  } catch (err) {
8123
9329
  deps.onError?.(deps.loop, err);
@@ -8171,184 +9377,1068 @@ function repairWedgedTerminalSlot(deps, loop, scheduledFor, now) {
8171
9377
  throw error;
8172
9378
  }
8173
9379
  }
8174
- async function runSlot(deps, loop, scheduledFor) {
8175
- const now = deps.now?.() ?? new Date;
8176
- deps.beforeRun?.(loop, scheduledFor);
8177
- if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
8178
- let skipped;
9380
+ async function runSlot(deps, loop, scheduledFor) {
9381
+ const now = deps.now?.() ?? new Date;
9382
+ deps.beforeRun?.(loop, scheduledFor);
9383
+ if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
9384
+ let skipped;
9385
+ try {
9386
+ skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
9387
+ daemonLeaseId: deps.daemonLeaseId
9388
+ });
9389
+ } catch (error) {
9390
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
9391
+ return;
9392
+ throw error;
9393
+ }
9394
+ advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
9395
+ deps.onRun?.(skipped);
9396
+ return skipped;
9397
+ }
9398
+ let claim;
9399
+ try {
9400
+ claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
9401
+ } catch (error) {
9402
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
9403
+ return;
9404
+ throw error;
9405
+ }
9406
+ if (!claim) {
9407
+ repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
9408
+ return;
9409
+ }
9410
+ deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
9411
+ deps.onRun?.(claim.run);
9412
+ const finalRun = await executeClaimedRun({
9413
+ store: deps.store,
9414
+ runnerId: deps.runnerId,
9415
+ loop: claim.loop,
9416
+ run: claim.run,
9417
+ now: deps.now,
9418
+ execute: deps.execute,
9419
+ beforeFinalize: deps.beforeFinalize,
9420
+ daemonLeaseId: deps.daemonLeaseId,
9421
+ onError: deps.onError
9422
+ });
9423
+ advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
9424
+ deps.onRun?.(finalRun);
9425
+ return finalRun;
9426
+ }
9427
+ function claimSlot(deps, loop, scheduledFor) {
9428
+ const now = deps.now?.() ?? new Date;
9429
+ deps.beforeRun?.(loop, scheduledFor);
9430
+ if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
9431
+ if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
9432
+ return;
9433
+ let skipped;
9434
+ try {
9435
+ skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
9436
+ daemonLeaseId: deps.daemonLeaseId
9437
+ });
9438
+ } catch (error) {
9439
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
9440
+ return;
9441
+ throw error;
9442
+ }
9443
+ advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
9444
+ deps.onRun?.(skipped);
9445
+ return skipped;
9446
+ }
9447
+ let claim;
9448
+ try {
9449
+ claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
9450
+ } catch (error) {
9451
+ if (deps.daemonLeaseId && isDaemonLeaseLost(error))
9452
+ return;
9453
+ throw error;
9454
+ }
9455
+ if (!claim) {
9456
+ repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
9457
+ return;
9458
+ }
9459
+ deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
9460
+ deps.onRun?.(claim.run);
9461
+ return claim;
9462
+ }
9463
+ function recoverAndExpire(deps, now) {
9464
+ const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
9465
+ const recoveredByLoop = new Map;
9466
+ for (const run of recovered) {
9467
+ recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
9468
+ }
9469
+ for (const runs of recoveredByLoop.values()) {
9470
+ const loop = deps.store.getLoop(runs[0].loopId);
9471
+ if (!loop)
9472
+ continue;
9473
+ const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
9474
+ if (retryable) {
9475
+ advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
9476
+ continue;
9477
+ }
9478
+ for (const run of runs) {
9479
+ const current = deps.store.getLoop(run.loopId);
9480
+ if (current) {
9481
+ advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
9482
+ }
9483
+ }
9484
+ }
9485
+ const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
9486
+ return { recovered, expired };
9487
+ }
9488
+ function claimDueRuns(deps) {
9489
+ const now = deps.now?.() ?? new Date;
9490
+ const { recovered, expired } = recoverAndExpire(deps, now);
9491
+ const claims = [];
9492
+ const claimed = [];
9493
+ const skipped = [];
9494
+ const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
9495
+ if (maxClaims === 0)
9496
+ return { claims, claimed, completed: [], skipped, recovered, expired };
9497
+ const laneLimits = deps.laneLimits;
9498
+ const laneClaims = { command: 0, agent: 0 };
9499
+ const laneCap = (lane) => laneLimits === undefined ? Number.POSITIVE_INFINITY : Math.max(0, laneLimits[lane] ?? Number.POSITIVE_INFINITY);
9500
+ const laneFull = (lane) => laneClaims[lane] >= laneCap(lane);
9501
+ for (const loop of deps.store.dueLoops(now)) {
9502
+ if (claims.length >= maxClaims)
9503
+ break;
9504
+ const lane = loopLane(loop);
9505
+ if (laneFull(lane))
9506
+ continue;
9507
+ const plan = dueSlots(loop, now);
9508
+ let loopSkips = 0;
9509
+ for (const slot of plan.slots) {
9510
+ if (claims.length >= maxClaims)
9511
+ break;
9512
+ if (laneFull(lane))
9513
+ break;
9514
+ if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
9515
+ break;
9516
+ const run = claimSlot(deps, loop, slot);
9517
+ if (!run)
9518
+ continue;
9519
+ if ("loop" in run) {
9520
+ claims.push(run);
9521
+ claimed.push(run.run);
9522
+ laneClaims[lane] += 1;
9523
+ } else if (run.status === "skipped") {
9524
+ skipped.push(run);
9525
+ loopSkips += 1;
9526
+ }
9527
+ }
9528
+ }
9529
+ return { claims, claimed, completed: [], skipped, recovered, expired };
9530
+ }
9531
+ async function tick(deps) {
9532
+ const now = deps.now?.() ?? new Date;
9533
+ const { recovered, expired } = recoverAndExpire(deps, now);
9534
+ const claimed = [];
9535
+ const completed = [];
9536
+ const skipped = [];
9537
+ for (const loop of deps.store.dueLoops(now)) {
9538
+ const plan = dueSlots(loop, now);
9539
+ let loopSkips = 0;
9540
+ for (const slot of plan.slots) {
9541
+ if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
9542
+ break;
9543
+ const run = await runSlot(deps, loop, slot);
9544
+ if (!run)
9545
+ continue;
9546
+ if (run.status === "running")
9547
+ claimed.push(run);
9548
+ else if (run.status === "skipped") {
9549
+ skipped.push(run);
9550
+ loopSkips += 1;
9551
+ } else
9552
+ completed.push(run);
9553
+ if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
9554
+ break;
9555
+ }
9556
+ }
9557
+ return { claimed, completed, skipped, recovered, expired };
9558
+ }
9559
+
9560
+ // src/lib/cloud/mode.ts
9561
+ var DEPRECATED_STORAGE_MODE_ALIASES = ["remote", "hybrid", "self_hosted"];
9562
+ function normalizeStorageMode(value) {
9563
+ const normalized = value.trim().toLowerCase().replace(/-/g, "_");
9564
+ if (normalized === "local")
9565
+ return { mode: "local", deprecatedAlias: null };
9566
+ if (normalized === "cloud")
9567
+ return { mode: "cloud", deprecatedAlias: null };
9568
+ if (DEPRECATED_STORAGE_MODE_ALIASES.includes(normalized)) {
9569
+ return { mode: "cloud", deprecatedAlias: normalized };
9570
+ }
9571
+ throw new Error(`Unknown storage mode: ${value}. Use local or cloud.`);
9572
+ }
9573
+ function envToken(name) {
9574
+ return name.toUpperCase().replace(/-/g, "_");
9575
+ }
9576
+
9577
+ // src/lib/cloud/transport.ts
9578
+ function defaultCloudBaseUrl(name) {
9579
+ return `https://${name}.hasna.xyz`;
9580
+ }
9581
+ function clientTransportEnvKeys(name) {
9582
+ const token = envToken(name);
9583
+ return {
9584
+ modeKeys: [
9585
+ `HASNA_${token}_STORAGE_MODE`,
9586
+ `HASNA_${token}_MODE`,
9587
+ `${token}_STORAGE_MODE`,
9588
+ `${token}_MODE`
9589
+ ],
9590
+ apiUrlKeys: [`HASNA_${token}_API_URL`, `${token}_API_URL`],
9591
+ apiKeyKeys: [
9592
+ `HASNA_${token}_API_KEY`,
9593
+ `${token}_API_KEY`,
9594
+ `HASNA_${token}_API_TOKEN`,
9595
+ `${token}_API_TOKEN`
9596
+ ]
9597
+ };
9598
+ }
9599
+ function firstEnv(env, keys) {
9600
+ for (const key of keys) {
9601
+ const value = env[key]?.trim();
9602
+ if (value)
9603
+ return { key, value };
9604
+ }
9605
+ return null;
9606
+ }
9607
+ function toV1BaseUrl(apiUrl) {
9608
+ const url = new URL(apiUrl);
9609
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
9610
+ throw new Error("API URL must use http or https.");
9611
+ }
9612
+ let path = url.pathname.replace(/\/+$/, "");
9613
+ if (path.endsWith("/v1"))
9614
+ path = path.slice(0, -"/v1".length);
9615
+ url.pathname = `${path}/v1`;
9616
+ url.search = "";
9617
+ url.hash = "";
9618
+ return url.toString().replace(/\/+$/, "");
9619
+ }
9620
+ function resolveClientTransport(name, env = process.env) {
9621
+ const keys = clientTransportEnvKeys(name);
9622
+ const modeHit = firstEnv(env, keys.modeKeys);
9623
+ const urlHit = firstEnv(env, keys.apiUrlKeys);
9624
+ const keyHit = firstEnv(env, keys.apiKeyKeys);
9625
+ let mode = "local";
9626
+ let deprecatedAlias = null;
9627
+ let modeSource = "default";
9628
+ const warnings = [];
9629
+ if (modeHit) {
9630
+ const normalized = normalizeStorageMode(modeHit.value);
9631
+ mode = normalized.mode;
9632
+ deprecatedAlias = normalized.deprecatedAlias;
9633
+ modeSource = modeHit.key;
9634
+ if (deprecatedAlias) {
9635
+ warnings.push(`Deprecated mode '${deprecatedAlias}' from ${modeHit.key} is treated as 'cloud'. Prefer ${keys.modeKeys[0]}=cloud.`);
9636
+ }
9637
+ }
9638
+ if (mode === "local") {
9639
+ return {
9640
+ transport: "local",
9641
+ mode,
9642
+ deprecatedAlias,
9643
+ modeSource,
9644
+ baseUrl: null,
9645
+ apiUrlSource: null,
9646
+ apiKeyPresent: Boolean(keyHit),
9647
+ apiKeySource: keyHit ? keyHit.key : null,
9648
+ misconfigured: false,
9649
+ warning: warnings.length > 0 ? warnings.join(" ") : null
9650
+ };
9651
+ }
9652
+ if (!keyHit) {
9653
+ warnings.push(`${modeSource}=cloud but no API key is set (${keys.apiKeyKeys[0]}). Refusing to route to cloud; using local store. Set ${keys.apiKeyKeys[0]} to enable the cloud client.`);
9654
+ return {
9655
+ transport: "local",
9656
+ mode,
9657
+ deprecatedAlias,
9658
+ modeSource,
9659
+ baseUrl: null,
9660
+ apiUrlSource: null,
9661
+ apiKeyPresent: false,
9662
+ apiKeySource: null,
9663
+ misconfigured: true,
9664
+ warning: warnings.join(" ")
9665
+ };
9666
+ }
9667
+ const rawUrl = urlHit?.value ?? defaultCloudBaseUrl(name);
9668
+ const apiUrlSource = urlHit ? urlHit.key : "default";
9669
+ let baseUrl;
9670
+ try {
9671
+ baseUrl = toV1BaseUrl(rawUrl);
9672
+ } catch (error) {
9673
+ const message = error instanceof Error ? error.message : String(error);
9674
+ warnings.push(`Invalid API URL from ${apiUrlSource}: ${message}. Using local store.`);
9675
+ return {
9676
+ transport: "local",
9677
+ mode,
9678
+ deprecatedAlias,
9679
+ modeSource,
9680
+ baseUrl: null,
9681
+ apiUrlSource: null,
9682
+ apiKeyPresent: true,
9683
+ apiKeySource: keyHit.key,
9684
+ misconfigured: true,
9685
+ warning: warnings.join(" ")
9686
+ };
9687
+ }
9688
+ return {
9689
+ transport: "cloud-http",
9690
+ mode,
9691
+ deprecatedAlias,
9692
+ modeSource,
9693
+ baseUrl,
9694
+ apiUrlSource,
9695
+ apiKeyPresent: true,
9696
+ apiKeySource: keyHit.key,
9697
+ misconfigured: false,
9698
+ warning: warnings.length > 0 ? warnings.join(" ") : null
9699
+ };
9700
+ }
9701
+
9702
+ class HasnaHttpError extends Error {
9703
+ status;
9704
+ method;
9705
+ path;
9706
+ body;
9707
+ constructor(method, path, status, body) {
9708
+ super(`Hasna cloud request failed: ${method} ${path} -> ${status}`);
9709
+ this.name = "HasnaHttpError";
9710
+ this.status = status;
9711
+ this.method = method;
9712
+ this.path = path;
9713
+ this.body = body;
9714
+ }
9715
+ }
9716
+ var DEFAULT_RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504];
9717
+ var IDEMPOTENT_METHODS = new Set(["GET", "HEAD", "PUT", "DELETE", "OPTIONS"]);
9718
+ function appendQuery(path, query) {
9719
+ if (!query)
9720
+ return path;
9721
+ const params = query instanceof URLSearchParams ? query : new URLSearchParams;
9722
+ if (!(query instanceof URLSearchParams)) {
9723
+ for (const [key, value] of Object.entries(query)) {
9724
+ if (value === null || value === undefined)
9725
+ continue;
9726
+ if (Array.isArray(value)) {
9727
+ for (const v of value)
9728
+ params.append(key, String(v));
9729
+ } else {
9730
+ params.append(key, String(value));
9731
+ }
9732
+ }
9733
+ }
9734
+ const qs = params.toString();
9735
+ if (!qs)
9736
+ return path;
9737
+ return `${path}${path.includes("?") ? "&" : "?"}${qs}`;
9738
+ }
9739
+ var defaultSleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
9740
+ function createHasnaHttpTransport(options) {
9741
+ const fetchImpl = options.fetchImpl ?? ((input, init) => fetch(input, init));
9742
+ const base = options.baseUrl.replace(/\/+$/, "");
9743
+ const timeoutMs = options.timeoutMs ?? 30000;
9744
+ const sleep = options.sleepImpl ?? defaultSleep;
9745
+ const defaultRetry = options.retry;
9746
+ function resolveRetry(callRetry) {
9747
+ const chosen = callRetry !== undefined ? callRetry : defaultRetry;
9748
+ if (chosen === false)
9749
+ return null;
9750
+ const r = chosen ?? {};
9751
+ return {
9752
+ retries: r.retries ?? 2,
9753
+ baseDelayMs: r.baseDelayMs ?? 200,
9754
+ maxDelayMs: r.maxDelayMs ?? 2000,
9755
+ retryStatuses: r.retryStatuses ?? [...DEFAULT_RETRY_STATUSES]
9756
+ };
9757
+ }
9758
+ async function once2(method, rel, url, body, opts) {
9759
+ const headers = {
9760
+ "x-api-key": options.apiKey,
9761
+ Authorization: `Bearer ${options.apiKey}`,
9762
+ Accept: "application/json",
9763
+ ...options.headers ?? {},
9764
+ ...opts.headers ?? {}
9765
+ };
9766
+ if (opts.idempotencyKey)
9767
+ headers["Idempotency-Key"] = opts.idempotencyKey;
9768
+ const init = { method, headers };
9769
+ if (body !== undefined) {
9770
+ headers["Content-Type"] = "application/json";
9771
+ init.body = JSON.stringify(body);
9772
+ }
9773
+ const controller = new AbortController;
9774
+ const onAbort = () => controller.abort();
9775
+ if (opts.signal) {
9776
+ if (opts.signal.aborted)
9777
+ controller.abort();
9778
+ else
9779
+ opts.signal.addEventListener("abort", onAbort, { once: true });
9780
+ }
9781
+ const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? timeoutMs);
9782
+ init.signal = controller.signal;
9783
+ let response;
9784
+ try {
9785
+ response = await fetchImpl(url, init);
9786
+ } catch (error) {
9787
+ const err = error instanceof Error ? error : new Error(String(error));
9788
+ if (opts.signal?.aborted)
9789
+ return { ok: false, retryable: false, error: err };
9790
+ return { ok: false, retryable: true, error: err };
9791
+ } finally {
9792
+ clearTimeout(timer);
9793
+ if (opts.signal)
9794
+ opts.signal.removeEventListener("abort", onAbort);
9795
+ }
9796
+ const text = await response.text();
9797
+ let parsed = undefined;
9798
+ if (text.length > 0) {
9799
+ try {
9800
+ parsed = JSON.parse(text);
9801
+ } catch {
9802
+ parsed = text;
9803
+ }
9804
+ }
9805
+ if (!response.ok) {
9806
+ const retry = resolveRetry(opts.retry);
9807
+ const retryable = retry ? retry.retryStatuses.includes(response.status) : false;
9808
+ return { ok: false, retryable, error: new HasnaHttpError(method, rel, response.status, parsed) };
9809
+ }
9810
+ return { ok: true, value: parsed };
9811
+ }
9812
+ async function request(method, path, body, opts = {}) {
9813
+ const upper = method.toUpperCase();
9814
+ const rel = appendQuery(path.startsWith("/") ? path : `/${path}`, opts.query);
9815
+ const url = `${base}${rel}`;
9816
+ const retry = resolveRetry(opts.retry);
9817
+ const methodRetryable = IDEMPOTENT_METHODS.has(upper) || Boolean(opts.idempotencyKey);
9818
+ const maxAttempts = retry && methodRetryable ? retry.retries + 1 : 1;
9819
+ let last = null;
9820
+ for (let attempt = 1;attempt <= maxAttempts; attempt++) {
9821
+ const result = await once2(upper, rel, url, body, opts);
9822
+ if (result.ok)
9823
+ return result.value;
9824
+ last = result;
9825
+ const canRetry = retry !== null && methodRetryable && result.retryable && attempt < maxAttempts;
9826
+ if (!canRetry)
9827
+ break;
9828
+ const backoff = Math.min(retry.maxDelayMs, retry.baseDelayMs * 2 ** (attempt - 1));
9829
+ const jitter = Math.floor(Math.random() * (backoff / 2 + 1));
9830
+ await sleep(backoff + jitter);
9831
+ }
9832
+ throw last.error;
9833
+ }
9834
+ return {
9835
+ baseUrl: base,
9836
+ request,
9837
+ get: (path, opts) => request("GET", path, undefined, opts),
9838
+ post: (path, body, opts) => request("POST", path, body, opts),
9839
+ put: (path, body, opts) => request("PUT", path, body, opts),
9840
+ patch: (path, body, opts) => request("PATCH", path, body, opts),
9841
+ del: (path, body, opts) => request("DELETE", path, body, opts)
9842
+ };
9843
+ }
9844
+ function createClientTransport(name, env = process.env, overrides) {
9845
+ const resolution = resolveClientTransport(name, env);
9846
+ if (resolution.misconfigured) {
9847
+ throw new Error(resolution.warning ?? `Client for '${name}' is misconfigured for cloud mode.`);
9848
+ }
9849
+ if (resolution.transport === "local" || !resolution.baseUrl) {
9850
+ return { transport: "local", client: null, resolution };
9851
+ }
9852
+ const keys = clientTransportEnvKeys(name);
9853
+ const apiKey = firstEnv(env, keys.apiKeyKeys)?.value;
9854
+ if (!apiKey) {
9855
+ throw new Error(`Client for '${name}' resolved to cloud-http without an API key.`);
9856
+ }
9857
+ return {
9858
+ transport: "cloud-http",
9859
+ client: createHasnaHttpTransport({
9860
+ name,
9861
+ baseUrl: resolution.baseUrl,
9862
+ apiKey,
9863
+ ...overrides?.fetchImpl ? { fetchImpl: overrides.fetchImpl } : {},
9864
+ ...overrides?.headers ? { headers: overrides.headers } : {},
9865
+ ...overrides?.timeoutMs ? { timeoutMs: overrides.timeoutMs } : {},
9866
+ ...overrides?.retry !== undefined ? { retry: overrides.retry } : {},
9867
+ ...overrides?.sleepImpl ? { sleepImpl: overrides.sleepImpl } : {}
9868
+ }),
9869
+ resolution
9870
+ };
9871
+ }
9872
+
9873
+ // src/lib/cloud/storage.ts
9874
+ function resourcePath(resource) {
9875
+ const trimmed = resource.replace(/^\/+|\/+$/g, "");
9876
+ if (!trimmed)
9877
+ throw new Error("resource must be a non-empty path segment");
9878
+ return `/${trimmed}`;
9879
+ }
9880
+ function entityPath(resource, id) {
9881
+ if (id === undefined || id === null || `${id}`.length === 0) {
9882
+ throw new Error("id must be a non-empty string");
9883
+ }
9884
+ return `${resourcePath(resource)}/${encodeURIComponent(String(id))}`;
9885
+ }
9886
+ function newIdempotencyKey() {
9887
+ const g = globalThis;
9888
+ if (g.crypto?.randomUUID)
9889
+ return g.crypto.randomUUID();
9890
+ return `idmp_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 12)}`;
9891
+ }
9892
+ function extractItems(raw) {
9893
+ if (Array.isArray(raw))
9894
+ return raw;
9895
+ if (raw && typeof raw === "object") {
9896
+ const obj = raw;
9897
+ for (const key of ["items", "data", "results", "rows", "records"]) {
9898
+ if (Array.isArray(obj[key]))
9899
+ return obj[key];
9900
+ }
9901
+ }
9902
+ return [];
9903
+ }
9904
+ function extractTotal(raw) {
9905
+ if (raw && typeof raw === "object") {
9906
+ const obj = raw;
9907
+ for (const key of ["total", "count", "totalCount", "total_count"]) {
9908
+ if (typeof obj[key] === "number")
9909
+ return obj[key];
9910
+ }
9911
+ }
9912
+ return null;
9913
+ }
9914
+ function extractCursor(raw) {
9915
+ if (raw && typeof raw === "object") {
9916
+ const obj = raw;
9917
+ for (const key of ["cursor", "nextCursor", "next_cursor", "next"]) {
9918
+ if (typeof obj[key] === "string")
9919
+ return obj[key];
9920
+ }
9921
+ }
9922
+ return null;
9923
+ }
9924
+ function createHasnaStorageClient(name, transport) {
9925
+ return {
9926
+ name,
9927
+ baseUrl: transport.baseUrl,
9928
+ transport,
9929
+ async list(resource, options = {}) {
9930
+ const raw = await transport.get(resourcePath(resource), options);
9931
+ return {
9932
+ items: extractItems(raw),
9933
+ total: extractTotal(raw),
9934
+ cursor: extractCursor(raw),
9935
+ raw
9936
+ };
9937
+ },
9938
+ async get(resource, id, options = {}) {
9939
+ try {
9940
+ return await transport.get(entityPath(resource, id), options);
9941
+ } catch (error) {
9942
+ if (error instanceof HasnaHttpError && error.status === 404)
9943
+ return null;
9944
+ throw error;
9945
+ }
9946
+ },
9947
+ async create(resource, body, options = {}) {
9948
+ const { idempotencyKey, ...rest } = options;
9949
+ return transport.post(resourcePath(resource), body, {
9950
+ ...rest,
9951
+ idempotencyKey: idempotencyKey ?? newIdempotencyKey()
9952
+ });
9953
+ },
9954
+ async update(resource, id, patch, options = {}) {
9955
+ const { method = "PATCH", idempotencyKey, ...rest } = options;
9956
+ const call = method === "PUT" ? transport.put : transport.patch;
9957
+ return call(entityPath(resource, id), patch, { ...rest, ...idempotencyKey ? { idempotencyKey } : {} });
9958
+ },
9959
+ async delete(resource, id, options = {}) {
9960
+ try {
9961
+ await transport.del(entityPath(resource, id), undefined, options);
9962
+ } catch (error) {
9963
+ if (error instanceof HasnaHttpError && error.status === 404)
9964
+ return;
9965
+ throw error;
9966
+ }
9967
+ }
9968
+ };
9969
+ }
9970
+
9971
+ // src/lib/cloud/resolve.ts
9972
+ function firstValue(env, keys) {
9973
+ for (const key of keys) {
9974
+ const value = env[key]?.trim();
9975
+ if (value)
9976
+ return value;
9977
+ }
9978
+ return;
9979
+ }
9980
+ function resolveCloudStorage(name, env = process.env) {
9981
+ const token = envToken(name);
9982
+ const modeKeys = [`HASNA_${token}_STORAGE_MODE`, `HASNA_${token}_MODE`, `${token}_STORAGE_MODE`, `${token}_MODE`];
9983
+ const apiUrlKeys = [`HASNA_${token}_API_URL`, `${token}_API_URL`];
9984
+ const apiKeyKeys = [
9985
+ `HASNA_${token}_API_KEY`,
9986
+ `${token}_API_KEY`,
9987
+ `HASNA_${token}_API_TOKEN`,
9988
+ `${token}_API_TOKEN`
9989
+ ];
9990
+ const explicitMode = firstValue(env, modeKeys);
9991
+ if (explicitMode && normalizeStorageMode(explicitMode).mode === "local") {
9992
+ return { transport: "local", client: null };
9993
+ }
9994
+ const apiUrl = firstValue(env, apiUrlKeys);
9995
+ const apiKey = firstValue(env, apiKeyKeys);
9996
+ if (!apiUrl || !apiKey) {
9997
+ return { transport: "local", client: null };
9998
+ }
9999
+ const cloudEnv = { ...env, [`HASNA_${token}_STORAGE_MODE`]: "self_hosted" };
10000
+ const wired = createClientTransport(name, cloudEnv);
10001
+ if (wired.transport !== "cloud-http") {
10002
+ return { transport: "local", client: null };
10003
+ }
10004
+ return {
10005
+ transport: "cloud-http",
10006
+ client: createHasnaStorageClient(name, wired.client),
10007
+ baseUrl: wired.client.baseUrl
10008
+ };
10009
+ }
10010
+
10011
+ // src/lib/store/index.ts
10012
+ class CloudUnsupportedError extends Error {
10013
+ constructor(operation) {
10014
+ super(`operation not supported over the hosted OpenLoops API: ${operation}. ` + `Run it on a machine using local storage, or unset HASNA_LOOPS_API_URL/HASNA_LOOPS_API_KEY.`);
10015
+ this.name = "CloudUnsupportedError";
10016
+ }
10017
+ }
10018
+
10019
+ class LocalStore {
10020
+ transport = "local";
10021
+ store;
10022
+ constructor(store = new Store) {
10023
+ this.store = store;
10024
+ }
10025
+ get raw() {
10026
+ return this.store;
10027
+ }
10028
+ async close() {
10029
+ this.store.close();
10030
+ }
10031
+ async createLoop(input, from) {
10032
+ return from ? this.store.createLoop(input, from) : this.store.createLoop(input);
10033
+ }
10034
+ async getLoop(id) {
10035
+ return this.store.getLoop(id);
10036
+ }
10037
+ async findLoopByName(name) {
10038
+ return this.store.findLoopByName(name);
10039
+ }
10040
+ async requireLoop(idOrName) {
10041
+ return this.store.requireLoop(idOrName);
10042
+ }
10043
+ async requireUniqueLoop(idOrName) {
10044
+ return this.store.requireUniqueLoop(idOrName);
10045
+ }
10046
+ async listLoops(opts = {}) {
10047
+ return this.store.listLoops(opts);
10048
+ }
10049
+ async countLoops(status, opts = {}) {
10050
+ return this.store.countLoops(status, opts);
10051
+ }
10052
+ async updateLoop(id, patch) {
10053
+ return this.store.updateLoop(id, patch);
10054
+ }
10055
+ async renameLoop(id, name) {
10056
+ return this.store.renameLoop(id, name);
10057
+ }
10058
+ async archiveLoop(idOrName) {
10059
+ return this.store.archiveLoop(idOrName);
10060
+ }
10061
+ async unarchiveLoop(idOrName) {
10062
+ return this.store.unarchiveLoop(idOrName);
10063
+ }
10064
+ async deleteLoop(idOrName) {
10065
+ return this.store.deleteLoop(idOrName);
10066
+ }
10067
+ async createWorkflow(input) {
10068
+ return this.store.createWorkflow(input);
10069
+ }
10070
+ async getWorkflow(id) {
10071
+ return this.store.getWorkflow(id);
10072
+ }
10073
+ async findWorkflowByName(name) {
10074
+ return this.store.findWorkflowByName(name);
10075
+ }
10076
+ async requireWorkflow(idOrName) {
10077
+ return this.store.requireWorkflow(idOrName);
10078
+ }
10079
+ async listWorkflows(opts = {}) {
10080
+ return this.store.listWorkflows(opts);
10081
+ }
10082
+ async countWorkflows(opts = {}) {
10083
+ return this.store.countWorkflows(opts);
10084
+ }
10085
+ async archiveWorkflow(idOrName) {
10086
+ return this.store.archiveWorkflow(idOrName);
10087
+ }
10088
+ async getWorkflowRun(id) {
10089
+ return this.store.getWorkflowRun(id);
10090
+ }
10091
+ async requireWorkflowRun(id) {
10092
+ return this.store.requireWorkflowRun(id);
10093
+ }
10094
+ async listWorkflowRuns(opts = {}) {
10095
+ return this.store.listWorkflowRuns(opts);
10096
+ }
10097
+ async listWorkflowStepRuns(workflowRunId) {
10098
+ return this.store.listWorkflowStepRuns(workflowRunId);
10099
+ }
10100
+ async listWorkflowEvents(workflowRunId, limit) {
10101
+ return limit === undefined ? this.store.listWorkflowEvents(workflowRunId) : this.store.listWorkflowEvents(workflowRunId, limit);
10102
+ }
10103
+ async recoverWorkflowRun(workflowRunId, reason) {
10104
+ return reason === undefined ? this.store.recoverWorkflowRun(workflowRunId) : this.store.recoverWorkflowRun(workflowRunId, reason);
10105
+ }
10106
+ async cancelWorkflowRun(workflowRunId, reason) {
10107
+ return reason === undefined ? this.store.cancelWorkflowRun(workflowRunId) : this.store.cancelWorkflowRun(workflowRunId, reason);
10108
+ }
10109
+ async listWorkflowWorkItems(opts = {}) {
10110
+ return this.store.listWorkflowWorkItems(opts);
10111
+ }
10112
+ async getWorkflowWorkItem(id) {
10113
+ return this.store.getWorkflowWorkItem(id);
10114
+ }
10115
+ async requeueWorkflowWorkItem(id, patch = {}) {
10116
+ return this.store.requeueWorkflowWorkItem(id, patch);
10117
+ }
10118
+ async listWorkflowInvocations(opts = {}) {
10119
+ return this.store.listWorkflowInvocations(opts);
10120
+ }
10121
+ async getWorkflowInvocation(id) {
10122
+ return this.store.getWorkflowInvocation(id);
10123
+ }
10124
+ async getGoal(id) {
10125
+ return this.store.getGoal(id);
10126
+ }
10127
+ async findGoalByLoop(idOrName) {
10128
+ return this.store.findGoalByLoop(idOrName);
10129
+ }
10130
+ async findGoalByRunId(id) {
10131
+ return this.store.findGoalByRunId(id);
10132
+ }
10133
+ async listGoals(opts = {}) {
10134
+ return this.store.listGoals(opts);
10135
+ }
10136
+ async listGoalPlanNodes(goalIdOrPlanId) {
10137
+ return this.store.listGoalPlanNodes(goalIdOrPlanId);
10138
+ }
10139
+ async listGoalRuns(opts = {}) {
10140
+ return this.store.listGoalRuns(opts);
10141
+ }
10142
+ async listRuns(opts = {}) {
10143
+ return this.store.listRuns(opts);
10144
+ }
10145
+ async getRun(id) {
10146
+ return this.store.getRun(id);
10147
+ }
10148
+ async writeRunReceipt(input) {
10149
+ return this.store.writeRunReceipt(input);
10150
+ }
10151
+ async getRunReceipt(runId) {
10152
+ return this.store.getRunReceipt(runId);
10153
+ }
10154
+ async listRunReceipts(opts = {}) {
10155
+ return this.store.listRunReceipts(opts);
10156
+ }
10157
+ async pruneHistory(opts) {
10158
+ return this.store.pruneHistory(opts);
10159
+ }
10160
+ }
10161
+ function clean(query) {
10162
+ const out = {};
10163
+ for (const [key, value] of Object.entries(query)) {
10164
+ if (value !== undefined && value !== null && value !== "")
10165
+ out[key] = value;
10166
+ }
10167
+ return out;
10168
+ }
10169
+ function pickArray(raw, key, fallback = []) {
10170
+ if (raw && typeof raw === "object" && Array.isArray(raw[key])) {
10171
+ return raw[key];
10172
+ }
10173
+ return fallback;
10174
+ }
10175
+ function pickObject(raw, key) {
10176
+ if (raw && typeof raw === "object") {
10177
+ const value = raw[key];
10178
+ return value == null ? undefined : value;
10179
+ }
10180
+ return raw == null ? undefined : raw;
10181
+ }
10182
+
10183
+ class ApiStore {
10184
+ client;
10185
+ baseUrl;
10186
+ transport = "cloud-http";
10187
+ constructor(client, baseUrl) {
10188
+ this.client = client;
10189
+ this.baseUrl = baseUrl;
10190
+ }
10191
+ get t() {
10192
+ return this.client.transport;
10193
+ }
10194
+ async close() {}
10195
+ async createLoop(input) {
10196
+ return pickObject(await this.t.post("/loops", input), "loop");
10197
+ }
10198
+ async getLoop(id) {
8179
10199
  try {
8180
- skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
8181
- daemonLeaseId: deps.daemonLeaseId
8182
- });
8183
- } catch (error) {
8184
- if (deps.daemonLeaseId && isDaemonLeaseLost(error))
8185
- return;
8186
- throw error;
10200
+ return pickObject(await this.t.get(`/loops/${encodeURIComponent(id)}`), "loop");
10201
+ } catch {
10202
+ return;
8187
10203
  }
8188
- advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
8189
- deps.onRun?.(skipped);
8190
- return skipped;
8191
10204
  }
8192
- let claim;
8193
- try {
8194
- claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
8195
- } catch (error) {
8196
- if (deps.daemonLeaseId && isDaemonLeaseLost(error))
10205
+ async findLoopByName(name) {
10206
+ const matches = (await this.listLoops({ name, limit: 1000 })).filter((loop) => loop.name === name);
10207
+ return matches[0];
10208
+ }
10209
+ async requireLoop(idOrName) {
10210
+ const loop = await this.resolveLoop(idOrName);
10211
+ if (!loop)
10212
+ throw new LoopNotFoundError(idOrName);
10213
+ return loop;
10214
+ }
10215
+ async requireUniqueLoop(idOrName) {
10216
+ const byId = await this.getLoop(idOrName);
10217
+ if (byId)
10218
+ return byId;
10219
+ const matches = (await this.listLoops({ name: idOrName, limit: 1000 })).filter((loop) => loop.name === idOrName);
10220
+ if (matches.length === 0)
10221
+ throw new LoopNotFoundError(idOrName);
10222
+ if (matches.length === 1)
10223
+ return matches[0];
10224
+ const active = matches.filter((loop) => !loop.archivedAt);
10225
+ if (active.length !== 1)
10226
+ throw new AmbiguousNameError(idOrName);
10227
+ return active[0];
10228
+ }
10229
+ async resolveLoop(idOrName) {
10230
+ const byId = await this.getLoop(idOrName);
10231
+ if (byId)
10232
+ return byId;
10233
+ const matches = (await this.listLoops({ name: idOrName, limit: 1000 })).filter((loop) => loop.name === idOrName);
10234
+ if (matches.length === 0)
8197
10235
  return;
8198
- throw error;
10236
+ if (matches.length === 1)
10237
+ return matches[0];
10238
+ const active = matches.filter((loop) => !loop.archivedAt);
10239
+ if (active.length === 1)
10240
+ return active[0];
10241
+ throw new AmbiguousNameError(idOrName);
8199
10242
  }
8200
- if (!claim) {
8201
- repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
8202
- return;
10243
+ async listLoops(opts = {}) {
10244
+ const raw = await this.t.get("/loops", { query: clean({ ...opts }) });
10245
+ return pickArray(raw, "loops");
8203
10246
  }
8204
- deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
8205
- deps.onRun?.(claim.run);
8206
- const finalRun = await executeClaimedRun({
8207
- store: deps.store,
8208
- runnerId: deps.runnerId,
8209
- loop: claim.loop,
8210
- run: claim.run,
8211
- now: deps.now,
8212
- execute: deps.execute,
8213
- beforeFinalize: deps.beforeFinalize,
8214
- daemonLeaseId: deps.daemonLeaseId,
8215
- onError: deps.onError
8216
- });
8217
- advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
8218
- deps.onRun?.(finalRun);
8219
- return finalRun;
8220
- }
8221
- function claimSlot(deps, loop, scheduledFor) {
8222
- const now = deps.now?.() ?? new Date;
8223
- deps.beforeRun?.(loop, scheduledFor);
8224
- if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
8225
- if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
10247
+ async countLoops(status, opts = {}) {
10248
+ const raw = await this.t.get("/loops/count", { query: clean({ status, ...opts }) });
10249
+ return Number(pickObject(raw, "count") ?? 0);
10250
+ }
10251
+ async updateLoop(id, patch) {
10252
+ return pickObject(await this.t.patch(`/loops/${encodeURIComponent(id)}`, patch), "loop");
10253
+ }
10254
+ async renameLoop(id, name) {
10255
+ return pickObject(await this.t.post(`/loops/${encodeURIComponent(id)}/rename`, { name }), "loop");
10256
+ }
10257
+ async archiveLoop(idOrName) {
10258
+ const loop = await this.requireLoop(idOrName);
10259
+ return pickObject(await this.t.post(`/loops/${encodeURIComponent(loop.id)}/archive`), "loop");
10260
+ }
10261
+ async unarchiveLoop(idOrName) {
10262
+ const loop = await this.requireLoop(idOrName);
10263
+ return pickObject(await this.t.post(`/loops/${encodeURIComponent(loop.id)}/unarchive`), "loop");
10264
+ }
10265
+ async deleteLoop(idOrName) {
10266
+ const loop = await this.resolveLoop(idOrName);
10267
+ if (!loop)
10268
+ return false;
10269
+ const raw = await this.t.request("DELETE", `/loops/${encodeURIComponent(loop.id)}`);
10270
+ return Boolean(pickObject(raw, "deleted") ?? true);
10271
+ }
10272
+ async createWorkflow(input) {
10273
+ return pickObject(await this.t.post("/workflows", input), "workflow");
10274
+ }
10275
+ async getWorkflow(id) {
10276
+ try {
10277
+ return pickObject(await this.t.get(`/workflows/${encodeURIComponent(id)}`), "workflow");
10278
+ } catch {
8226
10279
  return;
8227
- let skipped;
10280
+ }
10281
+ }
10282
+ async findWorkflowByName(name) {
10283
+ const matches = (await this.listWorkflows({ limit: 1000 })).filter((wf) => wf.name === name);
10284
+ return matches[0];
10285
+ }
10286
+ async requireWorkflow(idOrName) {
10287
+ const byId = await this.getWorkflow(idOrName);
10288
+ if (byId)
10289
+ return byId;
10290
+ const byName = await this.findWorkflowByName(idOrName);
10291
+ if (byName)
10292
+ return byName;
10293
+ throw new Error(`workflow not found: ${idOrName}`);
10294
+ }
10295
+ async listWorkflows(opts = {}) {
10296
+ const raw = await this.t.get("/workflows", { query: clean({ ...opts }) });
10297
+ return pickArray(raw, "workflows");
10298
+ }
10299
+ async countWorkflows(opts = {}) {
10300
+ const raw = await this.t.get("/workflows/count", { query: clean({ ...opts }) });
10301
+ return Number(pickObject(raw, "count") ?? 0);
10302
+ }
10303
+ async archiveWorkflow(idOrName) {
10304
+ const wf = await this.requireWorkflow(idOrName);
10305
+ return pickObject(await this.t.post(`/workflows/${encodeURIComponent(wf.id)}/archive`), "workflow");
10306
+ }
10307
+ async getWorkflowRun(id) {
8228
10308
  try {
8229
- skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
8230
- daemonLeaseId: deps.daemonLeaseId
8231
- });
8232
- } catch (error) {
8233
- if (deps.daemonLeaseId && isDaemonLeaseLost(error))
8234
- return;
8235
- throw error;
10309
+ return pickObject(await this.t.get(`/workflow-runs/${encodeURIComponent(id)}`), "workflowRun");
10310
+ } catch {
10311
+ return;
8236
10312
  }
8237
- advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
8238
- deps.onRun?.(skipped);
8239
- return skipped;
8240
10313
  }
8241
- let claim;
8242
- try {
8243
- claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
8244
- } catch (error) {
8245
- if (deps.daemonLeaseId && isDaemonLeaseLost(error))
10314
+ async requireWorkflowRun(id) {
10315
+ const run = await this.getWorkflowRun(id);
10316
+ if (!run)
10317
+ throw new Error(`workflow run not found: ${id}`);
10318
+ return run;
10319
+ }
10320
+ async listWorkflowRuns(opts = {}) {
10321
+ const raw = await this.t.get("/workflow-runs", { query: clean({ ...opts }) });
10322
+ return pickArray(raw, "workflowRuns");
10323
+ }
10324
+ async listWorkflowStepRuns(workflowRunId) {
10325
+ const raw = await this.t.get(`/workflow-runs/${encodeURIComponent(workflowRunId)}/steps`);
10326
+ return pickArray(raw, "steps");
10327
+ }
10328
+ async listWorkflowEvents(workflowRunId, limit) {
10329
+ const raw = await this.t.get(`/workflow-runs/${encodeURIComponent(workflowRunId)}/events`, { query: clean({ limit }) });
10330
+ return pickArray(raw, "events");
10331
+ }
10332
+ async recoverWorkflowRun(workflowRunId, reason) {
10333
+ const raw = await this.t.post(`/workflow-runs/${encodeURIComponent(workflowRunId)}/recover`, { reason });
10334
+ return {
10335
+ run: pickObject(raw, "workflowRun"),
10336
+ recoveredSteps: pickArray(raw, "recoveredSteps")
10337
+ };
10338
+ }
10339
+ async cancelWorkflowRun(_workflowRunId, _reason) {
10340
+ throw new CloudUnsupportedError("workflows cancel");
10341
+ }
10342
+ async listWorkflowWorkItems(opts = {}) {
10343
+ const raw = await this.t.get("/work-items", { query: clean({ ...opts }) });
10344
+ return pickArray(raw, "workItems");
10345
+ }
10346
+ async getWorkflowWorkItem(id) {
10347
+ try {
10348
+ return pickObject(await this.t.get(`/work-items/${encodeURIComponent(id)}`), "workItem");
10349
+ } catch {
8246
10350
  return;
8247
- throw error;
10351
+ }
8248
10352
  }
8249
- if (!claim) {
8250
- repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
8251
- return;
10353
+ async requeueWorkflowWorkItem(_id, _patch = {}) {
10354
+ throw new CloudUnsupportedError("routes requeue");
8252
10355
  }
8253
- deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
8254
- deps.onRun?.(claim.run);
8255
- return claim;
8256
- }
8257
- function recoverAndExpire(deps, now) {
8258
- const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
8259
- const recoveredByLoop = new Map;
8260
- for (const run of recovered) {
8261
- recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
10356
+ async listWorkflowInvocations(opts = {}) {
10357
+ const raw = await this.t.get("/invocations", { query: clean({ ...opts }) });
10358
+ return pickArray(raw, "invocations");
8262
10359
  }
8263
- for (const runs of recoveredByLoop.values()) {
8264
- const loop = deps.store.getLoop(runs[0].loopId);
8265
- if (!loop)
8266
- continue;
8267
- const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
8268
- if (retryable) {
8269
- advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
8270
- continue;
10360
+ async getWorkflowInvocation(id) {
10361
+ try {
10362
+ return pickObject(await this.t.get(`/invocations/${encodeURIComponent(id)}`), "invocation");
10363
+ } catch {
10364
+ return;
8271
10365
  }
8272
- for (const run of runs) {
8273
- const current = deps.store.getLoop(run.loopId);
8274
- if (current) {
8275
- advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
8276
- }
10366
+ }
10367
+ async getGoal(id) {
10368
+ try {
10369
+ return pickObject(await this.t.get(`/goals/${encodeURIComponent(id)}`), "goal");
10370
+ } catch {
10371
+ return;
8277
10372
  }
8278
10373
  }
8279
- const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
8280
- return { recovered, expired };
8281
- }
8282
- function claimDueRuns(deps) {
8283
- const now = deps.now?.() ?? new Date;
8284
- const { recovered, expired } = recoverAndExpire(deps, now);
8285
- const claims = [];
8286
- const claimed = [];
8287
- const skipped = [];
8288
- const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
8289
- if (maxClaims === 0)
8290
- return { claims, claimed, completed: [], skipped, recovered, expired };
8291
- const laneLimits = deps.laneLimits;
8292
- const laneClaims = { command: 0, agent: 0 };
8293
- const laneCap = (lane) => laneLimits === undefined ? Number.POSITIVE_INFINITY : Math.max(0, laneLimits[lane] ?? Number.POSITIVE_INFINITY);
8294
- const laneFull = (lane) => laneClaims[lane] >= laneCap(lane);
8295
- for (const loop of deps.store.dueLoops(now)) {
8296
- if (claims.length >= maxClaims)
8297
- break;
8298
- const lane = loopLane(loop);
8299
- if (laneFull(lane))
8300
- continue;
8301
- const plan = dueSlots(loop, now);
8302
- let loopSkips = 0;
8303
- for (const slot of plan.slots) {
8304
- if (claims.length >= maxClaims)
8305
- break;
8306
- if (laneFull(lane))
8307
- break;
8308
- if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
8309
- break;
8310
- const run = claimSlot(deps, loop, slot);
8311
- if (!run)
8312
- continue;
8313
- if ("loop" in run) {
8314
- claims.push(run);
8315
- claimed.push(run.run);
8316
- laneClaims[lane] += 1;
8317
- } else if (run.status === "skipped") {
8318
- skipped.push(run);
8319
- loopSkips += 1;
8320
- }
10374
+ async findGoalByLoop(idOrName) {
10375
+ try {
10376
+ return pickObject(await this.t.get("/goals", { query: clean({ loop: idOrName }) }), "goal");
10377
+ } catch {
10378
+ return;
8321
10379
  }
8322
10380
  }
8323
- return { claims, claimed, completed: [], skipped, recovered, expired };
8324
- }
8325
- async function tick(deps) {
8326
- const now = deps.now?.() ?? new Date;
8327
- const { recovered, expired } = recoverAndExpire(deps, now);
8328
- const claimed = [];
8329
- const completed = [];
8330
- const skipped = [];
8331
- for (const loop of deps.store.dueLoops(now)) {
8332
- const plan = dueSlots(loop, now);
8333
- let loopSkips = 0;
8334
- for (const slot of plan.slots) {
8335
- if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
8336
- break;
8337
- const run = await runSlot(deps, loop, slot);
8338
- if (!run)
8339
- continue;
8340
- if (run.status === "running")
8341
- claimed.push(run);
8342
- else if (run.status === "skipped") {
8343
- skipped.push(run);
8344
- loopSkips += 1;
8345
- } else
8346
- completed.push(run);
8347
- if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
8348
- break;
10381
+ async findGoalByRunId(id) {
10382
+ try {
10383
+ return pickObject(await this.t.get("/goals", { query: clean({ runId: id }) }), "goal");
10384
+ } catch {
10385
+ return;
8349
10386
  }
8350
10387
  }
8351
- return { claimed, completed, skipped, recovered, expired };
10388
+ async listGoals(opts = {}) {
10389
+ const raw = await this.t.get("/goals", { query: clean({ ...opts }) });
10390
+ return pickArray(raw, "goals");
10391
+ }
10392
+ async listGoalPlanNodes(goalIdOrPlanId) {
10393
+ const raw = await this.t.get(`/goals/${encodeURIComponent(goalIdOrPlanId)}/plan-nodes`);
10394
+ return pickArray(raw, "nodes");
10395
+ }
10396
+ async listGoalRuns(opts = {}) {
10397
+ const raw = await this.t.get("/goal-runs", { query: clean({ ...opts }) });
10398
+ return pickArray(raw, "goalRuns");
10399
+ }
10400
+ async listRuns(opts = {}) {
10401
+ const raw = await this.t.get("/runs", { query: clean({ ...opts, showOutput: true }) });
10402
+ return pickArray(raw, "runs");
10403
+ }
10404
+ async getRun(id) {
10405
+ try {
10406
+ return pickObject(await this.t.get(`/runs/${encodeURIComponent(id)}`, { query: { showOutput: true } }), "run");
10407
+ } catch {
10408
+ return;
10409
+ }
10410
+ }
10411
+ async writeRunReceipt(input) {
10412
+ return pickObject(await this.t.post("/receipts", input), "receipt");
10413
+ }
10414
+ async getRunReceipt(runId) {
10415
+ try {
10416
+ return pickObject(await this.t.get(`/receipts/${encodeURIComponent(runId)}`), "receipt");
10417
+ } catch {
10418
+ return;
10419
+ }
10420
+ }
10421
+ async listRunReceipts(opts = {}) {
10422
+ const raw = await this.t.get("/receipts", { query: clean({ ...opts }) });
10423
+ return pickArray(raw, "receipts");
10424
+ }
10425
+ async pruneHistory(opts) {
10426
+ const raw = await this.t.post("/history/prune", {
10427
+ maxAgeDays: opts.maxAgeDays,
10428
+ keepPerLoop: opts.keepPerLoop,
10429
+ dryRun: opts.dryRun
10430
+ });
10431
+ return pickObject(raw, "history");
10432
+ }
10433
+ }
10434
+ function getStore(env = process.env) {
10435
+ const resolution = resolveCloudStorage("loops", env);
10436
+ if (resolution.transport === "cloud-http")
10437
+ return new ApiStore(resolution.client, resolution.baseUrl);
10438
+ return new LocalStore;
10439
+ }
10440
+ function isCloudStore(env = process.env) {
10441
+ return resolveCloudStorage("loops", env).transport === "cloud-http";
8352
10442
  }
8353
10443
 
8354
10444
  // src/sdk/index.ts
@@ -8357,10 +10447,16 @@ class LoopsClient {
8357
10447
  ownStore;
8358
10448
  runnerId;
8359
10449
  constructor(opts = {}) {
8360
- this.store = opts.store ?? new Store;
10450
+ this.store = opts.store ? new LocalStore(opts.store) : getStore();
8361
10451
  this.ownStore = !opts.store;
8362
10452
  this.runnerId = opts.runnerId ?? `sdk:${process.pid}`;
8363
10453
  }
10454
+ localRuntime(operation) {
10455
+ if (this.store.transport !== "local") {
10456
+ throw new Error(`loops SDK ${operation} operates on this machine's local runtime and is not available while flipped to the hosted OpenLoops API. ` + `Unset HASNA_LOOPS_API_URL/HASNA_LOOPS_API_KEY (or set HASNA_LOOPS_STORAGE_MODE=local) to run it here.`);
10457
+ }
10458
+ return this.store.raw;
10459
+ }
8364
10460
  create(input) {
8365
10461
  return this.store.createLoop(input);
8366
10462
  }
@@ -8375,11 +10471,12 @@ class LoopsClient {
8375
10471
  get(idOrName) {
8376
10472
  return this.store.requireLoop(idOrName);
8377
10473
  }
8378
- pause(idOrName) {
8379
- return this.store.updateLoop(this.store.requireUniqueLoop(idOrName).id, { status: "paused" });
10474
+ async pause(idOrName) {
10475
+ const loop = await this.store.requireUniqueLoop(idOrName);
10476
+ return this.store.updateLoop(loop.id, { status: "paused" });
8380
10477
  }
8381
- resume(idOrName) {
8382
- const loop = this.store.requireUniqueLoop(idOrName);
10478
+ async resume(idOrName) {
10479
+ const loop = await this.store.requireUniqueLoop(idOrName);
8383
10480
  let nextRunAt = loop.nextRunAt;
8384
10481
  if (!nextRunAt) {
8385
10482
  const now = new Date;
@@ -8387,8 +10484,9 @@ class LoopsClient {
8387
10484
  }
8388
10485
  return this.store.updateLoop(loop.id, { status: "active", nextRunAt });
8389
10486
  }
8390
- stop(idOrName) {
8391
- return this.store.updateLoop(this.store.requireUniqueLoop(idOrName).id, { status: "stopped", nextRunAt: undefined });
10487
+ async stop(idOrName) {
10488
+ const loop = await this.store.requireUniqueLoop(idOrName);
10489
+ return this.store.updateLoop(loop.id, { status: "stopped", nextRunAt: undefined });
8392
10490
  }
8393
10491
  archive(idOrName) {
8394
10492
  return this.store.archiveLoop(idOrName);
@@ -8396,14 +10494,15 @@ class LoopsClient {
8396
10494
  unarchive(idOrName) {
8397
10495
  return this.store.unarchiveLoop(idOrName);
8398
10496
  }
8399
- delete(idOrName) {
8400
- return this.store.deleteLoop(this.store.requireUniqueLoop(idOrName).id);
10497
+ async delete(idOrName) {
10498
+ const loop = await this.store.requireUniqueLoop(idOrName);
10499
+ return this.store.deleteLoop(loop.id);
8401
10500
  }
8402
- runs(idOrName, filters = {}) {
10501
+ async runs(idOrName, filters = {}) {
8403
10502
  let loopId;
8404
10503
  if (idOrName) {
8405
10504
  try {
8406
- loopId = this.get(idOrName).id;
10505
+ loopId = (await this.get(idOrName)).id;
8407
10506
  } catch (error) {
8408
10507
  if (error instanceof LoopNotFoundError)
8409
10508
  return [];
@@ -8412,41 +10511,59 @@ class LoopsClient {
8412
10511
  }
8413
10512
  return this.store.listRuns({ loopId, status: filters.status, limit: filters.limit });
8414
10513
  }
8415
- doctor() {
8416
- return runDoctor(this.store);
10514
+ writeReceipt(input) {
10515
+ return this.store.writeRunReceipt(input);
8417
10516
  }
8418
- health(opts = {}) {
8419
- return buildHealthReport(this.store, opts);
10517
+ receipt(runId) {
10518
+ return this.store.getRunReceipt(runId);
10519
+ }
10520
+ receipts(filters = {}) {
10521
+ return this.store.listRunReceipts(filters);
8420
10522
  }
8421
- goal(idOrName) {
8422
- const goal = this.store.getGoal(idOrName) ?? this.store.findGoalByLoop(idOrName) ?? this.store.findGoalByRunId(idOrName);
10523
+ async goal(idOrName) {
10524
+ const goal = await this.store.getGoal(idOrName) ?? await this.store.findGoalByLoop(idOrName) ?? await this.store.findGoalByRunId(idOrName);
8423
10525
  return {
8424
10526
  goal,
8425
- runs: goal ? this.store.listGoalRuns({ goalId: goal.goalId }) : []
10527
+ runs: goal ? await this.store.listGoalRuns({ goalId: goal.goalId }) : []
8426
10528
  };
8427
10529
  }
10530
+ doctor() {
10531
+ return runDoctor(this.localRuntime("doctor()"));
10532
+ }
10533
+ health(opts = {}) {
10534
+ return buildHealthReport(this.localRuntime("health()"), opts);
10535
+ }
10536
+ healthScan(opts = {}) {
10537
+ const store = this.localRuntime("healthScan()");
10538
+ return buildHealthScan(store, {
10539
+ ...opts,
10540
+ doctor: opts.doctor ? runDoctor(store) : undefined,
10541
+ daemon: opts.daemon ? daemonStatus(store) : undefined
10542
+ });
10543
+ }
8428
10544
  async tick() {
8429
- return tick({ store: this.store, runnerId: this.runnerId });
10545
+ return tick({ store: this.localRuntime("tick()"), runnerId: this.runnerId });
8430
10546
  }
8431
10547
  async runNow(idOrName) {
8432
- const result = await runLoopNow({ store: this.store, idOrName: this.store.requireUniqueLoop(idOrName).id, runnerId: this.runnerId });
10548
+ const store = this.localRuntime("runNow()");
10549
+ const result = await runLoopNow({ store, idOrName: store.requireUniqueLoop(idOrName).id, runnerId: this.runnerId });
8433
10550
  return result.run;
8434
10551
  }
8435
10552
  exportBundle(opts = {}) {
8436
- return exportLoopsMigrationBundle(this.store, opts);
10553
+ return exportLoopsMigrationBundle(this.localRuntime("exportBundle()"), opts);
8437
10554
  }
8438
10555
  planImport(bundle, opts = {}) {
8439
- return buildImportMigrationPlan(this.store, bundle, opts);
10556
+ return buildImportMigrationPlan(this.localRuntime("planImport()"), bundle, opts);
8440
10557
  }
8441
10558
  importBundle(bundle, opts = {}) {
8442
- return applyImportMigrationBundle(this.store, bundle, opts);
10559
+ return applyImportMigrationBundle(this.localRuntime("importBundle()"), bundle, opts);
8443
10560
  }
8444
10561
  planSelfHostedMigration(opts = {}) {
8445
- return buildSelfHostedMigrationPlan(this.store, { ...opts, operation: opts.operation ?? "self-hosted-migrate" });
10562
+ return buildSelfHostedMigrationPlan(this.localRuntime("planSelfHostedMigration()"), { ...opts, operation: opts.operation ?? "self-hosted-migrate" });
8446
10563
  }
8447
- close() {
10564
+ async close() {
8448
10565
  if (this.ownStore)
8449
- this.store.close();
10566
+ await this.store.close();
8450
10567
  }
8451
10568
  }
8452
10569
  function loops(opts = {}) {