@hasna/loops 0.4.13 → 0.4.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -3
- package/README.md +171 -39
- package/dist/api/index.d.ts +36 -0
- package/dist/api/index.js +1518 -15
- package/dist/cli/index.js +7280 -3213
- package/dist/cli/ui.d.ts +44 -0
- package/dist/daemon/index.js +1433 -303
- package/dist/generated/storage-kit/health.d.ts +19 -0
- package/dist/generated/storage-kit/index.d.ts +7 -0
- package/dist/generated/storage-kit/migrations.d.ts +47 -0
- package/dist/generated/storage-kit/mode.d.ts +47 -0
- package/dist/generated/storage-kit/pool.d.ts +33 -0
- package/dist/generated/storage-kit/query.d.ts +35 -0
- package/dist/generated/storage-kit/tls.d.ts +25 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +8689 -4837
- package/dist/lib/cloud/mode.d.ts +17 -0
- package/dist/lib/cloud/resolve.d.ts +16 -0
- package/dist/lib/cloud/storage.d.ts +82 -0
- package/dist/lib/cloud/transport.d.ts +148 -0
- package/dist/lib/format.d.ts +2 -1
- package/dist/lib/health.d.ts +83 -3
- package/dist/lib/migration.d.ts +40 -0
- package/dist/lib/mode.js +15 -3
- package/dist/lib/route/index.d.ts +2 -0
- package/dist/lib/route/policies.d.ts +51 -0
- package/dist/lib/route/provider-admission.d.ts +37 -0
- package/dist/lib/route/throttle.d.ts +4 -0
- package/dist/lib/route/types.d.ts +8 -0
- package/dist/lib/run-receipts.d.ts +14 -0
- package/dist/lib/scheduler.d.ts +5 -2
- package/dist/lib/storage/contract.d.ts +7 -1
- package/dist/lib/storage/index.d.ts +1 -0
- package/dist/lib/storage/index.js +2028 -231
- package/dist/lib/storage/pg-executor.d.ts +27 -0
- package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
- package/dist/lib/storage/postgres-loop-storage.d.ts +120 -0
- package/dist/lib/storage/postgres-schema.js +29 -0
- package/dist/lib/storage/postgres.js +29 -0
- package/dist/lib/storage/sqlite.d.ts +6 -0
- package/dist/lib/storage/sqlite.js +748 -26
- package/dist/lib/store/index.d.ts +268 -0
- package/dist/lib/store.d.ts +282 -1
- package/dist/lib/store.js +746 -26
- package/dist/lib/template-kit.d.ts +25 -0
- package/dist/lib/templates.d.ts +16 -1
- package/dist/mcp/http.d.ts +16 -0
- package/dist/mcp/index.js +2885 -634
- package/dist/runner/index.js +198 -32
- package/dist/sdk/http.d.ts +182 -0
- package/dist/sdk/http.js +166 -0
- package/dist/sdk/index.d.ts +55 -18
- package/dist/sdk/index.js +2734 -617
- package/dist/serve/index.d.ts +4 -0
- package/dist/serve/index.js +9095 -0
- package/dist/types.d.ts +52 -0
- package/docs/AUTOMATION_RUNTIME_DESIGN.md +442 -1
- package/docs/CUTOVER-RUNBOOK.md +78 -0
- package/docs/DEPLOYMENT_MODES.md +35 -18
- package/docs/RUNTIME_BOUNDARY.md +203 -0
- package/docs/USAGE.md +195 -38
- package/package.json +15 -3
package/dist/mcp/index.js
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
|
|
4
4
|
// src/lib/store.ts
|
|
5
5
|
import { Database } from "bun:sqlite";
|
|
6
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as
|
|
7
|
-
import { tmpdir } from "os";
|
|
8
|
-
import { basename as basename2, dirname as dirname2, join as
|
|
6
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
|
|
7
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
8
|
+
import { basename as basename2, dirname as dirname2, join as join4 } from "path";
|
|
9
9
|
|
|
10
10
|
// src/lib/errors.ts
|
|
11
11
|
class CodedError extends Error {
|
|
@@ -1204,15 +1204,378 @@ function discardWorkflowRunManifest(staged) {
|
|
|
1204
1204
|
} catch {}
|
|
1205
1205
|
}
|
|
1206
1206
|
|
|
1207
|
+
// src/lib/run-receipts.ts
|
|
1208
|
+
import { createHash as createHash2 } from "crypto";
|
|
1209
|
+
import { hostname } from "os";
|
|
1210
|
+
|
|
1211
|
+
// src/lib/run-envelope.ts
|
|
1212
|
+
var ENVELOPE_EXCERPT_CHARS = 2048;
|
|
1213
|
+
var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
|
|
1214
|
+
function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
|
|
1215
|
+
if (!text)
|
|
1216
|
+
return;
|
|
1217
|
+
const scrubbed = scrubSecrets(text);
|
|
1218
|
+
if (scrubbed.length <= limit * 2)
|
|
1219
|
+
return scrubbed;
|
|
1220
|
+
const omitted = scrubbed.length - limit * 2;
|
|
1221
|
+
return `${scrubbed.slice(0, limit)}
|
|
1222
|
+
[... ${omitted} chars omitted ...]
|
|
1223
|
+
${scrubbed.slice(-limit)}`;
|
|
1224
|
+
}
|
|
1225
|
+
function summarizeOutput(stdout, stderr) {
|
|
1226
|
+
return {
|
|
1227
|
+
stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
|
|
1228
|
+
stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
|
|
1229
|
+
stdoutExcerpt: boundedExcerpt(stdout),
|
|
1230
|
+
stderrExcerpt: boundedExcerpt(stderr)
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
function summarizeExecutorResult(result) {
|
|
1234
|
+
return {
|
|
1235
|
+
...summarizeOutput(result.stdout, result.stderr),
|
|
1236
|
+
status: result.status,
|
|
1237
|
+
exitCode: result.exitCode,
|
|
1238
|
+
durationMs: result.durationMs,
|
|
1239
|
+
error: boundedExcerpt(result.error)
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
function isBlockedStepRun(step) {
|
|
1243
|
+
return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
|
|
1244
|
+
}
|
|
1245
|
+
function summarizeWorkflowStepRun(step) {
|
|
1246
|
+
return {
|
|
1247
|
+
...summarizeOutput(step.stdout, step.stderr),
|
|
1248
|
+
stepId: step.stepId,
|
|
1249
|
+
status: step.status,
|
|
1250
|
+
exitCode: step.exitCode,
|
|
1251
|
+
durationMs: step.durationMs,
|
|
1252
|
+
error: boundedExcerpt(step.error),
|
|
1253
|
+
blocked: isBlockedStepRun(step)
|
|
1254
|
+
};
|
|
1255
|
+
}
|
|
1256
|
+
function workflowRunEnvelope(run, steps) {
|
|
1257
|
+
return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
// src/lib/run-receipts.ts
|
|
1261
|
+
var RUN_RECEIPT_SUMMARY_TEXT_CHARS = 4096;
|
|
1262
|
+
var RUN_RECEIPT_MAX_IDS = 100;
|
|
1263
|
+
var RUN_RECEIPT_MAX_EVIDENCE_PATHS = 100;
|
|
1264
|
+
var RUN_RECEIPT_MAX_PATH_CHARS = 1024;
|
|
1265
|
+
function canonicalJson(value) {
|
|
1266
|
+
if (Array.isArray(value))
|
|
1267
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
1268
|
+
if (value && typeof value === "object") {
|
|
1269
|
+
return `{${Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
|
|
1270
|
+
}
|
|
1271
|
+
return JSON.stringify(value);
|
|
1272
|
+
}
|
|
1273
|
+
function digestReceipt(receipt) {
|
|
1274
|
+
return `sha256:${createHash2("sha256").update(canonicalJson(receipt)).digest("hex")}`;
|
|
1275
|
+
}
|
|
1276
|
+
function isoOrNull(value, label) {
|
|
1277
|
+
if (value === undefined || value === null || value === "")
|
|
1278
|
+
return null;
|
|
1279
|
+
const date = new Date(value);
|
|
1280
|
+
if (Number.isNaN(date.getTime()))
|
|
1281
|
+
throw new Error(`${label} must be a valid ISO date/time`);
|
|
1282
|
+
return date.toISOString();
|
|
1283
|
+
}
|
|
1284
|
+
function nonEmpty(value, label) {
|
|
1285
|
+
const trimmed = value?.trim();
|
|
1286
|
+
if (!trimmed)
|
|
1287
|
+
throw new Error(`${label} must be non-empty`);
|
|
1288
|
+
return trimmed;
|
|
1289
|
+
}
|
|
1290
|
+
function normalizedStringArray(values, opts) {
|
|
1291
|
+
const out = [];
|
|
1292
|
+
const seen = new Set;
|
|
1293
|
+
for (const raw of values ?? []) {
|
|
1294
|
+
const value = String(raw).trim();
|
|
1295
|
+
if (!value || seen.has(value))
|
|
1296
|
+
continue;
|
|
1297
|
+
seen.add(value);
|
|
1298
|
+
out.push(opts.itemMax ? value.slice(0, opts.itemMax) : value);
|
|
1299
|
+
if (out.length >= opts.max)
|
|
1300
|
+
break;
|
|
1301
|
+
}
|
|
1302
|
+
if ((values?.length ?? 0) > opts.max) {
|
|
1303
|
+
out.push(`[truncated ${values.length - opts.max} ${opts.label}]`);
|
|
1304
|
+
}
|
|
1305
|
+
return out;
|
|
1306
|
+
}
|
|
1307
|
+
function receiptMachine(input, opts) {
|
|
1308
|
+
if (typeof input.machine === "string")
|
|
1309
|
+
return nonEmpty(input.machine, "machine");
|
|
1310
|
+
if (input.machine && typeof input.machine === "object")
|
|
1311
|
+
return input.machine;
|
|
1312
|
+
if (opts.loop?.machine)
|
|
1313
|
+
return { ...opts.loop.machine };
|
|
1314
|
+
if (typeof opts.defaultMachine === "string")
|
|
1315
|
+
return nonEmpty(opts.defaultMachine, "machine");
|
|
1316
|
+
if (opts.defaultMachine && typeof opts.defaultMachine === "object")
|
|
1317
|
+
return opts.defaultMachine;
|
|
1318
|
+
return hostname();
|
|
1319
|
+
}
|
|
1320
|
+
function normalizeSummary(input, run) {
|
|
1321
|
+
const stdout = input.stdout ?? run?.stdout;
|
|
1322
|
+
const stderr = input.stderr ?? run?.stderr;
|
|
1323
|
+
const output = summarizeOutput(stdout, stderr);
|
|
1324
|
+
const summaryInput = input.summary;
|
|
1325
|
+
const provided = typeof summaryInput === "object" && summaryInput !== null ? summaryInput : {};
|
|
1326
|
+
const text = typeof summaryInput === "string" ? boundedExcerpt(summaryInput, Math.floor(RUN_RECEIPT_SUMMARY_TEXT_CHARS / 2)) : typeof provided.text === "string" ? boundedExcerpt(provided.text, Math.floor(RUN_RECEIPT_SUMMARY_TEXT_CHARS / 2)) : undefined;
|
|
1327
|
+
return {
|
|
1328
|
+
text,
|
|
1329
|
+
stdout_bytes: Number.isFinite(provided.stdout_bytes) ? Number(provided.stdout_bytes) : output.stdoutBytes,
|
|
1330
|
+
stderr_bytes: Number.isFinite(provided.stderr_bytes) ? Number(provided.stderr_bytes) : output.stderrBytes,
|
|
1331
|
+
stdout_excerpt: typeof provided.stdout_excerpt === "string" ? boundedExcerpt(provided.stdout_excerpt) : output.stdoutExcerpt,
|
|
1332
|
+
stderr_excerpt: typeof provided.stderr_excerpt === "string" ? boundedExcerpt(provided.stderr_excerpt) : output.stderrExcerpt,
|
|
1333
|
+
error: typeof provided.error === "string" ? boundedExcerpt(provided.error) : boundedExcerpt(input.error ?? run?.error),
|
|
1334
|
+
duration_ms: Number.isFinite(provided.duration_ms) ? Number(provided.duration_ms) : input.duration_ms ?? run?.durationMs
|
|
1335
|
+
};
|
|
1336
|
+
}
|
|
1337
|
+
function normalizeRunReceipt(input, opts = {}) {
|
|
1338
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1339
|
+
const run = opts.run;
|
|
1340
|
+
const loopId = input.loop_id ?? run?.loopId ?? opts.loop?.id;
|
|
1341
|
+
const normalized = {
|
|
1342
|
+
loop_id: nonEmpty(loopId, "loop_id"),
|
|
1343
|
+
run_id: nonEmpty(input.run_id ?? run?.id, "run_id"),
|
|
1344
|
+
machine: receiptMachine(input, opts),
|
|
1345
|
+
repo: nonEmpty(input.repo ?? opts.defaultRepo ?? targetRepo(opts.loop) ?? process.cwd(), "repo"),
|
|
1346
|
+
task_ids: normalizedStringArray(input.task_ids, { max: RUN_RECEIPT_MAX_IDS, label: "task_ids" }),
|
|
1347
|
+
knowledge_ids: normalizedStringArray(input.knowledge_ids, { max: RUN_RECEIPT_MAX_IDS, label: "knowledge_ids" }),
|
|
1348
|
+
started_at: isoOrNull(input.started_at ?? run?.startedAt, "started_at"),
|
|
1349
|
+
finished_at: isoOrNull(input.finished_at ?? run?.finishedAt, "finished_at"),
|
|
1350
|
+
status: nonEmpty(input.status ?? run?.status, "status"),
|
|
1351
|
+
exit_code: input.exit_code === undefined ? run?.exitCode ?? null : input.exit_code,
|
|
1352
|
+
summary: normalizeSummary(input, run),
|
|
1353
|
+
evidence_paths: normalizedStringArray(input.evidence_paths, {
|
|
1354
|
+
max: RUN_RECEIPT_MAX_EVIDENCE_PATHS,
|
|
1355
|
+
label: "evidence_paths",
|
|
1356
|
+
itemMax: RUN_RECEIPT_MAX_PATH_CHARS
|
|
1357
|
+
})
|
|
1358
|
+
};
|
|
1359
|
+
return {
|
|
1360
|
+
...normalized,
|
|
1361
|
+
digest_id: input.digest_id?.trim() || digestReceipt(normalized),
|
|
1362
|
+
created_at: opts.existing?.created_at ?? now,
|
|
1363
|
+
updated_at: now
|
|
1364
|
+
};
|
|
1365
|
+
}
|
|
1366
|
+
function targetRepo(loop) {
|
|
1367
|
+
if (!loop)
|
|
1368
|
+
return;
|
|
1369
|
+
if (loop.target.type === "command")
|
|
1370
|
+
return loop.target.cwd;
|
|
1371
|
+
if (loop.target.type === "agent")
|
|
1372
|
+
return loop.target.cwd;
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
// src/lib/route/todos-cli.ts
|
|
1377
|
+
import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
|
|
1378
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1379
|
+
import { join as join3 } from "path";
|
|
1380
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
1381
|
+
|
|
1382
|
+
// src/lib/format.ts
|
|
1383
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1384
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
1385
|
+
function redact(value, visible = 0) {
|
|
1386
|
+
if (!value)
|
|
1387
|
+
return value;
|
|
1388
|
+
if (value.length <= visible)
|
|
1389
|
+
return value;
|
|
1390
|
+
if (visible <= 0)
|
|
1391
|
+
return `[redacted ${value.length} chars]`;
|
|
1392
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1393
|
+
}
|
|
1394
|
+
function truncateTextOutput(value) {
|
|
1395
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1396
|
+
return value;
|
|
1397
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1398
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1399
|
+
}
|
|
1400
|
+
function redactSensitivePayload(value, key) {
|
|
1401
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
1402
|
+
if (typeof value === "string")
|
|
1403
|
+
return redact(value);
|
|
1404
|
+
if (value === undefined || value === null)
|
|
1405
|
+
return value;
|
|
1406
|
+
return "[redacted]";
|
|
1407
|
+
}
|
|
1408
|
+
if (Array.isArray(value))
|
|
1409
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
1410
|
+
if (value && typeof value === "object") {
|
|
1411
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
1412
|
+
}
|
|
1413
|
+
return value;
|
|
1414
|
+
}
|
|
1415
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1416
|
+
const indent = opts.indent ?? "";
|
|
1417
|
+
const nested = `${indent} `;
|
|
1418
|
+
const blocks = [];
|
|
1419
|
+
for (const [label, output] of [
|
|
1420
|
+
["stdout", value.stdout],
|
|
1421
|
+
["stderr", value.stderr]
|
|
1422
|
+
]) {
|
|
1423
|
+
if (!output)
|
|
1424
|
+
continue;
|
|
1425
|
+
blocks.push(`${indent}${label}:`);
|
|
1426
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1427
|
+
blocks.push(`${nested}${line}`);
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
return blocks;
|
|
1431
|
+
}
|
|
1432
|
+
function publicLoop(loop) {
|
|
1433
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1434
|
+
return {
|
|
1435
|
+
...loop,
|
|
1436
|
+
target
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
function publicRun(run, showOutput = false, opts = {}) {
|
|
1440
|
+
return {
|
|
1441
|
+
...run,
|
|
1442
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1443
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1444
|
+
error: opts.redactError ? redact(run.error) : run.error
|
|
1445
|
+
};
|
|
1446
|
+
}
|
|
1447
|
+
function publicRunReceipt(receipt) {
|
|
1448
|
+
return { ...receipt };
|
|
1449
|
+
}
|
|
1450
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
1451
|
+
return {
|
|
1452
|
+
...result,
|
|
1453
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
1454
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
1455
|
+
error: redact(result.error)
|
|
1456
|
+
};
|
|
1457
|
+
}
|
|
1458
|
+
function publicWorkflow(workflow) {
|
|
1459
|
+
return {
|
|
1460
|
+
...workflow,
|
|
1461
|
+
steps: workflow.steps.map((step) => ({
|
|
1462
|
+
...step,
|
|
1463
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
1464
|
+
}))
|
|
1465
|
+
};
|
|
1466
|
+
}
|
|
1467
|
+
function publicWorkflowRun(run) {
|
|
1468
|
+
return { ...run, error: redact(run.error) };
|
|
1469
|
+
}
|
|
1470
|
+
function publicWorkflowInvocation(invocation) {
|
|
1471
|
+
return redactSensitivePayload(invocation);
|
|
1472
|
+
}
|
|
1473
|
+
function publicWorkflowWorkItem(item) {
|
|
1474
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
1475
|
+
}
|
|
1476
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
1477
|
+
return {
|
|
1478
|
+
...run,
|
|
1479
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1480
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1481
|
+
error: redact(run.error)
|
|
1482
|
+
};
|
|
1483
|
+
}
|
|
1484
|
+
function publicWorkflowEvent(event) {
|
|
1485
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
1486
|
+
}
|
|
1487
|
+
function publicGoal(goal) {
|
|
1488
|
+
return {
|
|
1489
|
+
...goal,
|
|
1490
|
+
objective: redact(goal.objective, 120)
|
|
1491
|
+
};
|
|
1492
|
+
}
|
|
1493
|
+
function publicGoalRun(run) {
|
|
1494
|
+
return {
|
|
1495
|
+
...run,
|
|
1496
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
1497
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
1498
|
+
};
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
// src/lib/route/todos-cli.ts
|
|
1502
|
+
function defaultLoopsProject() {
|
|
1503
|
+
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
|
|
1504
|
+
}
|
|
1505
|
+
function runLocalCommand(command, args, opts = {}) {
|
|
1506
|
+
const result = spawnSync2(command, args, {
|
|
1507
|
+
input: opts.input,
|
|
1508
|
+
encoding: "utf8",
|
|
1509
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1510
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1511
|
+
env: process.env
|
|
1512
|
+
});
|
|
1513
|
+
return {
|
|
1514
|
+
ok: result.status === 0,
|
|
1515
|
+
status: result.status,
|
|
1516
|
+
stdout: result.stdout || "",
|
|
1517
|
+
stderr: result.stderr || "",
|
|
1518
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1519
|
+
};
|
|
1520
|
+
}
|
|
1521
|
+
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
1522
|
+
const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
|
|
1523
|
+
const stdoutPath = join3(tempDir, "stdout");
|
|
1524
|
+
const stdoutFd = openSync(stdoutPath, "w");
|
|
1525
|
+
let result;
|
|
1526
|
+
try {
|
|
1527
|
+
result = spawnSync2(command, args, {
|
|
1528
|
+
input: opts.input,
|
|
1529
|
+
encoding: "utf8",
|
|
1530
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1531
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1532
|
+
env: process.env,
|
|
1533
|
+
stdio: ["pipe", stdoutFd, "pipe"]
|
|
1534
|
+
});
|
|
1535
|
+
} finally {
|
|
1536
|
+
closeSync(stdoutFd);
|
|
1537
|
+
}
|
|
1538
|
+
try {
|
|
1539
|
+
return {
|
|
1540
|
+
ok: result.status === 0,
|
|
1541
|
+
status: result.status,
|
|
1542
|
+
stdout: readFileSync3(stdoutPath, "utf8"),
|
|
1543
|
+
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
1544
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1545
|
+
};
|
|
1546
|
+
} finally {
|
|
1547
|
+
rmSync2(tempDir, { recursive: true, force: true });
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
function ensureTodosTaskList(project, slug, name, description) {
|
|
1551
|
+
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
1552
|
+
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
1553
|
+
if (!list.ok)
|
|
1554
|
+
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
1555
|
+
const values = JSON.parse(list.stdout || "[]");
|
|
1556
|
+
const found = values.find((entry) => entry.slug === slug);
|
|
1557
|
+
if (!found)
|
|
1558
|
+
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
1559
|
+
return found.id;
|
|
1560
|
+
}
|
|
1561
|
+
function todosMutationSummary(result) {
|
|
1562
|
+
return {
|
|
1563
|
+
ok: result.ok,
|
|
1564
|
+
status: result.status,
|
|
1565
|
+
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1207
1569
|
// src/lib/store.ts
|
|
1208
1570
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
1209
1571
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
1210
1572
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1211
|
-
var SCHEMA_USER_VERSION =
|
|
1573
|
+
var SCHEMA_USER_VERSION = 8;
|
|
1212
1574
|
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1213
1575
|
var PRUNE_BATCH_SIZE = 400;
|
|
1214
1576
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
1215
1577
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
1578
|
+
var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
|
|
1216
1579
|
function rowToLoop(row) {
|
|
1217
1580
|
return {
|
|
1218
1581
|
id: row.id,
|
|
@@ -1263,6 +1626,28 @@ function rowToRun(row) {
|
|
|
1263
1626
|
updatedAt: row.updated_at
|
|
1264
1627
|
};
|
|
1265
1628
|
}
|
|
1629
|
+
function rowToRunReceipt(row) {
|
|
1630
|
+
return {
|
|
1631
|
+
loop_id: row.loop_id,
|
|
1632
|
+
run_id: row.run_id,
|
|
1633
|
+
machine: JSON.parse(row.machine_json),
|
|
1634
|
+
repo: row.repo,
|
|
1635
|
+
task_ids: JSON.parse(row.task_ids_json),
|
|
1636
|
+
knowledge_ids: JSON.parse(row.knowledge_ids_json),
|
|
1637
|
+
digest_id: row.digest_id,
|
|
1638
|
+
started_at: row.started_at,
|
|
1639
|
+
finished_at: row.finished_at,
|
|
1640
|
+
status: row.status,
|
|
1641
|
+
exit_code: row.exit_code,
|
|
1642
|
+
summary: JSON.parse(row.summary_json),
|
|
1643
|
+
evidence_paths: JSON.parse(row.evidence_paths_json),
|
|
1644
|
+
created_at: row.created_at,
|
|
1645
|
+
updated_at: row.updated_at
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1648
|
+
function latestRunTime(row) {
|
|
1649
|
+
return row.finished_at ?? row.started_at ?? row.created_at;
|
|
1650
|
+
}
|
|
1266
1651
|
function rowToWorkflow(row) {
|
|
1267
1652
|
return {
|
|
1268
1653
|
id: row.id,
|
|
@@ -1323,6 +1708,7 @@ function rowToWorkflowWorkItem(row) {
|
|
|
1323
1708
|
subjectRef: row.subject_ref,
|
|
1324
1709
|
projectKey: row.project_key ?? undefined,
|
|
1325
1710
|
projectGroup: row.project_group ?? undefined,
|
|
1711
|
+
machineId: row.machine_id ?? undefined,
|
|
1326
1712
|
routeScope: row.route_scope ?? undefined,
|
|
1327
1713
|
priority: row.priority,
|
|
1328
1714
|
status: row.status,
|
|
@@ -1480,6 +1866,7 @@ function scrubbedOrNull(value) {
|
|
|
1480
1866
|
return value == null ? null : scrubSecrets(value);
|
|
1481
1867
|
}
|
|
1482
1868
|
var MAX_PERSISTED_RUN_OUTPUT_CHARS = 64 * 1024;
|
|
1869
|
+
var MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS = 64 * 1024;
|
|
1483
1870
|
function clampPersistedRunOutput(value) {
|
|
1484
1871
|
if (value == null || value.length <= MAX_PERSISTED_RUN_OUTPUT_CHARS)
|
|
1485
1872
|
return value;
|
|
@@ -1494,6 +1881,42 @@ ${tail}`;
|
|
|
1494
1881
|
function persistedRunOutput(value) {
|
|
1495
1882
|
return clampPersistedRunOutput(scrubbedOrNull(value));
|
|
1496
1883
|
}
|
|
1884
|
+
function clampTextToChars(value, maxChars, reason) {
|
|
1885
|
+
if (value.length <= maxChars)
|
|
1886
|
+
return value;
|
|
1887
|
+
const marker = `
|
|
1888
|
+
...[truncated by ${reason}]...
|
|
1889
|
+
`;
|
|
1890
|
+
const budget = Math.max(0, maxChars - marker.length);
|
|
1891
|
+
const headLength = Math.ceil(budget / 2);
|
|
1892
|
+
const tailLength = Math.floor(budget / 2);
|
|
1893
|
+
return `${value.slice(0, headLength)}${marker}${value.slice(value.length - tailLength)}`;
|
|
1894
|
+
}
|
|
1895
|
+
function boundedWorkflowEventPayloadJson(scrubbedJson) {
|
|
1896
|
+
if (scrubbedJson.length <= MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS)
|
|
1897
|
+
return scrubbedJson;
|
|
1898
|
+
const base = {
|
|
1899
|
+
truncated: true,
|
|
1900
|
+
originalChars: scrubbedJson.length,
|
|
1901
|
+
maxChars: MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS,
|
|
1902
|
+
preview: ""
|
|
1903
|
+
};
|
|
1904
|
+
const baseChars = JSON.stringify(base).length;
|
|
1905
|
+
let previewBudget = Math.max(0, MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS - baseChars - 64);
|
|
1906
|
+
while (true) {
|
|
1907
|
+
const preview = clampTextToChars(scrubbedJson, previewBudget, "loops workflow-event payload retention");
|
|
1908
|
+
const bounded = JSON.stringify({ ...base, preview });
|
|
1909
|
+
if (bounded.length <= MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS || previewBudget === 0)
|
|
1910
|
+
return bounded;
|
|
1911
|
+
previewBudget = Math.max(0, previewBudget - (bounded.length - MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS) - 64);
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
function persistedWorkflowEventPayload(payload) {
|
|
1915
|
+
if (payload == null)
|
|
1916
|
+
return null;
|
|
1917
|
+
const scrubbed = scrubSecretsDeep(payload);
|
|
1918
|
+
return boundedWorkflowEventPayloadJson(scrubSecrets(JSON.stringify(scrubbed)));
|
|
1919
|
+
}
|
|
1497
1920
|
function chmodIfExists(path, mode) {
|
|
1498
1921
|
try {
|
|
1499
1922
|
if (existsSync(path))
|
|
@@ -1517,7 +1940,7 @@ class Store {
|
|
|
1517
1940
|
const file = path ?? dbPath();
|
|
1518
1941
|
if (file !== ":memory:")
|
|
1519
1942
|
ensurePrivateStorePath(file);
|
|
1520
|
-
this.rootDir = file === ":memory:" ?
|
|
1943
|
+
this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
|
|
1521
1944
|
if (file === ":memory:")
|
|
1522
1945
|
this.memoryRootDir = this.rootDir;
|
|
1523
1946
|
this.db = new Database(file);
|
|
@@ -1611,6 +2034,17 @@ class Store {
|
|
|
1611
2034
|
this.addColumnIfMissing("workflow_work_items", "route_scope", "TEXT");
|
|
1612
2035
|
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_scope ON workflow_work_items(route_scope, status)");
|
|
1613
2036
|
}
|
|
2037
|
+
},
|
|
2038
|
+
{
|
|
2039
|
+
id: "0009_run_receipts",
|
|
2040
|
+
apply: () => this.createRunReceiptsSchema()
|
|
2041
|
+
},
|
|
2042
|
+
{
|
|
2043
|
+
id: "0010_work_item_machine_id",
|
|
2044
|
+
apply: () => {
|
|
2045
|
+
this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
|
|
2046
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
|
|
2047
|
+
}
|
|
1614
2048
|
}
|
|
1615
2049
|
];
|
|
1616
2050
|
}
|
|
@@ -1672,6 +2106,28 @@ class Store {
|
|
|
1672
2106
|
CREATE INDEX IF NOT EXISTS idx_runs_status_lease ON loop_runs(status, lease_expires_at);
|
|
1673
2107
|
CREATE INDEX IF NOT EXISTS idx_runs_scheduled ON loop_runs(scheduled_for);
|
|
1674
2108
|
|
|
2109
|
+
CREATE TABLE IF NOT EXISTS run_receipts (
|
|
2110
|
+
run_id TEXT PRIMARY KEY,
|
|
2111
|
+
loop_id TEXT NOT NULL,
|
|
2112
|
+
machine_json TEXT NOT NULL,
|
|
2113
|
+
repo TEXT NOT NULL,
|
|
2114
|
+
task_ids_json TEXT NOT NULL,
|
|
2115
|
+
knowledge_ids_json TEXT NOT NULL,
|
|
2116
|
+
digest_id TEXT NOT NULL,
|
|
2117
|
+
started_at TEXT,
|
|
2118
|
+
finished_at TEXT,
|
|
2119
|
+
status TEXT NOT NULL,
|
|
2120
|
+
exit_code INTEGER,
|
|
2121
|
+
summary_json TEXT NOT NULL,
|
|
2122
|
+
evidence_paths_json TEXT NOT NULL,
|
|
2123
|
+
created_at TEXT NOT NULL,
|
|
2124
|
+
updated_at TEXT NOT NULL
|
|
2125
|
+
);
|
|
2126
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_loop ON run_receipts(loop_id, created_at DESC);
|
|
2127
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_repo ON run_receipts(repo, created_at DESC);
|
|
2128
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_digest ON run_receipts(digest_id);
|
|
2129
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_status ON run_receipts(status, created_at DESC);
|
|
2130
|
+
|
|
1675
2131
|
CREATE TABLE IF NOT EXISTS daemon_lease (
|
|
1676
2132
|
id TEXT PRIMARY KEY,
|
|
1677
2133
|
pid INTEGER NOT NULL,
|
|
@@ -1758,6 +2214,7 @@ class Store {
|
|
|
1758
2214
|
subject_ref TEXT NOT NULL,
|
|
1759
2215
|
project_key TEXT,
|
|
1760
2216
|
project_group TEXT,
|
|
2217
|
+
machine_id TEXT,
|
|
1761
2218
|
route_scope TEXT,
|
|
1762
2219
|
priority INTEGER NOT NULL,
|
|
1763
2220
|
status TEXT NOT NULL,
|
|
@@ -1776,10 +2233,10 @@ class Store {
|
|
|
1776
2233
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_project ON workflow_work_items(project_key, status);
|
|
1777
2234
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_group ON workflow_work_items(project_group, status);
|
|
1778
2235
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_invocation ON workflow_work_items(invocation_id);
|
|
1779
|
-
--
|
|
1780
|
-
--
|
|
2236
|
+
-- New-column indexes (route_scope, machine_id, etc.) are created ONLY by
|
|
2237
|
+
-- their additive migrations, never here: this baseline DDL
|
|
1781
2238
|
-- re-runs on EVERY open (0001 is not skip-guarded), and on a pre-0008
|
|
1782
|
-
-- database the CREATE TABLE above is a no-op, so an index on
|
|
2239
|
+
-- database the CREATE TABLE above is a no-op, so an index on a new column
|
|
1783
2240
|
-- here would execute before the column exists and crash the open
|
|
1784
2241
|
-- ("no such column: route_scope"). New columns may be folded into the
|
|
1785
2242
|
-- CREATE TABLE (fresh-db only); their indexes must live in the migration.
|
|
@@ -1890,6 +2347,31 @@ class Store {
|
|
|
1890
2347
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
1891
2348
|
`);
|
|
1892
2349
|
}
|
|
2350
|
+
createRunReceiptsSchema() {
|
|
2351
|
+
this.db.exec(`
|
|
2352
|
+
CREATE TABLE IF NOT EXISTS run_receipts (
|
|
2353
|
+
run_id TEXT PRIMARY KEY,
|
|
2354
|
+
loop_id TEXT NOT NULL,
|
|
2355
|
+
machine_json TEXT NOT NULL,
|
|
2356
|
+
repo TEXT NOT NULL,
|
|
2357
|
+
task_ids_json TEXT NOT NULL,
|
|
2358
|
+
knowledge_ids_json TEXT NOT NULL,
|
|
2359
|
+
digest_id TEXT NOT NULL,
|
|
2360
|
+
started_at TEXT,
|
|
2361
|
+
finished_at TEXT,
|
|
2362
|
+
status TEXT NOT NULL,
|
|
2363
|
+
exit_code INTEGER,
|
|
2364
|
+
summary_json TEXT NOT NULL,
|
|
2365
|
+
evidence_paths_json TEXT NOT NULL,
|
|
2366
|
+
created_at TEXT NOT NULL,
|
|
2367
|
+
updated_at TEXT NOT NULL
|
|
2368
|
+
);
|
|
2369
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_loop ON run_receipts(loop_id, created_at DESC);
|
|
2370
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_repo ON run_receipts(repo, created_at DESC);
|
|
2371
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_digest ON run_receipts(digest_id);
|
|
2372
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_status ON run_receipts(status, created_at DESC);
|
|
2373
|
+
`);
|
|
2374
|
+
}
|
|
1893
2375
|
addColumnIfMissing(table, column, definition) {
|
|
1894
2376
|
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
1895
2377
|
if (columns.some((c) => c.name === column))
|
|
@@ -2001,21 +2483,51 @@ class Store {
|
|
|
2001
2483
|
}
|
|
2002
2484
|
listLoops(opts = {}) {
|
|
2003
2485
|
const limit = opts.limit ?? 200;
|
|
2486
|
+
const offset = Math.max(0, Math.floor(opts.offset ?? 0));
|
|
2487
|
+
if (opts.name != null) {
|
|
2488
|
+
const rows2 = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.name, limit, offset);
|
|
2489
|
+
return this.withLatestRunSummaries(rows2.map(rowToLoop));
|
|
2490
|
+
}
|
|
2004
2491
|
let rows;
|
|
2005
2492
|
if (opts.status && opts.archived) {
|
|
2006
|
-
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NOT NULL ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
|
|
2493
|
+
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NOT NULL ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
2007
2494
|
} else if (opts.status && opts.includeArchived) {
|
|
2008
|
-
rows = this.db.query("SELECT * FROM loops WHERE status = ? ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
|
|
2495
|
+
rows = this.db.query("SELECT * FROM loops WHERE status = ? ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
2009
2496
|
} else if (opts.status) {
|
|
2010
|
-
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NULL ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
|
|
2497
|
+
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NULL ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
2011
2498
|
} else if (opts.archived) {
|
|
2012
|
-
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NOT NULL ORDER BY archived_at DESC LIMIT ?").all(limit);
|
|
2499
|
+
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NOT NULL ORDER BY archived_at DESC LIMIT ? OFFSET ?").all(limit, offset);
|
|
2013
2500
|
} else if (opts.includeArchived) {
|
|
2014
|
-
rows = this.db.query("SELECT * FROM loops ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2501
|
+
rows = this.db.query("SELECT * FROM loops ORDER BY status ASC, next_run_at ASC LIMIT ? OFFSET ?").all(limit, offset);
|
|
2015
2502
|
} else {
|
|
2016
|
-
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2017
|
-
}
|
|
2018
|
-
return rows.map(rowToLoop);
|
|
2503
|
+
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ? OFFSET ?").all(limit, offset);
|
|
2504
|
+
}
|
|
2505
|
+
return this.withLatestRunSummaries(rows.map(rowToLoop));
|
|
2506
|
+
}
|
|
2507
|
+
withLatestRunSummaries(loops) {
|
|
2508
|
+
if (loops.length === 0)
|
|
2509
|
+
return loops;
|
|
2510
|
+
const placeholders = loops.map(() => "?").join(",");
|
|
2511
|
+
const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
|
|
2512
|
+
FROM (
|
|
2513
|
+
SELECT loop_id, id, status, started_at, finished_at, created_at,
|
|
2514
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
|
|
2515
|
+
FROM loop_runs
|
|
2516
|
+
WHERE loop_id IN (${placeholders})
|
|
2517
|
+
)
|
|
2518
|
+
WHERE rn = 1`).all(...loops.map((loop) => loop.id));
|
|
2519
|
+
const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
|
|
2520
|
+
return loops.map((loop) => {
|
|
2521
|
+
const latest = latestByLoopId.get(loop.id);
|
|
2522
|
+
if (!latest)
|
|
2523
|
+
return loop;
|
|
2524
|
+
return {
|
|
2525
|
+
...loop,
|
|
2526
|
+
latestRunId: latest.id,
|
|
2527
|
+
latestRunStatus: latest.status,
|
|
2528
|
+
lastRunAt: latestRunTime(latest)
|
|
2529
|
+
};
|
|
2530
|
+
});
|
|
2019
2531
|
}
|
|
2020
2532
|
dueLoops(now, limit = 500) {
|
|
2021
2533
|
const rows = this.db.query(`SELECT * FROM loops
|
|
@@ -2134,6 +2646,45 @@ class Store {
|
|
|
2134
2646
|
throw error;
|
|
2135
2647
|
}
|
|
2136
2648
|
}
|
|
2649
|
+
updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
|
|
2650
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
2651
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2652
|
+
try {
|
|
2653
|
+
const current = this.requireUniqueLoop(idOrName);
|
|
2654
|
+
if (current.archivedAt)
|
|
2655
|
+
throw new LoopArchivedError(current.name || current.id);
|
|
2656
|
+
if (current.target.type !== "agent")
|
|
2657
|
+
throw new Error(`loop is not an agent loop: ${idOrName}`);
|
|
2658
|
+
if (this.hasRunningRun(current.id))
|
|
2659
|
+
throw new Error(`refusing to update running loop: ${current.id}`);
|
|
2660
|
+
const target = { ...current.target, timeoutMs };
|
|
2661
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
2662
|
+
delete target.idleTimeoutMs;
|
|
2663
|
+
const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
|
|
2664
|
+
WHERE id=$id
|
|
2665
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2666
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2667
|
+
))`).run({
|
|
2668
|
+
$id: current.id,
|
|
2669
|
+
$target: JSON.stringify(target),
|
|
2670
|
+
$updated: updated,
|
|
2671
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2672
|
+
$now: updated
|
|
2673
|
+
});
|
|
2674
|
+
if (res.changes !== 1)
|
|
2675
|
+
throw new Error("daemon lease lost");
|
|
2676
|
+
this.db.exec("COMMIT");
|
|
2677
|
+
const after = this.getLoop(current.id);
|
|
2678
|
+
if (!after)
|
|
2679
|
+
throw new Error(`loop not found after timeout update: ${current.id}`);
|
|
2680
|
+
return after;
|
|
2681
|
+
} catch (error) {
|
|
2682
|
+
try {
|
|
2683
|
+
this.db.exec("ROLLBACK");
|
|
2684
|
+
} catch {}
|
|
2685
|
+
throw error;
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2137
2688
|
createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
|
|
2138
2689
|
const normalized = normalizeCreateWorkflowInput(workflowInput);
|
|
2139
2690
|
const updated = (opts.now ?? new Date).toISOString();
|
|
@@ -2464,6 +3015,59 @@ class Store {
|
|
|
2464
3015
|
updated
|
|
2465
3016
|
});
|
|
2466
3017
|
}
|
|
3018
|
+
taskLifecycleTodosPointerContext(workflowRunId) {
|
|
3019
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
3020
|
+
if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
|
|
3021
|
+
return;
|
|
3022
|
+
const workItem = this.getWorkflowWorkItem(run.workItemId);
|
|
3023
|
+
if (!workItem || workItem.routeKey !== "todos-task")
|
|
3024
|
+
return;
|
|
3025
|
+
const invocation = this.getWorkflowInvocation(run.invocationId);
|
|
3026
|
+
if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
|
|
3027
|
+
return;
|
|
3028
|
+
const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
|
|
3029
|
+
const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
|
|
3030
|
+
if (!projectPath || !taskId)
|
|
3031
|
+
return;
|
|
3032
|
+
return {
|
|
3033
|
+
projectPath,
|
|
3034
|
+
taskId,
|
|
3035
|
+
invocationId: invocation.id,
|
|
3036
|
+
workflowRunId: run.id,
|
|
3037
|
+
manifestPath: run.manifestPath
|
|
3038
|
+
};
|
|
3039
|
+
}
|
|
3040
|
+
syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
|
|
3041
|
+
const context = this.taskLifecycleTodosPointerContext(workflowRunId);
|
|
3042
|
+
if (!context)
|
|
3043
|
+
return;
|
|
3044
|
+
const result = runLocalCommand("todos", [
|
|
3045
|
+
"--project",
|
|
3046
|
+
context.projectPath,
|
|
3047
|
+
"task",
|
|
3048
|
+
"workflow-pointers",
|
|
3049
|
+
context.taskId,
|
|
3050
|
+
"--clear",
|
|
3051
|
+
"--invocation",
|
|
3052
|
+
context.invocationId,
|
|
3053
|
+
"--run",
|
|
3054
|
+
context.workflowRunId,
|
|
3055
|
+
"--manifest",
|
|
3056
|
+
context.manifestPath,
|
|
3057
|
+
"--state",
|
|
3058
|
+
"succeeded",
|
|
3059
|
+
"--actor",
|
|
3060
|
+
"openloops:task-lifecycle"
|
|
3061
|
+
]);
|
|
3062
|
+
this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
|
|
3063
|
+
projectPath: context.projectPath,
|
|
3064
|
+
taskId: context.taskId,
|
|
3065
|
+
invocationId: context.invocationId,
|
|
3066
|
+
workflowRunId: context.workflowRunId,
|
|
3067
|
+
manifestPath: context.manifestPath,
|
|
3068
|
+
mutation: todosMutationSummary(result)
|
|
3069
|
+
});
|
|
3070
|
+
}
|
|
2467
3071
|
createWorkflowInvocation(input) {
|
|
2468
3072
|
const now = nowIso();
|
|
2469
3073
|
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
@@ -2574,10 +3178,10 @@ class Store {
|
|
|
2574
3178
|
const id = genId();
|
|
2575
3179
|
const status = input.status ?? "queued";
|
|
2576
3180
|
this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
|
|
2577
|
-
subject_ref, project_key, project_group, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
3181
|
+
subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
2578
3182
|
workflow_id, loop_id, workflow_run_id, last_reason, created_at, updated_at)
|
|
2579
3183
|
VALUES ($id, $routeKey, $idempotencyKey, $invocationId, $sourceType, $sourceRef, $subjectRef,
|
|
2580
|
-
$projectKey, $projectGroup, $routeScope, $priority, $status, 0, $nextAttemptAt, NULL, NULL, NULL, NULL,
|
|
3184
|
+
$projectKey, $projectGroup, $machineId, $routeScope, $priority, $status, 0, $nextAttemptAt, NULL, NULL, NULL, NULL,
|
|
2581
3185
|
$lastReason, $created, $updated)
|
|
2582
3186
|
ON CONFLICT(route_key, idempotency_key) DO UPDATE SET
|
|
2583
3187
|
invocation_id=excluded.invocation_id,
|
|
@@ -2586,6 +3190,10 @@ class Store {
|
|
|
2586
3190
|
subject_ref=excluded.subject_ref,
|
|
2587
3191
|
project_key=excluded.project_key,
|
|
2588
3192
|
project_group=excluded.project_group,
|
|
3193
|
+
machine_id=CASE
|
|
3194
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.machine_id
|
|
3195
|
+
ELSE excluded.machine_id
|
|
3196
|
+
END,
|
|
2589
3197
|
route_scope=excluded.route_scope,
|
|
2590
3198
|
priority=excluded.priority,
|
|
2591
3199
|
status=CASE
|
|
@@ -2628,6 +3236,7 @@ class Store {
|
|
|
2628
3236
|
$subjectRef: input.subjectRef,
|
|
2629
3237
|
$projectKey: input.projectKey ?? null,
|
|
2630
3238
|
$projectGroup: input.projectGroup ?? null,
|
|
3239
|
+
$machineId: input.machineId ?? null,
|
|
2631
3240
|
$routeScope: input.routeScope ?? null,
|
|
2632
3241
|
$priority: input.priority ?? 0,
|
|
2633
3242
|
$status: status,
|
|
@@ -3137,7 +3746,7 @@ class Store {
|
|
|
3137
3746
|
VALUES ($id, $workflowRunId, 1, 'created', NULL, $payload, $created)`).run({
|
|
3138
3747
|
$id: genId(),
|
|
3139
3748
|
$workflowRunId: runId,
|
|
3140
|
-
$payload:
|
|
3749
|
+
$payload: persistedWorkflowEventPayload({
|
|
3141
3750
|
workflowId: input.workflow.id,
|
|
3142
3751
|
workflowName: input.workflow.name,
|
|
3143
3752
|
stepCount: input.workflow.steps.length,
|
|
@@ -3426,6 +4035,8 @@ class Store {
|
|
|
3426
4035
|
const run = this.getWorkflowRun(workflowRunId);
|
|
3427
4036
|
if (!run)
|
|
3428
4037
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
4038
|
+
if (changed && status === "succeeded")
|
|
4039
|
+
this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
|
|
3429
4040
|
return run;
|
|
3430
4041
|
}
|
|
3431
4042
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -3466,7 +4077,7 @@ class Store {
|
|
|
3466
4077
|
$sequence: sequence,
|
|
3467
4078
|
$eventType: eventType,
|
|
3468
4079
|
$stepId: stepId ?? null,
|
|
3469
|
-
$payload:
|
|
4080
|
+
$payload: persistedWorkflowEventPayload(payload),
|
|
3470
4081
|
$created: now
|
|
3471
4082
|
});
|
|
3472
4083
|
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
@@ -3487,6 +4098,17 @@ class Store {
|
|
|
3487
4098
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
3488
4099
|
return (row?.count ?? 0) > 0;
|
|
3489
4100
|
}
|
|
4101
|
+
hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
|
|
4102
|
+
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
4103
|
+
WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
|
|
4104
|
+
return rows.some((row) => {
|
|
4105
|
+
if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
|
|
4106
|
+
return true;
|
|
4107
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at))
|
|
4108
|
+
return true;
|
|
4109
|
+
return this.hasLiveWorkflowStepProcesses(row.id);
|
|
4110
|
+
});
|
|
4111
|
+
}
|
|
3490
4112
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
3491
4113
|
const now = (opts.now ?? new Date).toISOString();
|
|
3492
4114
|
const startedMs = processStartTimeMs(pid);
|
|
@@ -3619,6 +4241,10 @@ class Store {
|
|
|
3619
4241
|
loop = currentLoop;
|
|
3620
4242
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
3621
4243
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
4244
|
+
if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
|
|
4245
|
+
this.db.exec("COMMIT");
|
|
4246
|
+
return;
|
|
4247
|
+
}
|
|
3622
4248
|
if (existing) {
|
|
3623
4249
|
if (existing.status === "running") {
|
|
3624
4250
|
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
@@ -3774,18 +4400,93 @@ class Store {
|
|
|
3774
4400
|
}
|
|
3775
4401
|
listRuns(opts = {}) {
|
|
3776
4402
|
const limit = opts.limit ?? 100;
|
|
4403
|
+
const offset = Math.max(0, Math.floor(opts.offset ?? 0));
|
|
3777
4404
|
let rows;
|
|
3778
4405
|
if (opts.loopId && opts.status) {
|
|
3779
|
-
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND status = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopId, opts.status, limit);
|
|
4406
|
+
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.loopId, opts.status, limit, offset);
|
|
3780
4407
|
} else if (opts.loopId) {
|
|
3781
|
-
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopId, limit);
|
|
4408
|
+
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.loopId, limit, offset);
|
|
3782
4409
|
} else if (opts.status) {
|
|
3783
|
-
rows = this.db.query("SELECT * FROM loop_runs WHERE status = ? ORDER BY created_at DESC LIMIT ?").all(opts.status, limit);
|
|
4410
|
+
rows = this.db.query("SELECT * FROM loop_runs WHERE status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
3784
4411
|
} else {
|
|
3785
|
-
rows = this.db.query("SELECT * FROM loop_runs ORDER BY created_at DESC LIMIT ?").all(limit);
|
|
4412
|
+
rows = this.db.query("SELECT * FROM loop_runs ORDER BY created_at DESC LIMIT ? OFFSET ?").all(limit, offset);
|
|
3786
4413
|
}
|
|
3787
4414
|
return rows.map(rowToRun);
|
|
3788
4415
|
}
|
|
4416
|
+
writeRunReceipt(input, opts = {}) {
|
|
4417
|
+
const inputRunId = typeof input.run_id === "string" && input.run_id.trim() ? input.run_id : undefined;
|
|
4418
|
+
const existing = inputRunId ? this.getRunReceipt(inputRunId) : undefined;
|
|
4419
|
+
const run = inputRunId ? this.getRun(inputRunId) : undefined;
|
|
4420
|
+
const loop = input.loop_id ? this.getLoop(input.loop_id) : run ? this.getLoop(run.loopId) : undefined;
|
|
4421
|
+
const receipt = normalizeRunReceipt(input, { now: opts.now, run, loop, existing });
|
|
4422
|
+
this.db.query(`INSERT INTO run_receipts (run_id, loop_id, machine_json, repo, task_ids_json, knowledge_ids_json, digest_id,
|
|
4423
|
+
started_at, finished_at, status, exit_code, summary_json, evidence_paths_json, created_at, updated_at)
|
|
4424
|
+
VALUES ($runId, $loopId, $machineJson, $repo, $taskIdsJson, $knowledgeIdsJson, $digestId,
|
|
4425
|
+
$startedAt, $finishedAt, $status, $exitCode, $summaryJson, $evidencePathsJson, $createdAt, $updatedAt)
|
|
4426
|
+
ON CONFLICT(run_id) DO UPDATE SET
|
|
4427
|
+
loop_id=excluded.loop_id,
|
|
4428
|
+
machine_json=excluded.machine_json,
|
|
4429
|
+
repo=excluded.repo,
|
|
4430
|
+
task_ids_json=excluded.task_ids_json,
|
|
4431
|
+
knowledge_ids_json=excluded.knowledge_ids_json,
|
|
4432
|
+
digest_id=excluded.digest_id,
|
|
4433
|
+
started_at=excluded.started_at,
|
|
4434
|
+
finished_at=excluded.finished_at,
|
|
4435
|
+
status=excluded.status,
|
|
4436
|
+
exit_code=excluded.exit_code,
|
|
4437
|
+
summary_json=excluded.summary_json,
|
|
4438
|
+
evidence_paths_json=excluded.evidence_paths_json,
|
|
4439
|
+
updated_at=excluded.updated_at`).run({
|
|
4440
|
+
$runId: receipt.run_id,
|
|
4441
|
+
$loopId: receipt.loop_id,
|
|
4442
|
+
$machineJson: JSON.stringify(receipt.machine),
|
|
4443
|
+
$repo: receipt.repo,
|
|
4444
|
+
$taskIdsJson: JSON.stringify(receipt.task_ids),
|
|
4445
|
+
$knowledgeIdsJson: JSON.stringify(receipt.knowledge_ids),
|
|
4446
|
+
$digestId: receipt.digest_id,
|
|
4447
|
+
$startedAt: receipt.started_at,
|
|
4448
|
+
$finishedAt: receipt.finished_at,
|
|
4449
|
+
$status: receipt.status,
|
|
4450
|
+
$exitCode: receipt.exit_code,
|
|
4451
|
+
$summaryJson: JSON.stringify(receipt.summary),
|
|
4452
|
+
$evidencePathsJson: JSON.stringify(receipt.evidence_paths),
|
|
4453
|
+
$createdAt: receipt.created_at,
|
|
4454
|
+
$updatedAt: receipt.updated_at
|
|
4455
|
+
});
|
|
4456
|
+
return this.getRunReceipt(receipt.run_id) ?? receipt;
|
|
4457
|
+
}
|
|
4458
|
+
getRunReceipt(runId) {
|
|
4459
|
+
const row = this.db.query("SELECT * FROM run_receipts WHERE run_id = ?").get(runId);
|
|
4460
|
+
return row ? rowToRunReceipt(row) : undefined;
|
|
4461
|
+
}
|
|
4462
|
+
listRunReceipts(opts = {}) {
|
|
4463
|
+
const limit = opts.limit ?? 100;
|
|
4464
|
+
const filters = [];
|
|
4465
|
+
const params = [];
|
|
4466
|
+
if (opts.loopId) {
|
|
4467
|
+
filters.push("loop_id = ?");
|
|
4468
|
+
params.push(opts.loopId);
|
|
4469
|
+
}
|
|
4470
|
+
if (opts.repo) {
|
|
4471
|
+
filters.push("repo = ?");
|
|
4472
|
+
params.push(opts.repo);
|
|
4473
|
+
}
|
|
4474
|
+
if (opts.status) {
|
|
4475
|
+
filters.push("status = ?");
|
|
4476
|
+
params.push(opts.status);
|
|
4477
|
+
}
|
|
4478
|
+
if (opts.taskId) {
|
|
4479
|
+
filters.push("EXISTS (SELECT 1 FROM json_each(run_receipts.task_ids_json) WHERE value = ?)");
|
|
4480
|
+
params.push(opts.taskId);
|
|
4481
|
+
}
|
|
4482
|
+
if (opts.knowledgeId) {
|
|
4483
|
+
filters.push("EXISTS (SELECT 1 FROM json_each(run_receipts.knowledge_ids_json) WHERE value = ?)");
|
|
4484
|
+
params.push(opts.knowledgeId);
|
|
4485
|
+
}
|
|
4486
|
+
const where = filters.length ? `WHERE ${filters.join(" AND ")}` : "";
|
|
4487
|
+
const rows = this.db.query(`SELECT * FROM run_receipts ${where} ORDER BY created_at DESC LIMIT ?`).all(...params, limit);
|
|
4488
|
+
return rows.map(rowToRunReceipt);
|
|
4489
|
+
}
|
|
3789
4490
|
deferLiveExpiredRun(id, now, opts = {}) {
|
|
3790
4491
|
const updated = now.toISOString();
|
|
3791
4492
|
const deferredUntil = new Date(now.getTime() + LIVE_EXPIRED_RUN_GRACE_MS).toISOString();
|
|
@@ -3944,6 +4645,9 @@ class Store {
|
|
|
3944
4645
|
const runs = includeRuns ? this.db.query("SELECT * FROM loop_runs ORDER BY created_at ASC, id ASC").all().map(rowToRun) : [];
|
|
3945
4646
|
return { schemaVersion: SCHEMA_USER_VERSION, workflows, loops, runs, checks: this.migrationChecks() };
|
|
3946
4647
|
}
|
|
4648
|
+
exportMigrationRunPage(opts) {
|
|
4649
|
+
return this.db.query("SELECT * FROM loop_runs ORDER BY created_at ASC, id ASC LIMIT ? OFFSET ?").all(opts.limit, opts.offset).map(rowToRun);
|
|
4650
|
+
}
|
|
3947
4651
|
countTable(table) {
|
|
3948
4652
|
const row = this.db.query(`SELECT COUNT(*) AS count FROM ${table}`).get();
|
|
3949
4653
|
return row?.count ?? 0;
|
|
@@ -4190,9 +4894,9 @@ class Store {
|
|
|
4190
4894
|
const runDir = dirname2(manifestPath);
|
|
4191
4895
|
try {
|
|
4192
4896
|
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
4193
|
-
|
|
4897
|
+
rmSync3(runDir, { recursive: true, force: true });
|
|
4194
4898
|
} else {
|
|
4195
|
-
|
|
4899
|
+
rmSync3(manifestPath, { force: true });
|
|
4196
4900
|
}
|
|
4197
4901
|
} catch {}
|
|
4198
4902
|
}
|
|
@@ -4259,7 +4963,7 @@ class Store {
|
|
|
4259
4963
|
this.db.close();
|
|
4260
4964
|
if (this.memoryRootDir) {
|
|
4261
4965
|
try {
|
|
4262
|
-
|
|
4966
|
+
rmSync3(this.memoryRootDir, { recursive: true, force: true });
|
|
4263
4967
|
} catch {}
|
|
4264
4968
|
this.memoryRootDir = undefined;
|
|
4265
4969
|
}
|
|
@@ -4268,7 +4972,7 @@ class Store {
|
|
|
4268
4972
|
// package.json
|
|
4269
4973
|
var package_default = {
|
|
4270
4974
|
name: "@hasna/loops",
|
|
4271
|
-
version: "0.4.
|
|
4975
|
+
version: "0.4.22",
|
|
4272
4976
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4273
4977
|
type: "module",
|
|
4274
4978
|
main: "dist/index.js",
|
|
@@ -4277,6 +4981,7 @@ var package_default = {
|
|
|
4277
4981
|
loops: "dist/cli/index.js",
|
|
4278
4982
|
"loops-daemon": "dist/daemon/index.js",
|
|
4279
4983
|
"loops-api": "dist/api/index.js",
|
|
4984
|
+
"loops-serve": "dist/serve/index.js",
|
|
4280
4985
|
"loops-runner": "dist/runner/index.js",
|
|
4281
4986
|
"loops-mcp": "dist/mcp/index.js"
|
|
4282
4987
|
},
|
|
@@ -4289,6 +4994,14 @@ var package_default = {
|
|
|
4289
4994
|
types: "./dist/sdk/index.d.ts",
|
|
4290
4995
|
import: "./dist/sdk/index.js"
|
|
4291
4996
|
},
|
|
4997
|
+
"./sdk/http": {
|
|
4998
|
+
types: "./dist/sdk/http.d.ts",
|
|
4999
|
+
import: "./dist/sdk/http.js"
|
|
5000
|
+
},
|
|
5001
|
+
"./serve": {
|
|
5002
|
+
types: "./dist/serve/index.d.ts",
|
|
5003
|
+
import: "./dist/serve/index.js"
|
|
5004
|
+
},
|
|
4292
5005
|
"./mcp": {
|
|
4293
5006
|
types: "./dist/mcp/index.d.ts",
|
|
4294
5007
|
import: "./dist/mcp/index.js"
|
|
@@ -4334,7 +5047,7 @@ var package_default = {
|
|
|
4334
5047
|
"LICENSE"
|
|
4335
5048
|
],
|
|
4336
5049
|
scripts: {
|
|
4337
|
-
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
5050
|
+
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4338
5051
|
"build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
|
|
4339
5052
|
typecheck: "tsc --noEmit",
|
|
4340
5053
|
test: "bun test",
|
|
@@ -4374,13 +5087,16 @@ var package_default = {
|
|
|
4374
5087
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
4375
5088
|
ai: "6.0.204",
|
|
4376
5089
|
commander: "^13.1.0",
|
|
5090
|
+
pg: "^8.13.1",
|
|
4377
5091
|
zod: "4.4.3"
|
|
4378
5092
|
},
|
|
4379
5093
|
optionalDependencies: {
|
|
4380
|
-
"@hasna/machines": "0.0.49"
|
|
5094
|
+
"@hasna/machines": "0.0.49",
|
|
5095
|
+
"@hasna/contracts": "^0.4.2"
|
|
4381
5096
|
},
|
|
4382
5097
|
devDependencies: {
|
|
4383
5098
|
"@types/bun": "latest",
|
|
5099
|
+
"@types/pg": "^8.11.10",
|
|
4384
5100
|
typescript: "^5.7.3"
|
|
4385
5101
|
},
|
|
4386
5102
|
publishConfig: {
|
|
@@ -4591,13 +5307,109 @@ function deploymentStatusLine(status) {
|
|
|
4591
5307
|
// src/mcp/index.ts
|
|
4592
5308
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
4593
5309
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
5310
|
+
|
|
5311
|
+
// src/mcp/http.ts
|
|
5312
|
+
import { createServer } from "http";
|
|
5313
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
5314
|
+
var MCP_HTTP_SERVICE_NAME = "loops";
|
|
5315
|
+
var DEFAULT_MCP_HTTP_PORT = 8890;
|
|
5316
|
+
function isStdioMode(argv = process.argv, env = process.env) {
|
|
5317
|
+
return argv.includes("--stdio") || env.MCP_STDIO === "1";
|
|
5318
|
+
}
|
|
5319
|
+
function resolveMcpHttpPort(argv = process.argv, env = process.env) {
|
|
5320
|
+
const portIdx = argv.indexOf("--port");
|
|
5321
|
+
if (portIdx !== -1 && argv[portIdx + 1]) {
|
|
5322
|
+
return parsePort(argv[portIdx + 1], "--port");
|
|
5323
|
+
}
|
|
5324
|
+
if (env.MCP_HTTP_PORT) {
|
|
5325
|
+
return parsePort(env.MCP_HTTP_PORT, "MCP_HTTP_PORT");
|
|
5326
|
+
}
|
|
5327
|
+
return DEFAULT_MCP_HTTP_PORT;
|
|
5328
|
+
}
|
|
5329
|
+
function parsePort(raw, source) {
|
|
5330
|
+
const parsed = Number(raw);
|
|
5331
|
+
if (!Number.isInteger(parsed) || parsed < 0 || parsed > 65535) {
|
|
5332
|
+
throw new Error(`Invalid ${source} value "${raw}". Expected 0-65535.`);
|
|
5333
|
+
}
|
|
5334
|
+
return parsed;
|
|
5335
|
+
}
|
|
5336
|
+
async function readJsonBody(req) {
|
|
5337
|
+
const chunks = [];
|
|
5338
|
+
for await (const chunk of req) {
|
|
5339
|
+
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
|
5340
|
+
}
|
|
5341
|
+
const text = Buffer.concat(chunks).toString("utf8");
|
|
5342
|
+
if (!text)
|
|
5343
|
+
return;
|
|
5344
|
+
return JSON.parse(text);
|
|
5345
|
+
}
|
|
5346
|
+
async function startMcpHttpServer(buildServer, options) {
|
|
5347
|
+
const host = options?.host ?? "127.0.0.1";
|
|
5348
|
+
const requestedPort = options?.port ?? resolveMcpHttpPort();
|
|
5349
|
+
const serviceName = options?.serviceName ?? MCP_HTTP_SERVICE_NAME;
|
|
5350
|
+
const httpServer = createServer(async (req, res) => {
|
|
5351
|
+
try {
|
|
5352
|
+
const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
|
|
5353
|
+
if (req.method === "GET" && url.pathname === "/health") {
|
|
5354
|
+
res.writeHead(200, { "Content-Type": "application/json" });
|
|
5355
|
+
res.end(JSON.stringify({ status: "ok", name: serviceName }));
|
|
5356
|
+
return;
|
|
5357
|
+
}
|
|
5358
|
+
if (url.pathname !== "/mcp") {
|
|
5359
|
+
res.writeHead(404, { "Content-Type": "text/plain" });
|
|
5360
|
+
res.end("Not Found");
|
|
5361
|
+
return;
|
|
5362
|
+
}
|
|
5363
|
+
const server = buildServer();
|
|
5364
|
+
const transport = new StreamableHTTPServerTransport({
|
|
5365
|
+
sessionIdGenerator: undefined
|
|
5366
|
+
});
|
|
5367
|
+
await server.connect(transport);
|
|
5368
|
+
let parsedBody;
|
|
5369
|
+
if (req.method === "POST") {
|
|
5370
|
+
parsedBody = await readJsonBody(req);
|
|
5371
|
+
}
|
|
5372
|
+
await transport.handleRequest(req, res, parsedBody);
|
|
5373
|
+
res.on("close", () => {
|
|
5374
|
+
transport.close();
|
|
5375
|
+
server.close();
|
|
5376
|
+
});
|
|
5377
|
+
} catch (error) {
|
|
5378
|
+
console.error(`[${serviceName}-mcp] HTTP error:`, error);
|
|
5379
|
+
if (!res.headersSent) {
|
|
5380
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
5381
|
+
res.end(JSON.stringify({
|
|
5382
|
+
jsonrpc: "2.0",
|
|
5383
|
+
error: { code: -32603, message: "Internal server error" },
|
|
5384
|
+
id: null
|
|
5385
|
+
}));
|
|
5386
|
+
}
|
|
5387
|
+
}
|
|
5388
|
+
});
|
|
5389
|
+
await new Promise((resolve2, reject) => {
|
|
5390
|
+
httpServer.once("error", reject);
|
|
5391
|
+
httpServer.listen(requestedPort, host, () => resolve2());
|
|
5392
|
+
});
|
|
5393
|
+
const addr = httpServer.address();
|
|
5394
|
+
const port = typeof addr === "object" && addr ? addr.port : requestedPort;
|
|
5395
|
+
console.error(`[${serviceName}-mcp] Streamable HTTP listening on http://${host}:${port}/mcp`);
|
|
5396
|
+
return {
|
|
5397
|
+
port,
|
|
5398
|
+
host,
|
|
5399
|
+
close: () => new Promise((resolve2, reject) => {
|
|
5400
|
+
httpServer.close((err) => err ? reject(err) : resolve2());
|
|
5401
|
+
})
|
|
5402
|
+
};
|
|
5403
|
+
}
|
|
5404
|
+
|
|
5405
|
+
// src/mcp/index.ts
|
|
4594
5406
|
import { z as z2 } from "zod/v3";
|
|
4595
5407
|
|
|
4596
5408
|
// src/daemon/control.ts
|
|
4597
|
-
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as
|
|
4598
|
-
import { spawnSync as
|
|
4599
|
-
import { hostname } from "os";
|
|
4600
|
-
import { dirname as dirname3, join as
|
|
5409
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
5410
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
5411
|
+
import { hostname as hostname2 } from "os";
|
|
5412
|
+
import { dirname as dirname3, join as join5 } from "path";
|
|
4601
5413
|
|
|
4602
5414
|
// src/daemon/loop.ts
|
|
4603
5415
|
function realSleep(ms) {
|
|
@@ -4627,7 +5439,7 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
4627
5439
|
return;
|
|
4628
5440
|
let raw;
|
|
4629
5441
|
try {
|
|
4630
|
-
raw =
|
|
5442
|
+
raw = readFileSync4(path, "utf8").trim();
|
|
4631
5443
|
} catch {
|
|
4632
5444
|
return;
|
|
4633
5445
|
}
|
|
@@ -4653,7 +5465,7 @@ function writePid(pid = process.pid, path = pidFilePath()) {
|
|
|
4653
5465
|
writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
4654
5466
|
}
|
|
4655
5467
|
function removePid(path = pidFilePath()) {
|
|
4656
|
-
|
|
5468
|
+
rmSync4(path, { force: true });
|
|
4657
5469
|
}
|
|
4658
5470
|
function isAlive(pid, startedAt) {
|
|
4659
5471
|
try {
|
|
@@ -4683,7 +5495,7 @@ function ownProcessGroupId() {
|
|
|
4683
5495
|
return pgrp;
|
|
4684
5496
|
}
|
|
4685
5497
|
try {
|
|
4686
|
-
const run =
|
|
5498
|
+
const run = spawnSync3("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
4687
5499
|
const pgid = Number(run.stdout.trim());
|
|
4688
5500
|
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
4689
5501
|
return pgid;
|
|
@@ -4749,7 +5561,7 @@ function daemonStatus(store, path = pidFilePath()) {
|
|
|
4749
5561
|
return {
|
|
4750
5562
|
...isDaemonRunning(path),
|
|
4751
5563
|
lease: store.getDaemonLease(),
|
|
4752
|
-
host:
|
|
5564
|
+
host: hostname2(),
|
|
4753
5565
|
loops: {
|
|
4754
5566
|
total: store.countLoops(),
|
|
4755
5567
|
active: store.countLoops("active"),
|
|
@@ -4779,7 +5591,7 @@ async function stopDaemon(opts = {}) {
|
|
|
4779
5591
|
if (!state.running || !state.pid)
|
|
4780
5592
|
return { wasRunning: false, stopped: false, forced: false };
|
|
4781
5593
|
const sleep = opts.sleep ?? realSleep;
|
|
4782
|
-
const store = new Store(
|
|
5594
|
+
const store = new Store(join5(dirname3(path), "loops.db"));
|
|
4783
5595
|
try {
|
|
4784
5596
|
const lease = store.getDaemonLease();
|
|
4785
5597
|
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
@@ -4817,18 +5629,18 @@ async function stopDaemon(opts = {}) {
|
|
|
4817
5629
|
}
|
|
4818
5630
|
|
|
4819
5631
|
// src/lib/doctor.ts
|
|
4820
|
-
import { spawnSync as
|
|
5632
|
+
import { spawnSync as spawnSync6 } from "child_process";
|
|
4821
5633
|
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
4822
5634
|
|
|
4823
5635
|
// src/lib/executor.ts
|
|
4824
|
-
import { spawn as spawn2, spawnSync as
|
|
5636
|
+
import { spawn as spawn2, spawnSync as spawnSync5 } from "child_process";
|
|
4825
5637
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
4826
5638
|
import { once } from "events";
|
|
4827
5639
|
import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
|
|
4828
5640
|
import { dirname as dirname4, resolve as resolve2 } from "path";
|
|
4829
5641
|
|
|
4830
5642
|
// src/lib/accounts.ts
|
|
4831
|
-
import { spawnSync as
|
|
5643
|
+
import { spawnSync as spawnSync4 } from "child_process";
|
|
4832
5644
|
import { existsSync as existsSync3 } from "fs";
|
|
4833
5645
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4834
5646
|
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
@@ -4933,7 +5745,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4933
5745
|
if (!account)
|
|
4934
5746
|
return {};
|
|
4935
5747
|
const tool = requiredAccountTool(account, toolHint);
|
|
4936
|
-
const result =
|
|
5748
|
+
const result = spawnSync4("accounts", ["env", account.profile, "--tool", tool], {
|
|
4937
5749
|
encoding: "utf8",
|
|
4938
5750
|
env,
|
|
4939
5751
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -4949,8 +5761,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4949
5761
|
|
|
4950
5762
|
// src/lib/env.ts
|
|
4951
5763
|
import { accessSync, constants } from "fs";
|
|
4952
|
-
import { homedir as
|
|
4953
|
-
import { delimiter, join as
|
|
5764
|
+
import { homedir as homedir3 } from "os";
|
|
5765
|
+
import { delimiter, join as join6 } from "path";
|
|
4954
5766
|
function compactPathParts(parts) {
|
|
4955
5767
|
const seen = new Set;
|
|
4956
5768
|
const result = [];
|
|
@@ -4964,16 +5776,16 @@ function compactPathParts(parts) {
|
|
|
4964
5776
|
return result;
|
|
4965
5777
|
}
|
|
4966
5778
|
function commonExecutableDirs(env = process.env) {
|
|
4967
|
-
const home = env.HOME ||
|
|
5779
|
+
const home = env.HOME || homedir3();
|
|
4968
5780
|
return compactPathParts([
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
4972
|
-
|
|
4973
|
-
|
|
4974
|
-
env.BUN_INSTALL ?
|
|
5781
|
+
join6(home, ".local", "bin"),
|
|
5782
|
+
join6(home, ".bun", "bin"),
|
|
5783
|
+
join6(home, ".cargo", "bin"),
|
|
5784
|
+
join6(home, ".npm-global", "bin"),
|
|
5785
|
+
join6(home, "bin"),
|
|
5786
|
+
env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
|
|
4975
5787
|
env.PNPM_HOME,
|
|
4976
|
-
env.NPM_CONFIG_PREFIX ?
|
|
5788
|
+
env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
4977
5789
|
"/opt/homebrew/bin",
|
|
4978
5790
|
"/usr/local/bin",
|
|
4979
5791
|
"/usr/bin",
|
|
@@ -4997,7 +5809,7 @@ function executableExists(command, env = process.env) {
|
|
|
4997
5809
|
if (command.includes("/"))
|
|
4998
5810
|
return isExecutable(command);
|
|
4999
5811
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
5000
|
-
if (dir && isExecutable(
|
|
5812
|
+
if (dir && isExecutable(join6(dir, command)))
|
|
5001
5813
|
return true;
|
|
5002
5814
|
}
|
|
5003
5815
|
return false;
|
|
@@ -5106,6 +5918,7 @@ var TRANSPORT_ENV_KEYS = new Set([
|
|
|
5106
5918
|
"LOGNAME",
|
|
5107
5919
|
"PATH",
|
|
5108
5920
|
"SHELL",
|
|
5921
|
+
"SHLVL",
|
|
5109
5922
|
"SSH_AGENT_PID",
|
|
5110
5923
|
"SSH_AUTH_SOCK",
|
|
5111
5924
|
"TERM",
|
|
@@ -5138,6 +5951,32 @@ function timeoutResult(startedAt, error, fields = {}) {
|
|
|
5138
5951
|
function successResult(startedAt, fields = {}) {
|
|
5139
5952
|
return buildResult("succeeded", startedAt, fields);
|
|
5140
5953
|
}
|
|
5954
|
+
function codewithJsonlHasTerminalSuccess(stdout) {
|
|
5955
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
5956
|
+
const trimmed = line.trim();
|
|
5957
|
+
if (!trimmed || !trimmed.startsWith("{"))
|
|
5958
|
+
continue;
|
|
5959
|
+
let event;
|
|
5960
|
+
try {
|
|
5961
|
+
event = JSON.parse(trimmed);
|
|
5962
|
+
} catch {
|
|
5963
|
+
continue;
|
|
5964
|
+
}
|
|
5965
|
+
if (!event || typeof event !== "object")
|
|
5966
|
+
continue;
|
|
5967
|
+
const record = event;
|
|
5968
|
+
if (record.type === "task_complete")
|
|
5969
|
+
return true;
|
|
5970
|
+
const payload = record.payload;
|
|
5971
|
+
if (payload && typeof payload === "object" && payload.type === "task_complete") {
|
|
5972
|
+
return true;
|
|
5973
|
+
}
|
|
5974
|
+
}
|
|
5975
|
+
return false;
|
|
5976
|
+
}
|
|
5977
|
+
function codewithJsonlReconciledSuccess(spec, fields) {
|
|
5978
|
+
return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
|
|
5979
|
+
}
|
|
5141
5980
|
function notifySpawn(pid, opts) {
|
|
5142
5981
|
if (!pid)
|
|
5143
5982
|
return;
|
|
@@ -5153,10 +5992,48 @@ function shellQuote(value) {
|
|
|
5153
5992
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
5154
5993
|
}
|
|
5155
5994
|
function codewithProfileCandidateFromLine(line) {
|
|
5156
|
-
const
|
|
5157
|
-
if (
|
|
5158
|
-
return
|
|
5159
|
-
|
|
5995
|
+
const trimmed = line.trim();
|
|
5996
|
+
if (!trimmed || trimmed === "No auth profiles saved.")
|
|
5997
|
+
return;
|
|
5998
|
+
const cols = trimmed.split(/\s+/);
|
|
5999
|
+
const candidate = cols[0] === "*" ? cols[1] : cols[0];
|
|
6000
|
+
if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
|
|
6001
|
+
return;
|
|
6002
|
+
if (candidate === '"name":') {
|
|
6003
|
+
const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
|
|
6004
|
+
if (jsonName)
|
|
6005
|
+
return jsonName;
|
|
6006
|
+
}
|
|
6007
|
+
return candidate;
|
|
6008
|
+
}
|
|
6009
|
+
function codewithProfileCandidatesFromJson(value) {
|
|
6010
|
+
const candidates = [];
|
|
6011
|
+
const root = value && typeof value === "object" ? value : undefined;
|
|
6012
|
+
const addProfile = (entry) => {
|
|
6013
|
+
if (!entry || typeof entry !== "object")
|
|
6014
|
+
return;
|
|
6015
|
+
const name = entry.name;
|
|
6016
|
+
if (typeof name === "string" && name)
|
|
6017
|
+
candidates.push(name);
|
|
6018
|
+
};
|
|
6019
|
+
const data = root?.data;
|
|
6020
|
+
if (Array.isArray(data))
|
|
6021
|
+
data.forEach(addProfile);
|
|
6022
|
+
const profiles = root?.profiles;
|
|
6023
|
+
if (Array.isArray(profiles))
|
|
6024
|
+
profiles.forEach(addProfile);
|
|
6025
|
+
return candidates;
|
|
6026
|
+
}
|
|
6027
|
+
function codewithProfileCandidatesFromOutput(output) {
|
|
6028
|
+
const trimmed = output.trim();
|
|
6029
|
+
const jsonStart = trimmed.indexOf("{");
|
|
6030
|
+
const jsonEnd = trimmed.lastIndexOf("}");
|
|
6031
|
+
if (jsonStart >= 0 && jsonEnd >= jsonStart) {
|
|
6032
|
+
try {
|
|
6033
|
+
return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
|
|
6034
|
+
} catch {}
|
|
6035
|
+
}
|
|
6036
|
+
return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
|
|
5160
6037
|
}
|
|
5161
6038
|
function codewithProfileForError(profile) {
|
|
5162
6039
|
return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
|
|
@@ -5249,6 +6126,7 @@ function commandSpec(target, opts) {
|
|
|
5249
6126
|
preflightAnyOf: invocation.preflightAnyOf,
|
|
5250
6127
|
stdin: invocation.stdin,
|
|
5251
6128
|
allowlist: agentTarget.allowlist,
|
|
6129
|
+
agentProvider: agentTarget.provider,
|
|
5252
6130
|
worktree: agentTarget.worktree
|
|
5253
6131
|
};
|
|
5254
6132
|
}
|
|
@@ -5262,6 +6140,7 @@ function composeExecutionEnv(spec, metadata, opts, accountEnv) {
|
|
|
5262
6140
|
Object.assign(env, spec.env ?? {});
|
|
5263
6141
|
Object.assign(env, allowlistEnv(spec.allowlist));
|
|
5264
6142
|
env.PATH = normalizeExecutionPath(env);
|
|
6143
|
+
env.SHLVL ||= "1";
|
|
5265
6144
|
Object.assign(env, metadataEnv(metadata));
|
|
5266
6145
|
return env;
|
|
5267
6146
|
}
|
|
@@ -5333,7 +6212,7 @@ function remoteWorktreePrepareLines(worktree) {
|
|
|
5333
6212
|
return [
|
|
5334
6213
|
"__openloops_prepare_worktree() {",
|
|
5335
6214
|
` local repo=${shellQuote(repoRoot)} path=${shellQuote(path)} branch=${shellQuote(branch)}`,
|
|
5336
|
-
" local top expected_common actual_common current",
|
|
6215
|
+
" local top expected_common actual_common current status recovered",
|
|
5337
6216
|
' if [ -L "$path" ]; then echo "refusing symlinked worktree path $path" >&2; return 1; fi',
|
|
5338
6217
|
' if [ -e "$path" ]; then',
|
|
5339
6218
|
' top="$(git -C "$path" rev-parse --show-toplevel 2>/dev/null)" || { echo "refusing to reuse non-worktree path: $path" >&2; return 1; }',
|
|
@@ -5342,7 +6221,19 @@ function remoteWorktreePrepareLines(worktree) {
|
|
|
5342
6221
|
' actual_common="$(cd "$path" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
|
|
5343
6222
|
' if [ -z "$expected_common" ] || [ "$expected_common" != "$actual_common" ]; then echo "existing worktree $path belongs to a different git common dir" >&2; return 1; fi',
|
|
5344
6223
|
' current="$(git -C "$path" branch --show-current 2>/dev/null)"',
|
|
5345
|
-
' if [ "$current" != "$branch" ]; then
|
|
6224
|
+
' if [ "$current" != "$branch" ]; then',
|
|
6225
|
+
' status="$(git -C "$path" status --porcelain=v1 --untracked-files=all 2>/dev/null)" || { echo "existing worktree $path is on branch ${current:-unknown}, expected $branch, and cleanliness could not be checked; inspect or remove the stale worktree" >&2; return 1; }',
|
|
6226
|
+
' if [ -n "$status" ]; then echo "existing worktree $path is on branch ${current:-unknown}, expected $branch, and has local changes; commit/stash them or remove the stale worktree before retrying" >&2; return 1; fi',
|
|
6227
|
+
' if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
|
|
6228
|
+
' git -C "$path" checkout "$branch" 1>&2 || { echo "existing worktree $path is clean but could not switch from branch ${current:-unknown} to expected $branch; remove/prune the stale worktree or free the branch before retrying" >&2; return 1; }',
|
|
6229
|
+
' elif [ -z "$current" ]; then',
|
|
6230
|
+
' git -C "$path" checkout -b "$branch" 1>&2 || { echo "existing worktree $path is clean but could not recreate expected branch $branch at the current detached HEAD; remove/prune the stale worktree before retrying" >&2; return 1; }',
|
|
6231
|
+
" else",
|
|
6232
|
+
' echo "existing worktree $path is on branch $current, expected $branch, but expected branch does not exist; remove/prune the stale worktree or recreate the expected branch before retrying" >&2; return 1',
|
|
6233
|
+
" fi",
|
|
6234
|
+
' recovered="$(git -C "$path" branch --show-current 2>/dev/null)"',
|
|
6235
|
+
' if [ "$recovered" != "$branch" ]; then echo "existing worktree $path branch recovery ended on ${recovered:-unknown}, expected $branch; remove/prune the stale worktree before retrying" >&2; return 1; fi',
|
|
6236
|
+
" fi",
|
|
5346
6237
|
" return 0",
|
|
5347
6238
|
" fi",
|
|
5348
6239
|
' git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
|
|
@@ -5408,7 +6299,7 @@ function remotePreflightScript(spec, metadata) {
|
|
|
5408
6299
|
}
|
|
5409
6300
|
if (spec.nativeAuthProfile?.provider === "codewith") {
|
|
5410
6301
|
const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
|
|
5411
|
-
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE",
|
|
6302
|
+
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
|
|
5412
6303
|
}
|
|
5413
6304
|
return lines.join(`
|
|
5414
6305
|
`);
|
|
@@ -5423,9 +6314,28 @@ function transportEnv(opts) {
|
|
|
5423
6314
|
env[key] = value;
|
|
5424
6315
|
}
|
|
5425
6316
|
env.PATH = normalizeExecutionPath(env);
|
|
6317
|
+
env.SHLVL ||= "1";
|
|
5426
6318
|
return env;
|
|
5427
6319
|
}
|
|
6320
|
+
function remotePreflightFailureMessage(machineId, detail) {
|
|
6321
|
+
const normalized = detail.trim();
|
|
6322
|
+
if (/Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED|Offending .*key in .*known_hosts/i.test(normalized)) {
|
|
6323
|
+
return [
|
|
6324
|
+
`remote preflight failed on ${machineId}: SSH host key verification failed.`,
|
|
6325
|
+
`Verify ${machineId}'s host identity and repair SSH known_hosts/trust material outside OpenLoops;`,
|
|
6326
|
+
"OpenLoops will not disable host-key checking or modify known_hosts automatically.",
|
|
6327
|
+
normalized ? `Transport detail: ${normalized}` : ""
|
|
6328
|
+
].filter(Boolean).join(" ");
|
|
6329
|
+
}
|
|
6330
|
+
return `remote preflight failed on ${machineId}${normalized ? `: ${normalized}` : ""}`;
|
|
6331
|
+
}
|
|
5428
6332
|
function assertCodewithProfileListed(profile, result) {
|
|
6333
|
+
const profiles = codewithProfileSetFromResult(result);
|
|
6334
|
+
if (!profiles.has(profile)) {
|
|
6335
|
+
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
6336
|
+
}
|
|
6337
|
+
}
|
|
6338
|
+
function codewithProfileSetFromResult(result) {
|
|
5429
6339
|
if (result.error) {
|
|
5430
6340
|
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
5431
6341
|
}
|
|
@@ -5433,32 +6343,45 @@ function assertCodewithProfileListed(profile, result) {
|
|
|
5433
6343
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5434
6344
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
5435
6345
|
}
|
|
5436
|
-
|
|
5437
|
-
if (!profiles.has(profile)) {
|
|
5438
|
-
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5439
|
-
}
|
|
6346
|
+
return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
|
|
5440
6347
|
}
|
|
5441
6348
|
function preflightNativeAuthProfileSync(spec, env) {
|
|
5442
6349
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5443
6350
|
return;
|
|
5444
|
-
const
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
5449
|
-
|
|
5450
|
-
|
|
5451
|
-
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
6351
|
+
const runProfileList = (args) => {
|
|
6352
|
+
const result = spawnSync5(spec.command, args, {
|
|
6353
|
+
encoding: "utf8",
|
|
6354
|
+
env,
|
|
6355
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
6356
|
+
timeout: 15000
|
|
6357
|
+
});
|
|
6358
|
+
return {
|
|
6359
|
+
status: result.status,
|
|
6360
|
+
stdout: result.stdout ?? "",
|
|
6361
|
+
stderr: result.stderr ?? "",
|
|
6362
|
+
error: result.error?.message
|
|
6363
|
+
};
|
|
6364
|
+
};
|
|
6365
|
+
const jsonResult = runProfileList(["profile", "list", "--json"]);
|
|
6366
|
+
try {
|
|
6367
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
6368
|
+
return;
|
|
6369
|
+
} catch {}
|
|
6370
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
|
|
5456
6371
|
}
|
|
5457
6372
|
async function preflightNativeAuthProfile(spec, env) {
|
|
5458
6373
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5459
6374
|
return;
|
|
5460
|
-
const
|
|
5461
|
-
|
|
6375
|
+
const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
|
|
6376
|
+
try {
|
|
6377
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
6378
|
+
return;
|
|
6379
|
+
} catch {}
|
|
6380
|
+
const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
6381
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
|
|
6382
|
+
}
|
|
6383
|
+
function spawnDetail(result) {
|
|
6384
|
+
return (result.stderr || result.stdout || result.error || "").toString().trim();
|
|
5462
6385
|
}
|
|
5463
6386
|
function resolvedDirEquals(left, right) {
|
|
5464
6387
|
try {
|
|
@@ -5509,8 +6432,45 @@ async function ensureLocalWorktree(worktree, env) {
|
|
|
5509
6432
|
}
|
|
5510
6433
|
const current = await git(["-C", path, "branch", "--show-current"]);
|
|
5511
6434
|
const actualBranch = current.stdout.trim();
|
|
5512
|
-
if (current.error || (current.status ?? 1) !== 0
|
|
5513
|
-
return { error: `existing worktree ${path}
|
|
6435
|
+
if (current.error || (current.status ?? 1) !== 0) {
|
|
6436
|
+
return { error: `existing worktree ${path} branch could not be inspected${spawnDetail(current) ? `: ${spawnDetail(current)}` : ""}` };
|
|
6437
|
+
}
|
|
6438
|
+
if (actualBranch !== branch) {
|
|
6439
|
+
const actualLabel = actualBranch || "unknown";
|
|
6440
|
+
const status = await git(["-C", path, "status", "--porcelain=v1", "--untracked-files=all"]);
|
|
6441
|
+
if (status.error || (status.status ?? 1) !== 0) {
|
|
6442
|
+
const detail = spawnDetail(status);
|
|
6443
|
+
return {
|
|
6444
|
+
error: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, and cleanliness could not be checked; inspect or remove the stale worktree${detail ? `: ${detail}` : ""}`
|
|
6445
|
+
};
|
|
6446
|
+
}
|
|
6447
|
+
if (status.stdout.trim()) {
|
|
6448
|
+
return {
|
|
6449
|
+
error: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, and has local changes; commit/stash them or remove the stale worktree before retrying`
|
|
6450
|
+
};
|
|
6451
|
+
}
|
|
6452
|
+
const hasBranch2 = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
6453
|
+
const checkout = hasBranch2.status === 0 ? await git(["-C", path, "checkout", branch]) : actualBranch ? {
|
|
6454
|
+
status: 1,
|
|
6455
|
+
stdout: "",
|
|
6456
|
+
stderr: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, but expected branch does not exist; remove/prune the stale worktree or recreate the expected branch before retrying`,
|
|
6457
|
+
signal: null,
|
|
6458
|
+
timedOut: false
|
|
6459
|
+
} : await git(["-C", path, "checkout", "-b", branch]);
|
|
6460
|
+
if (checkout.error || (checkout.status ?? 1) !== 0) {
|
|
6461
|
+
const detail = spawnDetail(checkout);
|
|
6462
|
+
return {
|
|
6463
|
+
error: `existing worktree ${path} is clean but could not recover branch ${branch} from ${actualLabel}; remove/prune the stale worktree before retrying${detail ? `: ${detail}` : ""}`
|
|
6464
|
+
};
|
|
6465
|
+
}
|
|
6466
|
+
const recovered = await git(["-C", path, "branch", "--show-current"]);
|
|
6467
|
+
if (recovered.error || (recovered.status ?? 1) !== 0 || recovered.stdout.trim() !== branch) {
|
|
6468
|
+
const recoveredBranch = recovered.stdout.trim() || "unknown";
|
|
6469
|
+
const detail = spawnDetail(recovered);
|
|
6470
|
+
return {
|
|
6471
|
+
error: `existing worktree ${path} branch recovery ended on ${recoveredBranch}, expected ${branch}; remove/prune the stale worktree before retrying${detail ? `: ${detail}` : ""}`
|
|
6472
|
+
};
|
|
6473
|
+
}
|
|
5514
6474
|
}
|
|
5515
6475
|
return { cwd: worktree.cwd };
|
|
5516
6476
|
}
|
|
@@ -5526,7 +6486,7 @@ async function ensureLocalWorktree(worktree, env) {
|
|
|
5526
6486
|
const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
5527
6487
|
const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
|
|
5528
6488
|
if (add.error || (add.status ?? 1) !== 0) {
|
|
5529
|
-
const detail = (add
|
|
6489
|
+
const detail = spawnDetail(add);
|
|
5530
6490
|
return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
|
|
5531
6491
|
}
|
|
5532
6492
|
return { cwd: worktree.cwd };
|
|
@@ -5561,7 +6521,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
|
5561
6521
|
}
|
|
5562
6522
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
5563
6523
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
5564
|
-
const result =
|
|
6524
|
+
const result = spawnSync5(plan.command, plan.args, {
|
|
5565
6525
|
encoding: "utf8",
|
|
5566
6526
|
env: transportEnv(opts),
|
|
5567
6527
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -5572,7 +6532,7 @@ function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
|
5572
6532
|
throw new Error(`remote preflight failed on ${machine.id}: ${result.error.message}`);
|
|
5573
6533
|
if ((result.status ?? 1) !== 0) {
|
|
5574
6534
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5575
|
-
throw new Error(
|
|
6535
|
+
throw new Error(remotePreflightFailureMessage(machine.id, detail));
|
|
5576
6536
|
}
|
|
5577
6537
|
}
|
|
5578
6538
|
async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
@@ -5660,6 +6620,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
|
5660
6620
|
if (timedOut || idleTimedOut) {
|
|
5661
6621
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5662
6622
|
}
|
|
6623
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6624
|
+
return successResult(startedAt, fields);
|
|
6625
|
+
}
|
|
5663
6626
|
if (error || exitCode !== 0) {
|
|
5664
6627
|
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5665
6628
|
}
|
|
@@ -5795,6 +6758,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
5795
6758
|
if (timedOut || idleTimedOut) {
|
|
5796
6759
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5797
6760
|
}
|
|
6761
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6762
|
+
return successResult(startedAt, fields);
|
|
6763
|
+
}
|
|
5798
6764
|
if (error || exitCode !== 0) {
|
|
5799
6765
|
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5800
6766
|
}
|
|
@@ -5825,320 +6791,392 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
5825
6791
|
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5826
6792
|
}
|
|
5827
6793
|
|
|
5828
|
-
// src/lib/
|
|
5829
|
-
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
5833
|
-
|
|
5834
|
-
|
|
5835
|
-
|
|
6794
|
+
// src/lib/health.ts
|
|
6795
|
+
import { createHash as createHash3 } from "crypto";
|
|
6796
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
6797
|
+
import { join as join7 } from "path";
|
|
6798
|
+
var EVIDENCE_CHARS = 2000;
|
|
6799
|
+
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6800
|
+
var DEFAULT_SCAN_LIMIT = 200;
|
|
6801
|
+
var DEFAULT_SCAN_STATUSES = ["active", "paused"];
|
|
6802
|
+
var DEFAULT_MAX_FINDINGS = 100;
|
|
6803
|
+
var MIN_STALE_RUNNING_MS = 10 * 60000;
|
|
6804
|
+
var CLASSIFICATIONS = [
|
|
6805
|
+
"rate_limit",
|
|
6806
|
+
"auth",
|
|
6807
|
+
"provider_unavailable",
|
|
6808
|
+
"model_not_found",
|
|
6809
|
+
"context_length",
|
|
6810
|
+
"schema_response_format",
|
|
6811
|
+
"node_init",
|
|
6812
|
+
"preflight",
|
|
6813
|
+
"route_functional",
|
|
6814
|
+
"timeout",
|
|
6815
|
+
"sigsegv",
|
|
6816
|
+
"restart_interrupted",
|
|
6817
|
+
"skipped_previous_active",
|
|
6818
|
+
"circuit_breaker",
|
|
6819
|
+
"unknown"
|
|
5836
6820
|
];
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
6821
|
+
var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
|
|
6822
|
+
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
6823
|
+
if (!value)
|
|
6824
|
+
return;
|
|
6825
|
+
if (value.length <= limit)
|
|
6826
|
+
return value;
|
|
6827
|
+
return `${value.slice(0, limit)}
|
|
6828
|
+
[truncated ${value.length - limit} chars]`;
|
|
5840
6829
|
}
|
|
5841
|
-
function
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
|
|
5845
|
-
|
|
5846
|
-
|
|
6830
|
+
function redactedEvidence(value) {
|
|
6831
|
+
return redact(bounded(value));
|
|
6832
|
+
}
|
|
6833
|
+
function searchableText(run) {
|
|
6834
|
+
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
6835
|
+
`);
|
|
6836
|
+
}
|
|
6837
|
+
function isRecord(value) {
|
|
6838
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
6839
|
+
}
|
|
6840
|
+
function stringValue(value) {
|
|
6841
|
+
return typeof value === "string" && value.trim() ? value : undefined;
|
|
6842
|
+
}
|
|
6843
|
+
function objectField(value, key) {
|
|
6844
|
+
if (!value)
|
|
5847
6845
|
return;
|
|
5848
|
-
|
|
6846
|
+
const field = value[key];
|
|
6847
|
+
return isRecord(field) ? field : undefined;
|
|
5849
6848
|
}
|
|
5850
|
-
function
|
|
5851
|
-
|
|
5852
|
-
|
|
5853
|
-
|
|
5854
|
-
|
|
5855
|
-
|
|
5856
|
-
} catch (error) {
|
|
5857
|
-
checks.push({
|
|
5858
|
-
id: "data-dir",
|
|
5859
|
-
status: "fail",
|
|
5860
|
-
message: "data directory is not writable",
|
|
5861
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5862
|
-
});
|
|
5863
|
-
}
|
|
5864
|
-
const bunVersion = commandVersion("bun");
|
|
5865
|
-
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
5866
|
-
const accountsVersion = commandVersion("accounts");
|
|
5867
|
-
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
5868
|
-
try {
|
|
5869
|
-
const machines = listOpenMachines();
|
|
5870
|
-
const local = machines.find((machine) => machine.local);
|
|
5871
|
-
checks.push({
|
|
5872
|
-
id: "machines",
|
|
5873
|
-
status: "ok",
|
|
5874
|
-
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
5875
|
-
detail: local ? `local=${local.id}` : undefined
|
|
5876
|
-
});
|
|
5877
|
-
} catch (error) {
|
|
5878
|
-
checks.push({
|
|
5879
|
-
id: "machines",
|
|
5880
|
-
status: "warn",
|
|
5881
|
-
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
5882
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5883
|
-
});
|
|
5884
|
-
}
|
|
5885
|
-
for (const command of PROVIDER_COMMANDS) {
|
|
5886
|
-
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
5887
|
-
}
|
|
5888
|
-
const status = daemonStatus(store);
|
|
5889
|
-
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
5890
|
-
const failedRuns = store.countRuns("failed");
|
|
5891
|
-
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
5892
|
-
const deployment = buildDeploymentStatus();
|
|
5893
|
-
const schedulerState = deployment.schedulerState;
|
|
5894
|
-
checks.push({
|
|
5895
|
-
id: "scheduler-state",
|
|
5896
|
-
status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
|
|
5897
|
-
message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
|
|
5898
|
-
detail: [
|
|
5899
|
-
`route_state=${schedulerState.routeAdmission.stateStore}`,
|
|
5900
|
-
`active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
|
|
5901
|
-
`gates=${schedulerState.routeAdmission.gates.join(",")}`,
|
|
5902
|
-
`artifacts=${schedulerState.localStore.runArtifacts}`,
|
|
5903
|
-
`remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
|
|
5904
|
-
`remote_apply=${String(schedulerState.remoteStore.applySupported)}`
|
|
5905
|
-
].join(" ")
|
|
5906
|
-
});
|
|
5907
|
-
for (const loop of store.listLoops({ status: "active" })) {
|
|
5908
|
-
try {
|
|
5909
|
-
if (loop.target.type === "workflow") {
|
|
5910
|
-
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
5911
|
-
for (const step of workflowExecutionOrder(workflow)) {
|
|
5912
|
-
preflightTarget({
|
|
5913
|
-
...step.target,
|
|
5914
|
-
account: step.account ?? step.target.account,
|
|
5915
|
-
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
5916
|
-
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
5917
|
-
}
|
|
5918
|
-
} else {
|
|
5919
|
-
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
5920
|
-
}
|
|
5921
|
-
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
5922
|
-
} catch (error) {
|
|
5923
|
-
checks.push({
|
|
5924
|
-
id: `loop:${loop.id}:preflight`,
|
|
5925
|
-
status: "fail",
|
|
5926
|
-
message: `active loop target preflight failed: ${loop.name}`,
|
|
5927
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5928
|
-
});
|
|
5929
|
-
}
|
|
5930
|
-
}
|
|
5931
|
-
return {
|
|
5932
|
-
ok: checks.every((check) => check.status !== "fail"),
|
|
5933
|
-
checks
|
|
5934
|
-
};
|
|
6849
|
+
function tagsFromValue(value) {
|
|
6850
|
+
if (Array.isArray(value))
|
|
6851
|
+
return value.map((entry) => String(entry).trim()).filter(Boolean);
|
|
6852
|
+
if (typeof value === "string")
|
|
6853
|
+
return value.split(",").map((entry) => entry.trim()).filter(Boolean);
|
|
6854
|
+
return [];
|
|
5935
6855
|
}
|
|
5936
|
-
|
|
5937
|
-
|
|
5938
|
-
|
|
5939
|
-
|
|
5940
|
-
function
|
|
6856
|
+
function stableFingerprint(parts) {
|
|
6857
|
+
return createHash3("sha256").update(parts.join(`
|
|
6858
|
+
`)).digest("hex").slice(0, 16);
|
|
6859
|
+
}
|
|
6860
|
+
function stableScanFingerprint(parts) {
|
|
6861
|
+
return `openloops:health-scan:${stableFingerprint(parts)}`;
|
|
6862
|
+
}
|
|
6863
|
+
function safeHost(value) {
|
|
5941
6864
|
if (!value)
|
|
5942
|
-
return
|
|
5943
|
-
|
|
5944
|
-
|
|
5945
|
-
|
|
5946
|
-
return `[redacted ${value.length} chars]`;
|
|
5947
|
-
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
6865
|
+
return;
|
|
6866
|
+
let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
|
|
6867
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
6868
|
+
return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
|
|
5948
6869
|
}
|
|
5949
|
-
function
|
|
5950
|
-
|
|
5951
|
-
return value;
|
|
5952
|
-
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
5953
|
-
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
6870
|
+
function isCursorHost(host) {
|
|
6871
|
+
return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
|
|
5954
6872
|
}
|
|
5955
|
-
function
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
if (
|
|
5960
|
-
return
|
|
5961
|
-
return "[redacted]";
|
|
5962
|
-
}
|
|
5963
|
-
if (Array.isArray(value))
|
|
5964
|
-
return value.map((item) => redactSensitivePayload(item));
|
|
5965
|
-
if (value && typeof value === "object") {
|
|
5966
|
-
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
6873
|
+
function providerUnavailableSummary(rawText) {
|
|
6874
|
+
const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
|
|
6875
|
+
if (dns) {
|
|
6876
|
+
const host = safeHost(dns[2]);
|
|
6877
|
+
if (isCursorHost(host))
|
|
6878
|
+
return `provider DNS lookup failed: ${dns[1]} ${host}`;
|
|
5967
6879
|
}
|
|
5968
|
-
return
|
|
6880
|
+
return;
|
|
5969
6881
|
}
|
|
5970
|
-
function
|
|
5971
|
-
const
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
5979
|
-
continue;
|
|
5980
|
-
blocks.push(`${indent}${label}:`);
|
|
5981
|
-
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
5982
|
-
blocks.push(`${nested}${line}`);
|
|
5983
|
-
}
|
|
5984
|
-
}
|
|
5985
|
-
return blocks;
|
|
6882
|
+
function stableFailureFingerprint(run, classification, summary) {
|
|
6883
|
+
const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
|
|
6884
|
+
return stableFingerprint([
|
|
6885
|
+
run.loopId,
|
|
6886
|
+
classification,
|
|
6887
|
+
String(run.status),
|
|
6888
|
+
String(run.exitCode ?? ""),
|
|
6889
|
+
evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6890
|
+
]);
|
|
5986
6891
|
}
|
|
5987
|
-
function
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5992
|
-
|
|
6892
|
+
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
6893
|
+
return stableFingerprint([
|
|
6894
|
+
loop.id,
|
|
6895
|
+
"route_functional",
|
|
6896
|
+
reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6897
|
+
]);
|
|
5993
6898
|
}
|
|
5994
|
-
function
|
|
6899
|
+
function healthRun(run) {
|
|
5995
6900
|
return {
|
|
5996
6901
|
...run,
|
|
5997
|
-
|
|
5998
|
-
|
|
5999
|
-
|
|
6902
|
+
error: redactedEvidence(run.error),
|
|
6903
|
+
stdout: redactedEvidence(run.stdout),
|
|
6904
|
+
stderr: redactedEvidence(run.stderr)
|
|
6000
6905
|
};
|
|
6001
6906
|
}
|
|
6002
|
-
function
|
|
6907
|
+
function compactText(value, limit = 500) {
|
|
6908
|
+
const text = redact(bounded(value, limit));
|
|
6909
|
+
return text?.replace(/\s+/g, " ").trim();
|
|
6910
|
+
}
|
|
6911
|
+
function publicDoctorCheck(check) {
|
|
6003
6912
|
return {
|
|
6004
|
-
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6913
|
+
id: check.id,
|
|
6914
|
+
status: check.status,
|
|
6915
|
+
message: redact(bounded(check.message, 500)) ?? "",
|
|
6916
|
+
detail: redact(bounded(check.detail, 800))
|
|
6008
6917
|
};
|
|
6009
6918
|
}
|
|
6010
|
-
function
|
|
6919
|
+
function publicDoctorReport(report) {
|
|
6011
6920
|
return {
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
...step,
|
|
6015
|
-
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
6016
|
-
}))
|
|
6921
|
+
ok: report.ok,
|
|
6922
|
+
checks: report.checks.map(publicDoctorCheck)
|
|
6017
6923
|
};
|
|
6018
6924
|
}
|
|
6019
|
-
function
|
|
6020
|
-
|
|
6925
|
+
function includedStatusSet(statuses) {
|
|
6926
|
+
const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
|
|
6927
|
+
return new Set(values);
|
|
6021
6928
|
}
|
|
6022
|
-
function
|
|
6023
|
-
return
|
|
6929
|
+
function statusCounts(loops) {
|
|
6930
|
+
return {
|
|
6931
|
+
loops: loops.length,
|
|
6932
|
+
active: loops.filter((loop) => loop.status === "active").length,
|
|
6933
|
+
paused: loops.filter((loop) => loop.status === "paused").length,
|
|
6934
|
+
stopped: loops.filter((loop) => loop.status === "stopped").length,
|
|
6935
|
+
expired: loops.filter((loop) => loop.status === "expired").length
|
|
6936
|
+
};
|
|
6024
6937
|
}
|
|
6025
|
-
function
|
|
6026
|
-
|
|
6938
|
+
function compareLoopsForScan(left, right) {
|
|
6939
|
+
const statusOrder = left.status.localeCompare(right.status);
|
|
6940
|
+
if (statusOrder !== 0)
|
|
6941
|
+
return statusOrder;
|
|
6942
|
+
return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
|
|
6027
6943
|
}
|
|
6028
|
-
function
|
|
6944
|
+
function scanLoops(store, statuses, opts) {
|
|
6945
|
+
const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
|
|
6946
|
+
return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
|
|
6947
|
+
}
|
|
6948
|
+
function healthReportForLoops(store, loops, generatedAt) {
|
|
6949
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
6950
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
6951
|
+
for (const expectation of expectations) {
|
|
6952
|
+
if (expectation.failure)
|
|
6953
|
+
classifications[expectation.failure.classification] += 1;
|
|
6954
|
+
}
|
|
6955
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
6956
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
6029
6957
|
return {
|
|
6030
|
-
|
|
6031
|
-
|
|
6032
|
-
|
|
6033
|
-
|
|
6958
|
+
ok: unhealthy === 0,
|
|
6959
|
+
generatedAt,
|
|
6960
|
+
summary: {
|
|
6961
|
+
loops: expectations.length,
|
|
6962
|
+
healthy: expectations.length - unhealthy,
|
|
6963
|
+
unhealthy,
|
|
6964
|
+
warnings
|
|
6965
|
+
},
|
|
6966
|
+
classifications,
|
|
6967
|
+
expectations
|
|
6034
6968
|
};
|
|
6035
6969
|
}
|
|
6036
|
-
function
|
|
6037
|
-
|
|
6970
|
+
function ageMs(run, now) {
|
|
6971
|
+
const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
|
|
6972
|
+
const time = Date.parse(stamp);
|
|
6973
|
+
return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
|
|
6038
6974
|
}
|
|
6039
|
-
function
|
|
6975
|
+
function shortLoop(loop) {
|
|
6040
6976
|
return {
|
|
6041
|
-
|
|
6042
|
-
|
|
6977
|
+
id: loop.id,
|
|
6978
|
+
name: loop.name,
|
|
6979
|
+
status: loop.status,
|
|
6980
|
+
nextRunAt: loop.nextRunAt,
|
|
6981
|
+
leaseMs: loop.leaseMs
|
|
6043
6982
|
};
|
|
6044
6983
|
}
|
|
6045
|
-
function
|
|
6984
|
+
function priorityForSeverity(severity) {
|
|
6985
|
+
if (severity === "critical")
|
|
6986
|
+
return "critical";
|
|
6987
|
+
if (severity === "high")
|
|
6988
|
+
return "high";
|
|
6989
|
+
if (severity === "low")
|
|
6990
|
+
return "low";
|
|
6991
|
+
return "medium";
|
|
6992
|
+
}
|
|
6993
|
+
function recommendedFindingTask(finding, route) {
|
|
6994
|
+
const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
|
|
6995
|
+
if (finding.classification)
|
|
6996
|
+
tags.push(finding.classification);
|
|
6997
|
+
const description = [
|
|
6998
|
+
`OpenLoops health scan found a ${finding.kind} issue.`,
|
|
6999
|
+
finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
|
|
7000
|
+
finding.run ? `Run: ${finding.run.id}` : undefined,
|
|
7001
|
+
finding.classification ? `Classification: ${finding.classification}` : undefined,
|
|
7002
|
+
finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
|
|
7003
|
+
finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
|
|
7004
|
+
`Fingerprint: ${finding.fingerprint}`,
|
|
7005
|
+
`Severity: ${finding.severity}`,
|
|
7006
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
7007
|
+
route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
7008
|
+
route?.provider ? `Provider: ${route.provider}` : undefined,
|
|
7009
|
+
"",
|
|
7010
|
+
finding.message,
|
|
7011
|
+
finding.run?.error ? `Error:
|
|
7012
|
+
${finding.run.error}` : undefined,
|
|
7013
|
+
finding.run?.stderr ? `Stderr:
|
|
7014
|
+
${finding.run.stderr}` : undefined
|
|
7015
|
+
].filter(Boolean).join(`
|
|
7016
|
+
|
|
7017
|
+
`);
|
|
6046
7018
|
return {
|
|
6047
|
-
|
|
6048
|
-
|
|
6049
|
-
|
|
7019
|
+
title: finding.title,
|
|
7020
|
+
description,
|
|
7021
|
+
priority: priorityForSeverity(finding.severity),
|
|
7022
|
+
tags,
|
|
7023
|
+
dedupeKey: finding.fingerprint,
|
|
7024
|
+
search: { query: finding.fingerprint },
|
|
7025
|
+
compatibilityFallback: {
|
|
7026
|
+
search: ["todos", "search", finding.fingerprint, "--json"],
|
|
7027
|
+
add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
|
|
7028
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
7029
|
+
},
|
|
7030
|
+
futureNativeUpsert: {
|
|
7031
|
+
command: "todos task upsert",
|
|
7032
|
+
fields: {
|
|
7033
|
+
title: finding.title,
|
|
7034
|
+
description,
|
|
7035
|
+
priority: priorityForSeverity(finding.severity),
|
|
7036
|
+
tags,
|
|
7037
|
+
dedupeKey: finding.fingerprint,
|
|
7038
|
+
routeSource: route?.source ?? "openloops",
|
|
7039
|
+
routeKind: route?.kind ?? "health_scan",
|
|
7040
|
+
routeLoopId: finding.loop?.id ?? "",
|
|
7041
|
+
routeLoopName: finding.loop?.name ?? ""
|
|
7042
|
+
}
|
|
7043
|
+
}
|
|
6050
7044
|
};
|
|
6051
7045
|
}
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
import { createHash as createHash2 } from "crypto";
|
|
6055
|
-
import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
|
|
6056
|
-
var EVIDENCE_CHARS = 2000;
|
|
6057
|
-
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6058
|
-
var CLASSIFICATIONS = [
|
|
6059
|
-
"rate_limit",
|
|
6060
|
-
"auth",
|
|
6061
|
-
"model_not_found",
|
|
6062
|
-
"context_length",
|
|
6063
|
-
"schema_response_format",
|
|
6064
|
-
"node_init",
|
|
6065
|
-
"preflight",
|
|
6066
|
-
"route_functional",
|
|
6067
|
-
"timeout",
|
|
6068
|
-
"sigsegv",
|
|
6069
|
-
"skipped_previous_active",
|
|
6070
|
-
"circuit_breaker",
|
|
6071
|
-
"unknown"
|
|
6072
|
-
];
|
|
6073
|
-
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
6074
|
-
if (!value)
|
|
7046
|
+
function daemonFinding(daemon) {
|
|
7047
|
+
if (daemon.running && !daemon.stale)
|
|
6075
7048
|
return;
|
|
6076
|
-
|
|
6077
|
-
|
|
6078
|
-
|
|
6079
|
-
|
|
7049
|
+
const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
|
|
7050
|
+
const severity = "critical";
|
|
7051
|
+
const finding = {
|
|
7052
|
+
kind: "daemon",
|
|
7053
|
+
severity,
|
|
7054
|
+
fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
|
|
7055
|
+
title: "OpenLoops daemon health issue",
|
|
7056
|
+
message: reason
|
|
7057
|
+
};
|
|
7058
|
+
return {
|
|
7059
|
+
...finding,
|
|
7060
|
+
recommendedTask: recommendedFindingTask(finding, undefined)
|
|
7061
|
+
};
|
|
6080
7062
|
}
|
|
6081
|
-
function
|
|
6082
|
-
|
|
7063
|
+
function doctorSeverity(check) {
|
|
7064
|
+
if (check.status === "fail" && check.id === "data-dir")
|
|
7065
|
+
return "critical";
|
|
7066
|
+
if (check.status === "fail")
|
|
7067
|
+
return "high";
|
|
7068
|
+
return "medium";
|
|
6083
7069
|
}
|
|
6084
|
-
function
|
|
6085
|
-
|
|
6086
|
-
|
|
7070
|
+
function doctorFinding(check, loop, route) {
|
|
7071
|
+
if (check.status === "ok" || check.id === "loop-runs")
|
|
7072
|
+
return;
|
|
7073
|
+
const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
|
|
7074
|
+
const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
|
|
7075
|
+
const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
|
|
7076
|
+
const finding = {
|
|
7077
|
+
kind,
|
|
7078
|
+
severity,
|
|
7079
|
+
fingerprint,
|
|
7080
|
+
title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
|
|
7081
|
+
message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
|
|
7082
|
+
loop: loop ? shortLoop(loop) : undefined,
|
|
7083
|
+
route,
|
|
7084
|
+
doctorCheck: publicDoctorCheck(check)
|
|
7085
|
+
};
|
|
7086
|
+
return {
|
|
7087
|
+
...finding,
|
|
7088
|
+
recommendedTask: recommendedFindingTask(finding, route)
|
|
7089
|
+
};
|
|
6087
7090
|
}
|
|
6088
|
-
function
|
|
6089
|
-
|
|
6090
|
-
}
|
|
6091
|
-
function stringValue(value) {
|
|
6092
|
-
return typeof value === "string" && value.trim() ? value : undefined;
|
|
6093
|
-
}
|
|
6094
|
-
function objectField(value, key) {
|
|
6095
|
-
if (!value)
|
|
7091
|
+
function latestRunFinding(expectation) {
|
|
7092
|
+
if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
|
|
6096
7093
|
return;
|
|
6097
|
-
const
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
|
|
6105
|
-
|
|
7094
|
+
const failure = expectation.failure;
|
|
7095
|
+
if (!failure)
|
|
7096
|
+
return;
|
|
7097
|
+
const severity = expectation.loop.status === "active" ? "high" : "medium";
|
|
7098
|
+
return {
|
|
7099
|
+
kind: "latest-run",
|
|
7100
|
+
severity,
|
|
7101
|
+
fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
|
|
7102
|
+
title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
|
|
7103
|
+
message: expectation.check.message,
|
|
7104
|
+
loop: expectation.loop,
|
|
7105
|
+
run: expectation.latestRun,
|
|
7106
|
+
route: expectation.route,
|
|
7107
|
+
classification: failure.classification,
|
|
7108
|
+
doctorCheck: undefined,
|
|
7109
|
+
recommendedTask: expectation.recommendedTask
|
|
7110
|
+
};
|
|
6106
7111
|
}
|
|
6107
|
-
function
|
|
6108
|
-
|
|
6109
|
-
|
|
7112
|
+
function staleRunningFinding(loop, expectation, now, staleRunningMs) {
|
|
7113
|
+
const run = expectation.latestRun;
|
|
7114
|
+
if (loop.status !== "active" || run?.status !== "running")
|
|
7115
|
+
return;
|
|
7116
|
+
const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
|
|
7117
|
+
const age = ageMs(run, now);
|
|
7118
|
+
if (age <= threshold)
|
|
7119
|
+
return;
|
|
7120
|
+
const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
|
|
7121
|
+
const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
|
|
7122
|
+
const finding = {
|
|
7123
|
+
kind: "stale-running",
|
|
7124
|
+
severity: "critical",
|
|
7125
|
+
fingerprint,
|
|
7126
|
+
title: `OpenLoops stale running run - ${loop.name}`,
|
|
7127
|
+
message,
|
|
7128
|
+
loop: shortLoop(loop),
|
|
7129
|
+
run,
|
|
7130
|
+
route: expectation.route,
|
|
7131
|
+
ageMs: age,
|
|
7132
|
+
staleThresholdMs: threshold
|
|
7133
|
+
};
|
|
7134
|
+
return {
|
|
7135
|
+
...finding,
|
|
7136
|
+
recommendedTask: recommendedFindingTask(finding, expectation.route)
|
|
7137
|
+
};
|
|
6110
7138
|
}
|
|
6111
|
-
function
|
|
6112
|
-
|
|
6113
|
-
|
|
6114
|
-
|
|
6115
|
-
|
|
6116
|
-
|
|
6117
|
-
(run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6118
|
-
]);
|
|
7139
|
+
function scanStatus(findings) {
|
|
7140
|
+
if (findings.some((finding) => finding.severity === "critical"))
|
|
7141
|
+
return "critical";
|
|
7142
|
+
if (findings.length > 0)
|
|
7143
|
+
return "degraded";
|
|
7144
|
+
return "ok";
|
|
6119
7145
|
}
|
|
6120
|
-
function
|
|
6121
|
-
|
|
6122
|
-
|
|
6123
|
-
"route_functional",
|
|
6124
|
-
reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6125
|
-
]);
|
|
7146
|
+
function timestampDir(root, generatedAt) {
|
|
7147
|
+
const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
|
|
7148
|
+
return join7(root, stamp);
|
|
6126
7149
|
}
|
|
6127
|
-
function
|
|
6128
|
-
return
|
|
6129
|
-
|
|
6130
|
-
|
|
6131
|
-
|
|
6132
|
-
|
|
6133
|
-
|
|
7150
|
+
function healthScanMarkdown(scan) {
|
|
7151
|
+
return [
|
|
7152
|
+
"# OpenLoops Health Scan",
|
|
7153
|
+
"",
|
|
7154
|
+
`- status: ${scan.status}`,
|
|
7155
|
+
`- generated_at: ${scan.generatedAt}`,
|
|
7156
|
+
`- included_statuses: ${scan.includedStatuses.join(",")}`,
|
|
7157
|
+
`- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
|
|
7158
|
+
`- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
|
|
7159
|
+
scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
|
|
7160
|
+
scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
|
|
7161
|
+
scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
|
|
7162
|
+
"",
|
|
7163
|
+
"## Findings",
|
|
7164
|
+
scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
|
|
7165
|
+
`) : "None."
|
|
7166
|
+
].join(`
|
|
7167
|
+
`);
|
|
6134
7168
|
}
|
|
6135
7169
|
function classifyRunFailure(run) {
|
|
6136
7170
|
if (run.status === "succeeded" || run.status === "running")
|
|
6137
7171
|
return;
|
|
6138
|
-
const
|
|
7172
|
+
const rawText = searchableText(run);
|
|
7173
|
+
const text = rawText.toLowerCase();
|
|
6139
7174
|
let classification = "unknown";
|
|
7175
|
+
let summary;
|
|
6140
7176
|
if (run.status === "timed_out")
|
|
6141
7177
|
classification = "timeout";
|
|
7178
|
+
else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
|
|
7179
|
+
classification = "restart_interrupted";
|
|
6142
7180
|
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
6143
7181
|
classification = "circuit_breaker";
|
|
6144
7182
|
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
@@ -6147,8 +7185,10 @@ function classifyRunFailure(run) {
|
|
|
6147
7185
|
classification = "preflight";
|
|
6148
7186
|
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
6149
7187
|
classification = "rate_limit";
|
|
6150
|
-
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
7188
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
|
|
6151
7189
|
classification = "auth";
|
|
7190
|
+
else if (summary = providerUnavailableSummary(rawText))
|
|
7191
|
+
classification = "provider_unavailable";
|
|
6152
7192
|
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
6153
7193
|
classification = "model_not_found";
|
|
6154
7194
|
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
@@ -6161,8 +7201,9 @@ function classifyRunFailure(run) {
|
|
|
6161
7201
|
classification = "sigsegv";
|
|
6162
7202
|
return {
|
|
6163
7203
|
classification,
|
|
6164
|
-
fingerprint: stableFailureFingerprint(run, classification),
|
|
7204
|
+
fingerprint: stableFailureFingerprint(run, classification, summary),
|
|
6165
7205
|
evidence: {
|
|
7206
|
+
summary,
|
|
6166
7207
|
error: redactedEvidence(run.error),
|
|
6167
7208
|
stdout: redactedEvidence(run.stdout),
|
|
6168
7209
|
stderr: redactedEvidence(run.stderr),
|
|
@@ -6207,7 +7248,7 @@ function routeEvidenceReport(run) {
|
|
|
6207
7248
|
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
6208
7249
|
if (evidencePath && existsSync4(evidencePath)) {
|
|
6209
7250
|
try {
|
|
6210
|
-
return parseJsonObject(
|
|
7251
|
+
return parseJsonObject(readFileSync5(evidencePath, "utf8")) ?? stdoutReport;
|
|
6211
7252
|
} catch {
|
|
6212
7253
|
return stdoutReport;
|
|
6213
7254
|
}
|
|
@@ -6339,6 +7380,12 @@ function targetRoute(loop) {
|
|
|
6339
7380
|
loopName: loop.name
|
|
6340
7381
|
};
|
|
6341
7382
|
}
|
|
7383
|
+
function expectationLoop(loop) {
|
|
7384
|
+
return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
|
|
7385
|
+
}
|
|
7386
|
+
function hasPendingRetry(loop, run) {
|
|
7387
|
+
return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
|
|
7388
|
+
}
|
|
6342
7389
|
function recommendedTask(loop, run, failure, route) {
|
|
6343
7390
|
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
6344
7391
|
const description = [
|
|
@@ -6350,6 +7397,7 @@ function recommendedTask(loop, run, failure, route) {
|
|
|
6350
7397
|
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6351
7398
|
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6352
7399
|
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
7400
|
+
failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
|
|
6353
7401
|
failure.evidence.error ? `Error:
|
|
6354
7402
|
${failure.evidence.error}` : undefined,
|
|
6355
7403
|
failure.evidence.stderr ? `Stderr:
|
|
@@ -6359,7 +7407,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6359
7407
|
`);
|
|
6360
7408
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6361
7409
|
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6362
|
-
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
7410
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
|
|
6363
7411
|
return {
|
|
6364
7412
|
title,
|
|
6365
7413
|
description,
|
|
@@ -6393,7 +7441,7 @@ function expectationForLoop(store, loop) {
|
|
|
6393
7441
|
const route = targetRoute(loop);
|
|
6394
7442
|
if (!latestRun) {
|
|
6395
7443
|
return {
|
|
6396
|
-
loop:
|
|
7444
|
+
loop: expectationLoop(loop),
|
|
6397
7445
|
ok: true,
|
|
6398
7446
|
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
6399
7447
|
route
|
|
@@ -6402,7 +7450,7 @@ function expectationForLoop(store, loop) {
|
|
|
6402
7450
|
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
6403
7451
|
if (routeFailure) {
|
|
6404
7452
|
return {
|
|
6405
|
-
loop:
|
|
7453
|
+
loop: expectationLoop(loop),
|
|
6406
7454
|
ok: false,
|
|
6407
7455
|
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
6408
7456
|
latestRun: healthRun(latestRun),
|
|
@@ -6413,7 +7461,7 @@ function expectationForLoop(store, loop) {
|
|
|
6413
7461
|
}
|
|
6414
7462
|
if (latestRun.status === "succeeded") {
|
|
6415
7463
|
return {
|
|
6416
|
-
loop:
|
|
7464
|
+
loop: expectationLoop(loop),
|
|
6417
7465
|
ok: true,
|
|
6418
7466
|
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
6419
7467
|
latestRun: healthRun(latestRun),
|
|
@@ -6424,7 +7472,7 @@ function expectationForLoop(store, loop) {
|
|
|
6424
7472
|
if (failure?.classification === "circuit_breaker") {
|
|
6425
7473
|
if (loop.status !== "paused") {
|
|
6426
7474
|
return {
|
|
6427
|
-
loop:
|
|
7475
|
+
loop: expectationLoop(loop),
|
|
6428
7476
|
ok: true,
|
|
6429
7477
|
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
6430
7478
|
latestRun: healthRun(latestRun),
|
|
@@ -6432,7 +7480,7 @@ function expectationForLoop(store, loop) {
|
|
|
6432
7480
|
};
|
|
6433
7481
|
}
|
|
6434
7482
|
return {
|
|
6435
|
-
loop:
|
|
7483
|
+
loop: expectationLoop(loop),
|
|
6436
7484
|
ok: false,
|
|
6437
7485
|
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
6438
7486
|
latestRun: healthRun(latestRun),
|
|
@@ -6441,8 +7489,33 @@ function expectationForLoop(store, loop) {
|
|
|
6441
7489
|
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
6442
7490
|
};
|
|
6443
7491
|
}
|
|
7492
|
+
if (failure?.classification === "restart_interrupted") {
|
|
7493
|
+
return {
|
|
7494
|
+
loop: expectationLoop(loop),
|
|
7495
|
+
ok: true,
|
|
7496
|
+
check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
|
|
7497
|
+
latestRun: healthRun(latestRun),
|
|
7498
|
+
failure,
|
|
7499
|
+
route
|
|
7500
|
+
};
|
|
7501
|
+
}
|
|
7502
|
+
if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
|
|
7503
|
+
const message = [
|
|
7504
|
+
"provider unavailable/network failure; retry is scheduled",
|
|
7505
|
+
loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
|
|
7506
|
+
failure.evidence.summary
|
|
7507
|
+
].filter(Boolean).join("; ");
|
|
7508
|
+
return {
|
|
7509
|
+
loop: expectationLoop(loop),
|
|
7510
|
+
ok: true,
|
|
7511
|
+
check: { id: "latest-run-succeeded", status: "warn", message },
|
|
7512
|
+
latestRun: healthRun(latestRun),
|
|
7513
|
+
failure,
|
|
7514
|
+
route
|
|
7515
|
+
};
|
|
7516
|
+
}
|
|
6444
7517
|
return {
|
|
6445
|
-
loop:
|
|
7518
|
+
loop: expectationLoop(loop),
|
|
6446
7519
|
ok: false,
|
|
6447
7520
|
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
6448
7521
|
latestRun: healthRun(latestRun),
|
|
@@ -6474,6 +7547,206 @@ function buildHealthReport(store, opts = {}) {
|
|
|
6474
7547
|
expectations
|
|
6475
7548
|
};
|
|
6476
7549
|
}
|
|
7550
|
+
function buildHealthScan(store, opts = {}) {
|
|
7551
|
+
const generatedAt = (opts.now ?? new Date).toISOString();
|
|
7552
|
+
const now = opts.now ?? new Date(generatedAt);
|
|
7553
|
+
const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
|
|
7554
|
+
const loops = scanLoops(store, includeStatuses, opts);
|
|
7555
|
+
const health = healthReportForLoops(store, loops, generatedAt);
|
|
7556
|
+
const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
|
|
7557
|
+
const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
|
|
7558
|
+
const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
|
|
7559
|
+
const allFindings = [];
|
|
7560
|
+
const pushFinding = (finding) => {
|
|
7561
|
+
if (!finding)
|
|
7562
|
+
return;
|
|
7563
|
+
allFindings.push(finding);
|
|
7564
|
+
};
|
|
7565
|
+
if (opts.daemon)
|
|
7566
|
+
pushFinding(daemonFinding(opts.daemon));
|
|
7567
|
+
if (opts.latestRun !== false) {
|
|
7568
|
+
for (const loop of loops) {
|
|
7569
|
+
const expectation = expectationsByLoopId.get(loop.id);
|
|
7570
|
+
if (!expectation)
|
|
7571
|
+
continue;
|
|
7572
|
+
pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
|
|
7573
|
+
pushFinding(latestRunFinding(expectation));
|
|
7574
|
+
}
|
|
7575
|
+
}
|
|
7576
|
+
const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
|
|
7577
|
+
if (doctor) {
|
|
7578
|
+
for (const check of doctor.checks) {
|
|
7579
|
+
const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
|
|
7580
|
+
const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
|
|
7581
|
+
const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
|
|
7582
|
+
pushFinding(doctorFinding(check, loop, route));
|
|
7583
|
+
}
|
|
7584
|
+
}
|
|
7585
|
+
const findings = allFindings.slice(0, maxFindings);
|
|
7586
|
+
const status = scanStatus(allFindings);
|
|
7587
|
+
const baseCounts = statusCounts(loops);
|
|
7588
|
+
return {
|
|
7589
|
+
ok: status === "ok",
|
|
7590
|
+
status,
|
|
7591
|
+
generatedAt,
|
|
7592
|
+
includedStatuses: includeStatuses,
|
|
7593
|
+
counts: {
|
|
7594
|
+
...baseCounts,
|
|
7595
|
+
latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
|
|
7596
|
+
staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
|
|
7597
|
+
daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
|
|
7598
|
+
doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
|
|
7599
|
+
preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
|
|
7600
|
+
findings: allFindings.length,
|
|
7601
|
+
reportedFindings: findings.length,
|
|
7602
|
+
truncatedFindings: Math.max(0, allFindings.length - findings.length)
|
|
7603
|
+
},
|
|
7604
|
+
daemon: opts.daemon ? {
|
|
7605
|
+
running: opts.daemon.running,
|
|
7606
|
+
stale: opts.daemon.stale,
|
|
7607
|
+
pid: opts.daemon.pid,
|
|
7608
|
+
host: opts.daemon.host,
|
|
7609
|
+
loops: opts.daemon.loops,
|
|
7610
|
+
runs: opts.daemon.runs,
|
|
7611
|
+
logPath: opts.daemon.logPath
|
|
7612
|
+
} : undefined,
|
|
7613
|
+
doctor,
|
|
7614
|
+
health,
|
|
7615
|
+
selfHeals: opts.selfHeals ?? [],
|
|
7616
|
+
findings
|
|
7617
|
+
};
|
|
7618
|
+
}
|
|
7619
|
+
function writeHealthScanReports(scan, opts = {}) {
|
|
7620
|
+
const root = opts.reportDir ?? join7(dataDir(), "reports", "health-scan");
|
|
7621
|
+
const dir = timestampDir(root, scan.generatedAt);
|
|
7622
|
+
mkdirSync6(dir, { recursive: true, mode: 448 });
|
|
7623
|
+
const json = join7(dir, "summary.json");
|
|
7624
|
+
const markdown = join7(dir, "report.md");
|
|
7625
|
+
const withReports = {
|
|
7626
|
+
...scan,
|
|
7627
|
+
reports: { dir, json, markdown }
|
|
7628
|
+
};
|
|
7629
|
+
writeFileSync3(json, JSON.stringify(withReports, null, 2), { mode: 384 });
|
|
7630
|
+
writeFileSync3(markdown, healthScanMarkdown(withReports), { mode: 384 });
|
|
7631
|
+
return withReports;
|
|
7632
|
+
}
|
|
7633
|
+
|
|
7634
|
+
// src/lib/doctor.ts
|
|
7635
|
+
var PROVIDER_COMMANDS = [
|
|
7636
|
+
"claude",
|
|
7637
|
+
"agent",
|
|
7638
|
+
"codewith",
|
|
7639
|
+
"aicopilot",
|
|
7640
|
+
"opencode",
|
|
7641
|
+
"codex"
|
|
7642
|
+
];
|
|
7643
|
+
function hasCommand(command) {
|
|
7644
|
+
const result = spawnSync6("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
7645
|
+
return (result.status ?? 1) === 0;
|
|
7646
|
+
}
|
|
7647
|
+
function commandVersion(command) {
|
|
7648
|
+
const result = spawnSync6(command, ["--version"], {
|
|
7649
|
+
encoding: "utf8",
|
|
7650
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
7651
|
+
});
|
|
7652
|
+
if ((result.status ?? 1) !== 0)
|
|
7653
|
+
return;
|
|
7654
|
+
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
7655
|
+
}
|
|
7656
|
+
function runDoctor(store) {
|
|
7657
|
+
const checks = [];
|
|
7658
|
+
try {
|
|
7659
|
+
const dir = ensureDataDir();
|
|
7660
|
+
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
7661
|
+
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
7662
|
+
} catch (error) {
|
|
7663
|
+
checks.push({
|
|
7664
|
+
id: "data-dir",
|
|
7665
|
+
status: "fail",
|
|
7666
|
+
message: "data directory is not writable",
|
|
7667
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7668
|
+
});
|
|
7669
|
+
}
|
|
7670
|
+
const bunVersion = commandVersion("bun");
|
|
7671
|
+
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
7672
|
+
const accountsVersion = commandVersion("accounts");
|
|
7673
|
+
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
7674
|
+
try {
|
|
7675
|
+
const machines = listOpenMachines();
|
|
7676
|
+
const local = machines.find((machine) => machine.local);
|
|
7677
|
+
checks.push({
|
|
7678
|
+
id: "machines",
|
|
7679
|
+
status: "ok",
|
|
7680
|
+
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
7681
|
+
detail: local ? `local=${local.id}` : undefined
|
|
7682
|
+
});
|
|
7683
|
+
} catch (error) {
|
|
7684
|
+
checks.push({
|
|
7685
|
+
id: "machines",
|
|
7686
|
+
status: "warn",
|
|
7687
|
+
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
7688
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7689
|
+
});
|
|
7690
|
+
}
|
|
7691
|
+
for (const command of PROVIDER_COMMANDS) {
|
|
7692
|
+
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
7693
|
+
}
|
|
7694
|
+
const status = daemonStatus(store);
|
|
7695
|
+
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
7696
|
+
const failedRuns = store.countRuns("failed");
|
|
7697
|
+
const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
|
|
7698
|
+
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
7699
|
+
if (restartInterruptedRuns > 0) {
|
|
7700
|
+
checks.push({
|
|
7701
|
+
id: "loop-runs:restart-interrupted",
|
|
7702
|
+
status: "warn",
|
|
7703
|
+
message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
|
|
7704
|
+
});
|
|
7705
|
+
}
|
|
7706
|
+
const deployment = buildDeploymentStatus();
|
|
7707
|
+
const schedulerState = deployment.schedulerState;
|
|
7708
|
+
checks.push({
|
|
7709
|
+
id: "scheduler-state",
|
|
7710
|
+
status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
|
|
7711
|
+
message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
|
|
7712
|
+
detail: [
|
|
7713
|
+
`route_state=${schedulerState.routeAdmission.stateStore}`,
|
|
7714
|
+
`active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
|
|
7715
|
+
`gates=${schedulerState.routeAdmission.gates.join(",")}`,
|
|
7716
|
+
`artifacts=${schedulerState.localStore.runArtifacts}`,
|
|
7717
|
+
`remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
|
|
7718
|
+
`remote_apply=${String(schedulerState.remoteStore.applySupported)}`
|
|
7719
|
+
].join(" ")
|
|
7720
|
+
});
|
|
7721
|
+
for (const loop of store.listLoops({ status: "active" })) {
|
|
7722
|
+
try {
|
|
7723
|
+
if (loop.target.type === "workflow") {
|
|
7724
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
7725
|
+
for (const step of workflowExecutionOrder(workflow)) {
|
|
7726
|
+
preflightTarget({
|
|
7727
|
+
...step.target,
|
|
7728
|
+
account: step.account ?? step.target.account,
|
|
7729
|
+
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
7730
|
+
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
7731
|
+
}
|
|
7732
|
+
} else {
|
|
7733
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
7734
|
+
}
|
|
7735
|
+
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
7736
|
+
} catch (error) {
|
|
7737
|
+
checks.push({
|
|
7738
|
+
id: `loop:${loop.id}:preflight`,
|
|
7739
|
+
status: "fail",
|
|
7740
|
+
message: `active loop target preflight failed: ${loop.name}`,
|
|
7741
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7742
|
+
});
|
|
7743
|
+
}
|
|
7744
|
+
}
|
|
7745
|
+
return {
|
|
7746
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
7747
|
+
checks
|
|
7748
|
+
};
|
|
7749
|
+
}
|
|
6477
7750
|
|
|
6478
7751
|
// src/lib/goal/metadata.ts
|
|
6479
7752
|
function goalExecutionContext(parts) {
|
|
@@ -6560,55 +7833,6 @@ ${evidence.join(`
|
|
|
6560
7833
|
import { generateObject } from "ai";
|
|
6561
7834
|
import { z } from "zod";
|
|
6562
7835
|
|
|
6563
|
-
// src/lib/run-envelope.ts
|
|
6564
|
-
var ENVELOPE_EXCERPT_CHARS = 2048;
|
|
6565
|
-
var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
|
|
6566
|
-
function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
|
|
6567
|
-
if (!text)
|
|
6568
|
-
return;
|
|
6569
|
-
const scrubbed = scrubSecrets(text);
|
|
6570
|
-
if (scrubbed.length <= limit * 2)
|
|
6571
|
-
return scrubbed;
|
|
6572
|
-
const omitted = scrubbed.length - limit * 2;
|
|
6573
|
-
return `${scrubbed.slice(0, limit)}
|
|
6574
|
-
[... ${omitted} chars omitted ...]
|
|
6575
|
-
${scrubbed.slice(-limit)}`;
|
|
6576
|
-
}
|
|
6577
|
-
function summarizeOutput(stdout, stderr) {
|
|
6578
|
-
return {
|
|
6579
|
-
stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
|
|
6580
|
-
stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
|
|
6581
|
-
stdoutExcerpt: boundedExcerpt(stdout),
|
|
6582
|
-
stderrExcerpt: boundedExcerpt(stderr)
|
|
6583
|
-
};
|
|
6584
|
-
}
|
|
6585
|
-
function summarizeExecutorResult(result) {
|
|
6586
|
-
return {
|
|
6587
|
-
...summarizeOutput(result.stdout, result.stderr),
|
|
6588
|
-
status: result.status,
|
|
6589
|
-
exitCode: result.exitCode,
|
|
6590
|
-
durationMs: result.durationMs,
|
|
6591
|
-
error: boundedExcerpt(result.error)
|
|
6592
|
-
};
|
|
6593
|
-
}
|
|
6594
|
-
function isBlockedStepRun(step) {
|
|
6595
|
-
return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
|
|
6596
|
-
}
|
|
6597
|
-
function summarizeWorkflowStepRun(step) {
|
|
6598
|
-
return {
|
|
6599
|
-
...summarizeOutput(step.stdout, step.stderr),
|
|
6600
|
-
stepId: step.stepId,
|
|
6601
|
-
status: step.status,
|
|
6602
|
-
exitCode: step.exitCode,
|
|
6603
|
-
durationMs: step.durationMs,
|
|
6604
|
-
error: boundedExcerpt(step.error),
|
|
6605
|
-
blocked: isBlockedStepRun(step)
|
|
6606
|
-
};
|
|
6607
|
-
}
|
|
6608
|
-
function workflowRunEnvelope(run, steps) {
|
|
6609
|
-
return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
|
|
6610
|
-
}
|
|
6611
|
-
|
|
6612
7836
|
// src/lib/goal/model-factory.ts
|
|
6613
7837
|
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
|
|
6614
7838
|
var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
|
|
@@ -7473,7 +8697,7 @@ var MAX_RETRY_EXPONENT = 20;
|
|
|
7473
8697
|
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
7474
8698
|
const attempt = Math.max(1, run.attempt);
|
|
7475
8699
|
const failure = classifyRunFailure(run);
|
|
7476
|
-
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
8700
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
|
|
7477
8701
|
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
7478
8702
|
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
7479
8703
|
const jitter = 0.5 + random();
|
|
@@ -7600,20 +8824,21 @@ async function executeClaimedRun(deps) {
|
|
|
7600
8824
|
daemonLeaseId: deps.daemonLeaseId,
|
|
7601
8825
|
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
7602
8826
|
})))(deps.loop, deps.run);
|
|
8827
|
+
const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
|
|
7603
8828
|
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
7604
8829
|
return deps.store.finalizeRun(deps.run.id, {
|
|
7605
|
-
status:
|
|
7606
|
-
finishedAt:
|
|
7607
|
-
durationMs:
|
|
7608
|
-
stdout:
|
|
7609
|
-
stderr:
|
|
7610
|
-
exitCode:
|
|
7611
|
-
error:
|
|
7612
|
-
pid:
|
|
8830
|
+
status: finalResult.status,
|
|
8831
|
+
finishedAt: finalResult.finishedAt,
|
|
8832
|
+
durationMs: finalResult.durationMs,
|
|
8833
|
+
stdout: finalResult.stdout,
|
|
8834
|
+
stderr: finalResult.stderr,
|
|
8835
|
+
exitCode: finalResult.exitCode,
|
|
8836
|
+
error: finalResult.error,
|
|
8837
|
+
pid: finalResult.pid
|
|
7613
8838
|
}, {
|
|
7614
8839
|
claimedBy: deps.runnerId,
|
|
7615
8840
|
daemonLeaseId: deps.daemonLeaseId,
|
|
7616
|
-
now: deps.now?.() ?? new Date(
|
|
8841
|
+
now: deps.now?.() ?? new Date(finalResult.finishedAt)
|
|
7617
8842
|
});
|
|
7618
8843
|
} catch (err) {
|
|
7619
8844
|
deps.onError?.(deps.loop, err);
|
|
@@ -7673,178 +8898,1062 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
7673
8898
|
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
7674
8899
|
let skipped;
|
|
7675
8900
|
try {
|
|
7676
|
-
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
7677
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
7678
|
-
});
|
|
7679
|
-
} catch (error) {
|
|
7680
|
-
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
7681
|
-
return;
|
|
7682
|
-
throw error;
|
|
8901
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
8902
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
8903
|
+
});
|
|
8904
|
+
} catch (error) {
|
|
8905
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
8906
|
+
return;
|
|
8907
|
+
throw error;
|
|
8908
|
+
}
|
|
8909
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
8910
|
+
deps.onRun?.(skipped);
|
|
8911
|
+
return skipped;
|
|
8912
|
+
}
|
|
8913
|
+
let claim;
|
|
8914
|
+
try {
|
|
8915
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
8916
|
+
} catch (error) {
|
|
8917
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
8918
|
+
return;
|
|
8919
|
+
throw error;
|
|
8920
|
+
}
|
|
8921
|
+
if (!claim) {
|
|
8922
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
8923
|
+
return;
|
|
8924
|
+
}
|
|
8925
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
8926
|
+
deps.onRun?.(claim.run);
|
|
8927
|
+
const finalRun = await executeClaimedRun({
|
|
8928
|
+
store: deps.store,
|
|
8929
|
+
runnerId: deps.runnerId,
|
|
8930
|
+
loop: claim.loop,
|
|
8931
|
+
run: claim.run,
|
|
8932
|
+
now: deps.now,
|
|
8933
|
+
execute: deps.execute,
|
|
8934
|
+
beforeFinalize: deps.beforeFinalize,
|
|
8935
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
8936
|
+
onError: deps.onError
|
|
8937
|
+
});
|
|
8938
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
|
|
8939
|
+
deps.onRun?.(finalRun);
|
|
8940
|
+
return finalRun;
|
|
8941
|
+
}
|
|
8942
|
+
function claimSlot(deps, loop, scheduledFor) {
|
|
8943
|
+
const now = deps.now?.() ?? new Date;
|
|
8944
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
8945
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
8946
|
+
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
8947
|
+
return;
|
|
8948
|
+
let skipped;
|
|
8949
|
+
try {
|
|
8950
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
8951
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
8952
|
+
});
|
|
8953
|
+
} catch (error) {
|
|
8954
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
8955
|
+
return;
|
|
8956
|
+
throw error;
|
|
8957
|
+
}
|
|
8958
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
8959
|
+
deps.onRun?.(skipped);
|
|
8960
|
+
return skipped;
|
|
8961
|
+
}
|
|
8962
|
+
let claim;
|
|
8963
|
+
try {
|
|
8964
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
8965
|
+
} catch (error) {
|
|
8966
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
8967
|
+
return;
|
|
8968
|
+
throw error;
|
|
8969
|
+
}
|
|
8970
|
+
if (!claim) {
|
|
8971
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
8972
|
+
return;
|
|
8973
|
+
}
|
|
8974
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
8975
|
+
deps.onRun?.(claim.run);
|
|
8976
|
+
return claim;
|
|
8977
|
+
}
|
|
8978
|
+
function recoverAndExpire(deps, now) {
|
|
8979
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
8980
|
+
const recoveredByLoop = new Map;
|
|
8981
|
+
for (const run of recovered) {
|
|
8982
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
8983
|
+
}
|
|
8984
|
+
for (const runs of recoveredByLoop.values()) {
|
|
8985
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
8986
|
+
if (!loop)
|
|
8987
|
+
continue;
|
|
8988
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
8989
|
+
if (retryable) {
|
|
8990
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
|
|
8991
|
+
continue;
|
|
8992
|
+
}
|
|
8993
|
+
for (const run of runs) {
|
|
8994
|
+
const current = deps.store.getLoop(run.loopId);
|
|
8995
|
+
if (current) {
|
|
8996
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
|
|
8997
|
+
}
|
|
8998
|
+
}
|
|
8999
|
+
}
|
|
9000
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
9001
|
+
return { recovered, expired };
|
|
9002
|
+
}
|
|
9003
|
+
function claimDueRuns(deps) {
|
|
9004
|
+
const now = deps.now?.() ?? new Date;
|
|
9005
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
9006
|
+
const claims = [];
|
|
9007
|
+
const claimed = [];
|
|
9008
|
+
const skipped = [];
|
|
9009
|
+
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
9010
|
+
if (maxClaims === 0)
|
|
9011
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
9012
|
+
const laneLimits = deps.laneLimits;
|
|
9013
|
+
const laneClaims = { command: 0, agent: 0 };
|
|
9014
|
+
const laneCap = (lane) => laneLimits === undefined ? Number.POSITIVE_INFINITY : Math.max(0, laneLimits[lane] ?? Number.POSITIVE_INFINITY);
|
|
9015
|
+
const laneFull = (lane) => laneClaims[lane] >= laneCap(lane);
|
|
9016
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
9017
|
+
if (claims.length >= maxClaims)
|
|
9018
|
+
break;
|
|
9019
|
+
const lane = loopLane(loop);
|
|
9020
|
+
if (laneFull(lane))
|
|
9021
|
+
continue;
|
|
9022
|
+
const plan = dueSlots(loop, now);
|
|
9023
|
+
let loopSkips = 0;
|
|
9024
|
+
for (const slot of plan.slots) {
|
|
9025
|
+
if (claims.length >= maxClaims)
|
|
9026
|
+
break;
|
|
9027
|
+
if (laneFull(lane))
|
|
9028
|
+
break;
|
|
9029
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
9030
|
+
break;
|
|
9031
|
+
const run = claimSlot(deps, loop, slot);
|
|
9032
|
+
if (!run)
|
|
9033
|
+
continue;
|
|
9034
|
+
if ("loop" in run) {
|
|
9035
|
+
claims.push(run);
|
|
9036
|
+
claimed.push(run.run);
|
|
9037
|
+
laneClaims[lane] += 1;
|
|
9038
|
+
} else if (run.status === "skipped") {
|
|
9039
|
+
skipped.push(run);
|
|
9040
|
+
loopSkips += 1;
|
|
9041
|
+
}
|
|
9042
|
+
}
|
|
9043
|
+
}
|
|
9044
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
9045
|
+
}
|
|
9046
|
+
async function tick(deps) {
|
|
9047
|
+
const now = deps.now?.() ?? new Date;
|
|
9048
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
9049
|
+
const claimed = [];
|
|
9050
|
+
const completed = [];
|
|
9051
|
+
const skipped = [];
|
|
9052
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
9053
|
+
const plan = dueSlots(loop, now);
|
|
9054
|
+
let loopSkips = 0;
|
|
9055
|
+
for (const slot of plan.slots) {
|
|
9056
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
9057
|
+
break;
|
|
9058
|
+
const run = await runSlot(deps, loop, slot);
|
|
9059
|
+
if (!run)
|
|
9060
|
+
continue;
|
|
9061
|
+
if (run.status === "running")
|
|
9062
|
+
claimed.push(run);
|
|
9063
|
+
else if (run.status === "skipped") {
|
|
9064
|
+
skipped.push(run);
|
|
9065
|
+
loopSkips += 1;
|
|
9066
|
+
} else
|
|
9067
|
+
completed.push(run);
|
|
9068
|
+
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
9069
|
+
break;
|
|
9070
|
+
}
|
|
9071
|
+
}
|
|
9072
|
+
return { claimed, completed, skipped, recovered, expired };
|
|
9073
|
+
}
|
|
9074
|
+
|
|
9075
|
+
// src/lib/cloud/mode.ts
|
|
9076
|
+
var DEPRECATED_STORAGE_MODE_ALIASES = ["remote", "hybrid", "self_hosted"];
|
|
9077
|
+
function normalizeStorageMode(value) {
|
|
9078
|
+
const normalized = value.trim().toLowerCase().replace(/-/g, "_");
|
|
9079
|
+
if (normalized === "local")
|
|
9080
|
+
return { mode: "local", deprecatedAlias: null };
|
|
9081
|
+
if (normalized === "cloud")
|
|
9082
|
+
return { mode: "cloud", deprecatedAlias: null };
|
|
9083
|
+
if (DEPRECATED_STORAGE_MODE_ALIASES.includes(normalized)) {
|
|
9084
|
+
return { mode: "cloud", deprecatedAlias: normalized };
|
|
9085
|
+
}
|
|
9086
|
+
throw new Error(`Unknown storage mode: ${value}. Use local or cloud.`);
|
|
9087
|
+
}
|
|
9088
|
+
function envToken(name) {
|
|
9089
|
+
return name.toUpperCase().replace(/-/g, "_");
|
|
9090
|
+
}
|
|
9091
|
+
|
|
9092
|
+
// src/lib/cloud/transport.ts
|
|
9093
|
+
function defaultCloudBaseUrl(name) {
|
|
9094
|
+
return `https://${name}.hasna.xyz`;
|
|
9095
|
+
}
|
|
9096
|
+
function clientTransportEnvKeys(name) {
|
|
9097
|
+
const token = envToken(name);
|
|
9098
|
+
return {
|
|
9099
|
+
modeKeys: [
|
|
9100
|
+
`HASNA_${token}_STORAGE_MODE`,
|
|
9101
|
+
`HASNA_${token}_MODE`,
|
|
9102
|
+
`${token}_STORAGE_MODE`,
|
|
9103
|
+
`${token}_MODE`
|
|
9104
|
+
],
|
|
9105
|
+
apiUrlKeys: [`HASNA_${token}_API_URL`, `${token}_API_URL`],
|
|
9106
|
+
apiKeyKeys: [
|
|
9107
|
+
`HASNA_${token}_API_KEY`,
|
|
9108
|
+
`${token}_API_KEY`,
|
|
9109
|
+
`HASNA_${token}_API_TOKEN`,
|
|
9110
|
+
`${token}_API_TOKEN`
|
|
9111
|
+
]
|
|
9112
|
+
};
|
|
9113
|
+
}
|
|
9114
|
+
function firstEnv(env, keys) {
|
|
9115
|
+
for (const key of keys) {
|
|
9116
|
+
const value = env[key]?.trim();
|
|
9117
|
+
if (value)
|
|
9118
|
+
return { key, value };
|
|
9119
|
+
}
|
|
9120
|
+
return null;
|
|
9121
|
+
}
|
|
9122
|
+
function toV1BaseUrl(apiUrl) {
|
|
9123
|
+
const url = new URL(apiUrl);
|
|
9124
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
9125
|
+
throw new Error("API URL must use http or https.");
|
|
9126
|
+
}
|
|
9127
|
+
let path = url.pathname.replace(/\/+$/, "");
|
|
9128
|
+
if (path.endsWith("/v1"))
|
|
9129
|
+
path = path.slice(0, -"/v1".length);
|
|
9130
|
+
url.pathname = `${path}/v1`;
|
|
9131
|
+
url.search = "";
|
|
9132
|
+
url.hash = "";
|
|
9133
|
+
return url.toString().replace(/\/+$/, "");
|
|
9134
|
+
}
|
|
9135
|
+
function resolveClientTransport(name, env = process.env) {
|
|
9136
|
+
const keys = clientTransportEnvKeys(name);
|
|
9137
|
+
const modeHit = firstEnv(env, keys.modeKeys);
|
|
9138
|
+
const urlHit = firstEnv(env, keys.apiUrlKeys);
|
|
9139
|
+
const keyHit = firstEnv(env, keys.apiKeyKeys);
|
|
9140
|
+
let mode = "local";
|
|
9141
|
+
let deprecatedAlias = null;
|
|
9142
|
+
let modeSource = "default";
|
|
9143
|
+
const warnings = [];
|
|
9144
|
+
if (modeHit) {
|
|
9145
|
+
const normalized = normalizeStorageMode(modeHit.value);
|
|
9146
|
+
mode = normalized.mode;
|
|
9147
|
+
deprecatedAlias = normalized.deprecatedAlias;
|
|
9148
|
+
modeSource = modeHit.key;
|
|
9149
|
+
if (deprecatedAlias) {
|
|
9150
|
+
warnings.push(`Deprecated mode '${deprecatedAlias}' from ${modeHit.key} is treated as 'cloud'. Prefer ${keys.modeKeys[0]}=cloud.`);
|
|
9151
|
+
}
|
|
9152
|
+
}
|
|
9153
|
+
if (mode === "local") {
|
|
9154
|
+
return {
|
|
9155
|
+
transport: "local",
|
|
9156
|
+
mode,
|
|
9157
|
+
deprecatedAlias,
|
|
9158
|
+
modeSource,
|
|
9159
|
+
baseUrl: null,
|
|
9160
|
+
apiUrlSource: null,
|
|
9161
|
+
apiKeyPresent: Boolean(keyHit),
|
|
9162
|
+
apiKeySource: keyHit ? keyHit.key : null,
|
|
9163
|
+
misconfigured: false,
|
|
9164
|
+
warning: warnings.length > 0 ? warnings.join(" ") : null
|
|
9165
|
+
};
|
|
9166
|
+
}
|
|
9167
|
+
if (!keyHit) {
|
|
9168
|
+
warnings.push(`${modeSource}=cloud but no API key is set (${keys.apiKeyKeys[0]}). Refusing to route to cloud; using local store. Set ${keys.apiKeyKeys[0]} to enable the cloud client.`);
|
|
9169
|
+
return {
|
|
9170
|
+
transport: "local",
|
|
9171
|
+
mode,
|
|
9172
|
+
deprecatedAlias,
|
|
9173
|
+
modeSource,
|
|
9174
|
+
baseUrl: null,
|
|
9175
|
+
apiUrlSource: null,
|
|
9176
|
+
apiKeyPresent: false,
|
|
9177
|
+
apiKeySource: null,
|
|
9178
|
+
misconfigured: true,
|
|
9179
|
+
warning: warnings.join(" ")
|
|
9180
|
+
};
|
|
9181
|
+
}
|
|
9182
|
+
const rawUrl = urlHit?.value ?? defaultCloudBaseUrl(name);
|
|
9183
|
+
const apiUrlSource = urlHit ? urlHit.key : "default";
|
|
9184
|
+
let baseUrl;
|
|
9185
|
+
try {
|
|
9186
|
+
baseUrl = toV1BaseUrl(rawUrl);
|
|
9187
|
+
} catch (error) {
|
|
9188
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
9189
|
+
warnings.push(`Invalid API URL from ${apiUrlSource}: ${message}. Using local store.`);
|
|
9190
|
+
return {
|
|
9191
|
+
transport: "local",
|
|
9192
|
+
mode,
|
|
9193
|
+
deprecatedAlias,
|
|
9194
|
+
modeSource,
|
|
9195
|
+
baseUrl: null,
|
|
9196
|
+
apiUrlSource: null,
|
|
9197
|
+
apiKeyPresent: true,
|
|
9198
|
+
apiKeySource: keyHit.key,
|
|
9199
|
+
misconfigured: true,
|
|
9200
|
+
warning: warnings.join(" ")
|
|
9201
|
+
};
|
|
9202
|
+
}
|
|
9203
|
+
return {
|
|
9204
|
+
transport: "cloud-http",
|
|
9205
|
+
mode,
|
|
9206
|
+
deprecatedAlias,
|
|
9207
|
+
modeSource,
|
|
9208
|
+
baseUrl,
|
|
9209
|
+
apiUrlSource,
|
|
9210
|
+
apiKeyPresent: true,
|
|
9211
|
+
apiKeySource: keyHit.key,
|
|
9212
|
+
misconfigured: false,
|
|
9213
|
+
warning: warnings.length > 0 ? warnings.join(" ") : null
|
|
9214
|
+
};
|
|
9215
|
+
}
|
|
9216
|
+
|
|
9217
|
+
class HasnaHttpError extends Error {
|
|
9218
|
+
status;
|
|
9219
|
+
method;
|
|
9220
|
+
path;
|
|
9221
|
+
body;
|
|
9222
|
+
constructor(method, path, status, body) {
|
|
9223
|
+
super(`Hasna cloud request failed: ${method} ${path} -> ${status}`);
|
|
9224
|
+
this.name = "HasnaHttpError";
|
|
9225
|
+
this.status = status;
|
|
9226
|
+
this.method = method;
|
|
9227
|
+
this.path = path;
|
|
9228
|
+
this.body = body;
|
|
9229
|
+
}
|
|
9230
|
+
}
|
|
9231
|
+
var DEFAULT_RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504];
|
|
9232
|
+
var IDEMPOTENT_METHODS = new Set(["GET", "HEAD", "PUT", "DELETE", "OPTIONS"]);
|
|
9233
|
+
function appendQuery(path, query) {
|
|
9234
|
+
if (!query)
|
|
9235
|
+
return path;
|
|
9236
|
+
const params = query instanceof URLSearchParams ? query : new URLSearchParams;
|
|
9237
|
+
if (!(query instanceof URLSearchParams)) {
|
|
9238
|
+
for (const [key, value] of Object.entries(query)) {
|
|
9239
|
+
if (value === null || value === undefined)
|
|
9240
|
+
continue;
|
|
9241
|
+
if (Array.isArray(value)) {
|
|
9242
|
+
for (const v of value)
|
|
9243
|
+
params.append(key, String(v));
|
|
9244
|
+
} else {
|
|
9245
|
+
params.append(key, String(value));
|
|
9246
|
+
}
|
|
9247
|
+
}
|
|
9248
|
+
}
|
|
9249
|
+
const qs = params.toString();
|
|
9250
|
+
if (!qs)
|
|
9251
|
+
return path;
|
|
9252
|
+
return `${path}${path.includes("?") ? "&" : "?"}${qs}`;
|
|
9253
|
+
}
|
|
9254
|
+
var defaultSleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
9255
|
+
function createHasnaHttpTransport(options) {
|
|
9256
|
+
const fetchImpl = options.fetchImpl ?? ((input, init) => fetch(input, init));
|
|
9257
|
+
const base = options.baseUrl.replace(/\/+$/, "");
|
|
9258
|
+
const timeoutMs = options.timeoutMs ?? 30000;
|
|
9259
|
+
const sleep = options.sleepImpl ?? defaultSleep;
|
|
9260
|
+
const defaultRetry = options.retry;
|
|
9261
|
+
function resolveRetry(callRetry) {
|
|
9262
|
+
const chosen = callRetry !== undefined ? callRetry : defaultRetry;
|
|
9263
|
+
if (chosen === false)
|
|
9264
|
+
return null;
|
|
9265
|
+
const r = chosen ?? {};
|
|
9266
|
+
return {
|
|
9267
|
+
retries: r.retries ?? 2,
|
|
9268
|
+
baseDelayMs: r.baseDelayMs ?? 200,
|
|
9269
|
+
maxDelayMs: r.maxDelayMs ?? 2000,
|
|
9270
|
+
retryStatuses: r.retryStatuses ?? [...DEFAULT_RETRY_STATUSES]
|
|
9271
|
+
};
|
|
9272
|
+
}
|
|
9273
|
+
async function once2(method, rel, url, body, opts) {
|
|
9274
|
+
const headers = {
|
|
9275
|
+
"x-api-key": options.apiKey,
|
|
9276
|
+
Authorization: `Bearer ${options.apiKey}`,
|
|
9277
|
+
Accept: "application/json",
|
|
9278
|
+
...options.headers ?? {},
|
|
9279
|
+
...opts.headers ?? {}
|
|
9280
|
+
};
|
|
9281
|
+
if (opts.idempotencyKey)
|
|
9282
|
+
headers["Idempotency-Key"] = opts.idempotencyKey;
|
|
9283
|
+
const init = { method, headers };
|
|
9284
|
+
if (body !== undefined) {
|
|
9285
|
+
headers["Content-Type"] = "application/json";
|
|
9286
|
+
init.body = JSON.stringify(body);
|
|
9287
|
+
}
|
|
9288
|
+
const controller = new AbortController;
|
|
9289
|
+
const onAbort = () => controller.abort();
|
|
9290
|
+
if (opts.signal) {
|
|
9291
|
+
if (opts.signal.aborted)
|
|
9292
|
+
controller.abort();
|
|
9293
|
+
else
|
|
9294
|
+
opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
9295
|
+
}
|
|
9296
|
+
const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? timeoutMs);
|
|
9297
|
+
init.signal = controller.signal;
|
|
9298
|
+
let response;
|
|
9299
|
+
try {
|
|
9300
|
+
response = await fetchImpl(url, init);
|
|
9301
|
+
} catch (error) {
|
|
9302
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
9303
|
+
if (opts.signal?.aborted)
|
|
9304
|
+
return { ok: false, retryable: false, error: err };
|
|
9305
|
+
return { ok: false, retryable: true, error: err };
|
|
9306
|
+
} finally {
|
|
9307
|
+
clearTimeout(timer);
|
|
9308
|
+
if (opts.signal)
|
|
9309
|
+
opts.signal.removeEventListener("abort", onAbort);
|
|
9310
|
+
}
|
|
9311
|
+
const text = await response.text();
|
|
9312
|
+
let parsed = undefined;
|
|
9313
|
+
if (text.length > 0) {
|
|
9314
|
+
try {
|
|
9315
|
+
parsed = JSON.parse(text);
|
|
9316
|
+
} catch {
|
|
9317
|
+
parsed = text;
|
|
9318
|
+
}
|
|
9319
|
+
}
|
|
9320
|
+
if (!response.ok) {
|
|
9321
|
+
const retry = resolveRetry(opts.retry);
|
|
9322
|
+
const retryable = retry ? retry.retryStatuses.includes(response.status) : false;
|
|
9323
|
+
return { ok: false, retryable, error: new HasnaHttpError(method, rel, response.status, parsed) };
|
|
9324
|
+
}
|
|
9325
|
+
return { ok: true, value: parsed };
|
|
9326
|
+
}
|
|
9327
|
+
async function request(method, path, body, opts = {}) {
|
|
9328
|
+
const upper = method.toUpperCase();
|
|
9329
|
+
const rel = appendQuery(path.startsWith("/") ? path : `/${path}`, opts.query);
|
|
9330
|
+
const url = `${base}${rel}`;
|
|
9331
|
+
const retry = resolveRetry(opts.retry);
|
|
9332
|
+
const methodRetryable = IDEMPOTENT_METHODS.has(upper) || Boolean(opts.idempotencyKey);
|
|
9333
|
+
const maxAttempts = retry && methodRetryable ? retry.retries + 1 : 1;
|
|
9334
|
+
let last = null;
|
|
9335
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt++) {
|
|
9336
|
+
const result = await once2(upper, rel, url, body, opts);
|
|
9337
|
+
if (result.ok)
|
|
9338
|
+
return result.value;
|
|
9339
|
+
last = result;
|
|
9340
|
+
const canRetry = retry !== null && methodRetryable && result.retryable && attempt < maxAttempts;
|
|
9341
|
+
if (!canRetry)
|
|
9342
|
+
break;
|
|
9343
|
+
const backoff = Math.min(retry.maxDelayMs, retry.baseDelayMs * 2 ** (attempt - 1));
|
|
9344
|
+
const jitter = Math.floor(Math.random() * (backoff / 2 + 1));
|
|
9345
|
+
await sleep(backoff + jitter);
|
|
9346
|
+
}
|
|
9347
|
+
throw last.error;
|
|
9348
|
+
}
|
|
9349
|
+
return {
|
|
9350
|
+
baseUrl: base,
|
|
9351
|
+
request,
|
|
9352
|
+
get: (path, opts) => request("GET", path, undefined, opts),
|
|
9353
|
+
post: (path, body, opts) => request("POST", path, body, opts),
|
|
9354
|
+
put: (path, body, opts) => request("PUT", path, body, opts),
|
|
9355
|
+
patch: (path, body, opts) => request("PATCH", path, body, opts),
|
|
9356
|
+
del: (path, body, opts) => request("DELETE", path, body, opts)
|
|
9357
|
+
};
|
|
9358
|
+
}
|
|
9359
|
+
function createClientTransport(name, env = process.env, overrides) {
|
|
9360
|
+
const resolution = resolveClientTransport(name, env);
|
|
9361
|
+
if (resolution.misconfigured) {
|
|
9362
|
+
throw new Error(resolution.warning ?? `Client for '${name}' is misconfigured for cloud mode.`);
|
|
9363
|
+
}
|
|
9364
|
+
if (resolution.transport === "local" || !resolution.baseUrl) {
|
|
9365
|
+
return { transport: "local", client: null, resolution };
|
|
9366
|
+
}
|
|
9367
|
+
const keys = clientTransportEnvKeys(name);
|
|
9368
|
+
const apiKey = firstEnv(env, keys.apiKeyKeys)?.value;
|
|
9369
|
+
if (!apiKey) {
|
|
9370
|
+
throw new Error(`Client for '${name}' resolved to cloud-http without an API key.`);
|
|
9371
|
+
}
|
|
9372
|
+
return {
|
|
9373
|
+
transport: "cloud-http",
|
|
9374
|
+
client: createHasnaHttpTransport({
|
|
9375
|
+
name,
|
|
9376
|
+
baseUrl: resolution.baseUrl,
|
|
9377
|
+
apiKey,
|
|
9378
|
+
...overrides?.fetchImpl ? { fetchImpl: overrides.fetchImpl } : {},
|
|
9379
|
+
...overrides?.headers ? { headers: overrides.headers } : {},
|
|
9380
|
+
...overrides?.timeoutMs ? { timeoutMs: overrides.timeoutMs } : {},
|
|
9381
|
+
...overrides?.retry !== undefined ? { retry: overrides.retry } : {},
|
|
9382
|
+
...overrides?.sleepImpl ? { sleepImpl: overrides.sleepImpl } : {}
|
|
9383
|
+
}),
|
|
9384
|
+
resolution
|
|
9385
|
+
};
|
|
9386
|
+
}
|
|
9387
|
+
|
|
9388
|
+
// src/lib/cloud/storage.ts
|
|
9389
|
+
function resourcePath(resource) {
|
|
9390
|
+
const trimmed = resource.replace(/^\/+|\/+$/g, "");
|
|
9391
|
+
if (!trimmed)
|
|
9392
|
+
throw new Error("resource must be a non-empty path segment");
|
|
9393
|
+
return `/${trimmed}`;
|
|
9394
|
+
}
|
|
9395
|
+
function entityPath(resource, id) {
|
|
9396
|
+
if (id === undefined || id === null || `${id}`.length === 0) {
|
|
9397
|
+
throw new Error("id must be a non-empty string");
|
|
9398
|
+
}
|
|
9399
|
+
return `${resourcePath(resource)}/${encodeURIComponent(String(id))}`;
|
|
9400
|
+
}
|
|
9401
|
+
function newIdempotencyKey() {
|
|
9402
|
+
const g = globalThis;
|
|
9403
|
+
if (g.crypto?.randomUUID)
|
|
9404
|
+
return g.crypto.randomUUID();
|
|
9405
|
+
return `idmp_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 12)}`;
|
|
9406
|
+
}
|
|
9407
|
+
function extractItems(raw) {
|
|
9408
|
+
if (Array.isArray(raw))
|
|
9409
|
+
return raw;
|
|
9410
|
+
if (raw && typeof raw === "object") {
|
|
9411
|
+
const obj = raw;
|
|
9412
|
+
for (const key of ["items", "data", "results", "rows", "records"]) {
|
|
9413
|
+
if (Array.isArray(obj[key]))
|
|
9414
|
+
return obj[key];
|
|
9415
|
+
}
|
|
9416
|
+
}
|
|
9417
|
+
return [];
|
|
9418
|
+
}
|
|
9419
|
+
function extractTotal(raw) {
|
|
9420
|
+
if (raw && typeof raw === "object") {
|
|
9421
|
+
const obj = raw;
|
|
9422
|
+
for (const key of ["total", "count", "totalCount", "total_count"]) {
|
|
9423
|
+
if (typeof obj[key] === "number")
|
|
9424
|
+
return obj[key];
|
|
9425
|
+
}
|
|
9426
|
+
}
|
|
9427
|
+
return null;
|
|
9428
|
+
}
|
|
9429
|
+
function extractCursor(raw) {
|
|
9430
|
+
if (raw && typeof raw === "object") {
|
|
9431
|
+
const obj = raw;
|
|
9432
|
+
for (const key of ["cursor", "nextCursor", "next_cursor", "next"]) {
|
|
9433
|
+
if (typeof obj[key] === "string")
|
|
9434
|
+
return obj[key];
|
|
9435
|
+
}
|
|
9436
|
+
}
|
|
9437
|
+
return null;
|
|
9438
|
+
}
|
|
9439
|
+
function createHasnaStorageClient(name, transport) {
|
|
9440
|
+
return {
|
|
9441
|
+
name,
|
|
9442
|
+
baseUrl: transport.baseUrl,
|
|
9443
|
+
transport,
|
|
9444
|
+
async list(resource, options = {}) {
|
|
9445
|
+
const raw = await transport.get(resourcePath(resource), options);
|
|
9446
|
+
return {
|
|
9447
|
+
items: extractItems(raw),
|
|
9448
|
+
total: extractTotal(raw),
|
|
9449
|
+
cursor: extractCursor(raw),
|
|
9450
|
+
raw
|
|
9451
|
+
};
|
|
9452
|
+
},
|
|
9453
|
+
async get(resource, id, options = {}) {
|
|
9454
|
+
try {
|
|
9455
|
+
return await transport.get(entityPath(resource, id), options);
|
|
9456
|
+
} catch (error) {
|
|
9457
|
+
if (error instanceof HasnaHttpError && error.status === 404)
|
|
9458
|
+
return null;
|
|
9459
|
+
throw error;
|
|
9460
|
+
}
|
|
9461
|
+
},
|
|
9462
|
+
async create(resource, body, options = {}) {
|
|
9463
|
+
const { idempotencyKey, ...rest } = options;
|
|
9464
|
+
return transport.post(resourcePath(resource), body, {
|
|
9465
|
+
...rest,
|
|
9466
|
+
idempotencyKey: idempotencyKey ?? newIdempotencyKey()
|
|
9467
|
+
});
|
|
9468
|
+
},
|
|
9469
|
+
async update(resource, id, patch, options = {}) {
|
|
9470
|
+
const { method = "PATCH", idempotencyKey, ...rest } = options;
|
|
9471
|
+
const call = method === "PUT" ? transport.put : transport.patch;
|
|
9472
|
+
return call(entityPath(resource, id), patch, { ...rest, ...idempotencyKey ? { idempotencyKey } : {} });
|
|
9473
|
+
},
|
|
9474
|
+
async delete(resource, id, options = {}) {
|
|
9475
|
+
try {
|
|
9476
|
+
await transport.del(entityPath(resource, id), undefined, options);
|
|
9477
|
+
} catch (error) {
|
|
9478
|
+
if (error instanceof HasnaHttpError && error.status === 404)
|
|
9479
|
+
return;
|
|
9480
|
+
throw error;
|
|
9481
|
+
}
|
|
9482
|
+
}
|
|
9483
|
+
};
|
|
9484
|
+
}
|
|
9485
|
+
|
|
9486
|
+
// src/lib/cloud/resolve.ts
|
|
9487
|
+
function firstValue(env, keys) {
|
|
9488
|
+
for (const key of keys) {
|
|
9489
|
+
const value = env[key]?.trim();
|
|
9490
|
+
if (value)
|
|
9491
|
+
return value;
|
|
9492
|
+
}
|
|
9493
|
+
return;
|
|
9494
|
+
}
|
|
9495
|
+
function resolveCloudStorage(name, env = process.env) {
|
|
9496
|
+
const token = envToken(name);
|
|
9497
|
+
const modeKeys = [`HASNA_${token}_STORAGE_MODE`, `HASNA_${token}_MODE`, `${token}_STORAGE_MODE`, `${token}_MODE`];
|
|
9498
|
+
const apiUrlKeys = [`HASNA_${token}_API_URL`, `${token}_API_URL`];
|
|
9499
|
+
const apiKeyKeys = [
|
|
9500
|
+
`HASNA_${token}_API_KEY`,
|
|
9501
|
+
`${token}_API_KEY`,
|
|
9502
|
+
`HASNA_${token}_API_TOKEN`,
|
|
9503
|
+
`${token}_API_TOKEN`
|
|
9504
|
+
];
|
|
9505
|
+
const explicitMode = firstValue(env, modeKeys);
|
|
9506
|
+
if (explicitMode && normalizeStorageMode(explicitMode).mode === "local") {
|
|
9507
|
+
return { transport: "local", client: null };
|
|
9508
|
+
}
|
|
9509
|
+
const apiUrl = firstValue(env, apiUrlKeys);
|
|
9510
|
+
const apiKey = firstValue(env, apiKeyKeys);
|
|
9511
|
+
if (!apiUrl || !apiKey) {
|
|
9512
|
+
return { transport: "local", client: null };
|
|
9513
|
+
}
|
|
9514
|
+
const cloudEnv = { ...env, [`HASNA_${token}_STORAGE_MODE`]: "self_hosted" };
|
|
9515
|
+
const wired = createClientTransport(name, cloudEnv);
|
|
9516
|
+
if (wired.transport !== "cloud-http") {
|
|
9517
|
+
return { transport: "local", client: null };
|
|
9518
|
+
}
|
|
9519
|
+
return {
|
|
9520
|
+
transport: "cloud-http",
|
|
9521
|
+
client: createHasnaStorageClient(name, wired.client),
|
|
9522
|
+
baseUrl: wired.client.baseUrl
|
|
9523
|
+
};
|
|
9524
|
+
}
|
|
9525
|
+
|
|
9526
|
+
// src/lib/store/index.ts
|
|
9527
|
+
class CloudUnsupportedError extends Error {
|
|
9528
|
+
constructor(operation) {
|
|
9529
|
+
super(`operation not supported over the hosted OpenLoops API: ${operation}. ` + `Run it on a machine using local storage, or unset HASNA_LOOPS_API_URL/HASNA_LOOPS_API_KEY.`);
|
|
9530
|
+
this.name = "CloudUnsupportedError";
|
|
9531
|
+
}
|
|
9532
|
+
}
|
|
9533
|
+
|
|
9534
|
+
class LocalStore {
|
|
9535
|
+
transport = "local";
|
|
9536
|
+
store;
|
|
9537
|
+
constructor(store = new Store) {
|
|
9538
|
+
this.store = store;
|
|
9539
|
+
}
|
|
9540
|
+
get raw() {
|
|
9541
|
+
return this.store;
|
|
9542
|
+
}
|
|
9543
|
+
async close() {
|
|
9544
|
+
this.store.close();
|
|
9545
|
+
}
|
|
9546
|
+
async createLoop(input, from) {
|
|
9547
|
+
return from ? this.store.createLoop(input, from) : this.store.createLoop(input);
|
|
9548
|
+
}
|
|
9549
|
+
async getLoop(id) {
|
|
9550
|
+
return this.store.getLoop(id);
|
|
9551
|
+
}
|
|
9552
|
+
async findLoopByName(name) {
|
|
9553
|
+
return this.store.findLoopByName(name);
|
|
9554
|
+
}
|
|
9555
|
+
async requireLoop(idOrName) {
|
|
9556
|
+
return this.store.requireLoop(idOrName);
|
|
9557
|
+
}
|
|
9558
|
+
async requireUniqueLoop(idOrName) {
|
|
9559
|
+
return this.store.requireUniqueLoop(idOrName);
|
|
9560
|
+
}
|
|
9561
|
+
async listLoops(opts = {}) {
|
|
9562
|
+
return this.store.listLoops(opts);
|
|
9563
|
+
}
|
|
9564
|
+
async countLoops(status, opts = {}) {
|
|
9565
|
+
return this.store.countLoops(status, opts);
|
|
9566
|
+
}
|
|
9567
|
+
async updateLoop(id, patch) {
|
|
9568
|
+
return this.store.updateLoop(id, patch);
|
|
9569
|
+
}
|
|
9570
|
+
async renameLoop(id, name) {
|
|
9571
|
+
return this.store.renameLoop(id, name);
|
|
9572
|
+
}
|
|
9573
|
+
async archiveLoop(idOrName) {
|
|
9574
|
+
return this.store.archiveLoop(idOrName);
|
|
9575
|
+
}
|
|
9576
|
+
async unarchiveLoop(idOrName) {
|
|
9577
|
+
return this.store.unarchiveLoop(idOrName);
|
|
9578
|
+
}
|
|
9579
|
+
async deleteLoop(idOrName) {
|
|
9580
|
+
return this.store.deleteLoop(idOrName);
|
|
9581
|
+
}
|
|
9582
|
+
async createWorkflow(input) {
|
|
9583
|
+
return this.store.createWorkflow(input);
|
|
9584
|
+
}
|
|
9585
|
+
async getWorkflow(id) {
|
|
9586
|
+
return this.store.getWorkflow(id);
|
|
9587
|
+
}
|
|
9588
|
+
async findWorkflowByName(name) {
|
|
9589
|
+
return this.store.findWorkflowByName(name);
|
|
9590
|
+
}
|
|
9591
|
+
async requireWorkflow(idOrName) {
|
|
9592
|
+
return this.store.requireWorkflow(idOrName);
|
|
9593
|
+
}
|
|
9594
|
+
async listWorkflows(opts = {}) {
|
|
9595
|
+
return this.store.listWorkflows(opts);
|
|
9596
|
+
}
|
|
9597
|
+
async countWorkflows(opts = {}) {
|
|
9598
|
+
return this.store.countWorkflows(opts);
|
|
9599
|
+
}
|
|
9600
|
+
async archiveWorkflow(idOrName) {
|
|
9601
|
+
return this.store.archiveWorkflow(idOrName);
|
|
9602
|
+
}
|
|
9603
|
+
async getWorkflowRun(id) {
|
|
9604
|
+
return this.store.getWorkflowRun(id);
|
|
9605
|
+
}
|
|
9606
|
+
async requireWorkflowRun(id) {
|
|
9607
|
+
return this.store.requireWorkflowRun(id);
|
|
9608
|
+
}
|
|
9609
|
+
async listWorkflowRuns(opts = {}) {
|
|
9610
|
+
return this.store.listWorkflowRuns(opts);
|
|
9611
|
+
}
|
|
9612
|
+
async listWorkflowStepRuns(workflowRunId) {
|
|
9613
|
+
return this.store.listWorkflowStepRuns(workflowRunId);
|
|
9614
|
+
}
|
|
9615
|
+
async listWorkflowEvents(workflowRunId, limit) {
|
|
9616
|
+
return limit === undefined ? this.store.listWorkflowEvents(workflowRunId) : this.store.listWorkflowEvents(workflowRunId, limit);
|
|
9617
|
+
}
|
|
9618
|
+
async recoverWorkflowRun(workflowRunId, reason) {
|
|
9619
|
+
return reason === undefined ? this.store.recoverWorkflowRun(workflowRunId) : this.store.recoverWorkflowRun(workflowRunId, reason);
|
|
9620
|
+
}
|
|
9621
|
+
async cancelWorkflowRun(workflowRunId, reason) {
|
|
9622
|
+
return reason === undefined ? this.store.cancelWorkflowRun(workflowRunId) : this.store.cancelWorkflowRun(workflowRunId, reason);
|
|
9623
|
+
}
|
|
9624
|
+
async listWorkflowWorkItems(opts = {}) {
|
|
9625
|
+
return this.store.listWorkflowWorkItems(opts);
|
|
9626
|
+
}
|
|
9627
|
+
async getWorkflowWorkItem(id) {
|
|
9628
|
+
return this.store.getWorkflowWorkItem(id);
|
|
9629
|
+
}
|
|
9630
|
+
async requeueWorkflowWorkItem(id, patch = {}) {
|
|
9631
|
+
return this.store.requeueWorkflowWorkItem(id, patch);
|
|
9632
|
+
}
|
|
9633
|
+
async listWorkflowInvocations(opts = {}) {
|
|
9634
|
+
return this.store.listWorkflowInvocations(opts);
|
|
9635
|
+
}
|
|
9636
|
+
async getWorkflowInvocation(id) {
|
|
9637
|
+
return this.store.getWorkflowInvocation(id);
|
|
9638
|
+
}
|
|
9639
|
+
async getGoal(id) {
|
|
9640
|
+
return this.store.getGoal(id);
|
|
9641
|
+
}
|
|
9642
|
+
async findGoalByLoop(idOrName) {
|
|
9643
|
+
return this.store.findGoalByLoop(idOrName);
|
|
9644
|
+
}
|
|
9645
|
+
async findGoalByRunId(id) {
|
|
9646
|
+
return this.store.findGoalByRunId(id);
|
|
9647
|
+
}
|
|
9648
|
+
async listGoals(opts = {}) {
|
|
9649
|
+
return this.store.listGoals(opts);
|
|
9650
|
+
}
|
|
9651
|
+
async listGoalPlanNodes(goalIdOrPlanId) {
|
|
9652
|
+
return this.store.listGoalPlanNodes(goalIdOrPlanId);
|
|
9653
|
+
}
|
|
9654
|
+
async listGoalRuns(opts = {}) {
|
|
9655
|
+
return this.store.listGoalRuns(opts);
|
|
9656
|
+
}
|
|
9657
|
+
async listRuns(opts = {}) {
|
|
9658
|
+
return this.store.listRuns(opts);
|
|
9659
|
+
}
|
|
9660
|
+
async getRun(id) {
|
|
9661
|
+
return this.store.getRun(id);
|
|
9662
|
+
}
|
|
9663
|
+
async writeRunReceipt(input) {
|
|
9664
|
+
return this.store.writeRunReceipt(input);
|
|
9665
|
+
}
|
|
9666
|
+
async getRunReceipt(runId) {
|
|
9667
|
+
return this.store.getRunReceipt(runId);
|
|
9668
|
+
}
|
|
9669
|
+
async listRunReceipts(opts = {}) {
|
|
9670
|
+
return this.store.listRunReceipts(opts);
|
|
9671
|
+
}
|
|
9672
|
+
async pruneHistory(opts) {
|
|
9673
|
+
return this.store.pruneHistory(opts);
|
|
9674
|
+
}
|
|
9675
|
+
}
|
|
9676
|
+
function clean(query) {
|
|
9677
|
+
const out = {};
|
|
9678
|
+
for (const [key, value] of Object.entries(query)) {
|
|
9679
|
+
if (value !== undefined && value !== null && value !== "")
|
|
9680
|
+
out[key] = value;
|
|
9681
|
+
}
|
|
9682
|
+
return out;
|
|
9683
|
+
}
|
|
9684
|
+
function pickArray(raw, key, fallback = []) {
|
|
9685
|
+
if (raw && typeof raw === "object" && Array.isArray(raw[key])) {
|
|
9686
|
+
return raw[key];
|
|
9687
|
+
}
|
|
9688
|
+
return fallback;
|
|
9689
|
+
}
|
|
9690
|
+
function pickObject(raw, key) {
|
|
9691
|
+
if (raw && typeof raw === "object") {
|
|
9692
|
+
const value = raw[key];
|
|
9693
|
+
return value == null ? undefined : value;
|
|
9694
|
+
}
|
|
9695
|
+
return raw == null ? undefined : raw;
|
|
9696
|
+
}
|
|
9697
|
+
|
|
9698
|
+
class ApiStore {
|
|
9699
|
+
client;
|
|
9700
|
+
baseUrl;
|
|
9701
|
+
transport = "cloud-http";
|
|
9702
|
+
constructor(client, baseUrl) {
|
|
9703
|
+
this.client = client;
|
|
9704
|
+
this.baseUrl = baseUrl;
|
|
9705
|
+
}
|
|
9706
|
+
get t() {
|
|
9707
|
+
return this.client.transport;
|
|
9708
|
+
}
|
|
9709
|
+
async close() {}
|
|
9710
|
+
async createLoop(input) {
|
|
9711
|
+
return pickObject(await this.t.post("/loops", input), "loop");
|
|
9712
|
+
}
|
|
9713
|
+
async getLoop(id) {
|
|
9714
|
+
try {
|
|
9715
|
+
return pickObject(await this.t.get(`/loops/${encodeURIComponent(id)}`), "loop");
|
|
9716
|
+
} catch {
|
|
9717
|
+
return;
|
|
9718
|
+
}
|
|
9719
|
+
}
|
|
9720
|
+
async findLoopByName(name) {
|
|
9721
|
+
const matches = (await this.listLoops({ name, limit: 1000 })).filter((loop) => loop.name === name);
|
|
9722
|
+
return matches[0];
|
|
9723
|
+
}
|
|
9724
|
+
async requireLoop(idOrName) {
|
|
9725
|
+
const loop = await this.resolveLoop(idOrName);
|
|
9726
|
+
if (!loop)
|
|
9727
|
+
throw new LoopNotFoundError(idOrName);
|
|
9728
|
+
return loop;
|
|
9729
|
+
}
|
|
9730
|
+
async requireUniqueLoop(idOrName) {
|
|
9731
|
+
const byId = await this.getLoop(idOrName);
|
|
9732
|
+
if (byId)
|
|
9733
|
+
return byId;
|
|
9734
|
+
const matches = (await this.listLoops({ name: idOrName, limit: 1000 })).filter((loop) => loop.name === idOrName);
|
|
9735
|
+
if (matches.length === 0)
|
|
9736
|
+
throw new LoopNotFoundError(idOrName);
|
|
9737
|
+
if (matches.length === 1)
|
|
9738
|
+
return matches[0];
|
|
9739
|
+
const active = matches.filter((loop) => !loop.archivedAt);
|
|
9740
|
+
if (active.length !== 1)
|
|
9741
|
+
throw new AmbiguousNameError(idOrName);
|
|
9742
|
+
return active[0];
|
|
9743
|
+
}
|
|
9744
|
+
async resolveLoop(idOrName) {
|
|
9745
|
+
const byId = await this.getLoop(idOrName);
|
|
9746
|
+
if (byId)
|
|
9747
|
+
return byId;
|
|
9748
|
+
const matches = (await this.listLoops({ name: idOrName, limit: 1000 })).filter((loop) => loop.name === idOrName);
|
|
9749
|
+
if (matches.length === 0)
|
|
9750
|
+
return;
|
|
9751
|
+
if (matches.length === 1)
|
|
9752
|
+
return matches[0];
|
|
9753
|
+
const active = matches.filter((loop) => !loop.archivedAt);
|
|
9754
|
+
if (active.length === 1)
|
|
9755
|
+
return active[0];
|
|
9756
|
+
throw new AmbiguousNameError(idOrName);
|
|
9757
|
+
}
|
|
9758
|
+
async listLoops(opts = {}) {
|
|
9759
|
+
const raw = await this.t.get("/loops", { query: clean({ ...opts }) });
|
|
9760
|
+
return pickArray(raw, "loops");
|
|
9761
|
+
}
|
|
9762
|
+
async countLoops(status, opts = {}) {
|
|
9763
|
+
const raw = await this.t.get("/loops/count", { query: clean({ status, ...opts }) });
|
|
9764
|
+
return Number(pickObject(raw, "count") ?? 0);
|
|
9765
|
+
}
|
|
9766
|
+
async updateLoop(id, patch) {
|
|
9767
|
+
return pickObject(await this.t.patch(`/loops/${encodeURIComponent(id)}`, patch), "loop");
|
|
9768
|
+
}
|
|
9769
|
+
async renameLoop(id, name) {
|
|
9770
|
+
return pickObject(await this.t.post(`/loops/${encodeURIComponent(id)}/rename`, { name }), "loop");
|
|
9771
|
+
}
|
|
9772
|
+
async archiveLoop(idOrName) {
|
|
9773
|
+
const loop = await this.requireLoop(idOrName);
|
|
9774
|
+
return pickObject(await this.t.post(`/loops/${encodeURIComponent(loop.id)}/archive`), "loop");
|
|
9775
|
+
}
|
|
9776
|
+
async unarchiveLoop(idOrName) {
|
|
9777
|
+
const loop = await this.requireLoop(idOrName);
|
|
9778
|
+
return pickObject(await this.t.post(`/loops/${encodeURIComponent(loop.id)}/unarchive`), "loop");
|
|
9779
|
+
}
|
|
9780
|
+
async deleteLoop(idOrName) {
|
|
9781
|
+
const loop = await this.resolveLoop(idOrName);
|
|
9782
|
+
if (!loop)
|
|
9783
|
+
return false;
|
|
9784
|
+
const raw = await this.t.request("DELETE", `/loops/${encodeURIComponent(loop.id)}`);
|
|
9785
|
+
return Boolean(pickObject(raw, "deleted") ?? true);
|
|
9786
|
+
}
|
|
9787
|
+
async createWorkflow(input) {
|
|
9788
|
+
return pickObject(await this.t.post("/workflows", input), "workflow");
|
|
9789
|
+
}
|
|
9790
|
+
async getWorkflow(id) {
|
|
9791
|
+
try {
|
|
9792
|
+
return pickObject(await this.t.get(`/workflows/${encodeURIComponent(id)}`), "workflow");
|
|
9793
|
+
} catch {
|
|
9794
|
+
return;
|
|
9795
|
+
}
|
|
9796
|
+
}
|
|
9797
|
+
async findWorkflowByName(name) {
|
|
9798
|
+
const matches = (await this.listWorkflows({ limit: 1000 })).filter((wf) => wf.name === name);
|
|
9799
|
+
return matches[0];
|
|
9800
|
+
}
|
|
9801
|
+
async requireWorkflow(idOrName) {
|
|
9802
|
+
const byId = await this.getWorkflow(idOrName);
|
|
9803
|
+
if (byId)
|
|
9804
|
+
return byId;
|
|
9805
|
+
const byName = await this.findWorkflowByName(idOrName);
|
|
9806
|
+
if (byName)
|
|
9807
|
+
return byName;
|
|
9808
|
+
throw new Error(`workflow not found: ${idOrName}`);
|
|
9809
|
+
}
|
|
9810
|
+
async listWorkflows(opts = {}) {
|
|
9811
|
+
const raw = await this.t.get("/workflows", { query: clean({ ...opts }) });
|
|
9812
|
+
return pickArray(raw, "workflows");
|
|
9813
|
+
}
|
|
9814
|
+
async countWorkflows(opts = {}) {
|
|
9815
|
+
const raw = await this.t.get("/workflows/count", { query: clean({ ...opts }) });
|
|
9816
|
+
return Number(pickObject(raw, "count") ?? 0);
|
|
9817
|
+
}
|
|
9818
|
+
async archiveWorkflow(idOrName) {
|
|
9819
|
+
const wf = await this.requireWorkflow(idOrName);
|
|
9820
|
+
return pickObject(await this.t.post(`/workflows/${encodeURIComponent(wf.id)}/archive`), "workflow");
|
|
9821
|
+
}
|
|
9822
|
+
async getWorkflowRun(id) {
|
|
9823
|
+
try {
|
|
9824
|
+
return pickObject(await this.t.get(`/workflow-runs/${encodeURIComponent(id)}`), "workflowRun");
|
|
9825
|
+
} catch {
|
|
9826
|
+
return;
|
|
9827
|
+
}
|
|
9828
|
+
}
|
|
9829
|
+
async requireWorkflowRun(id) {
|
|
9830
|
+
const run = await this.getWorkflowRun(id);
|
|
9831
|
+
if (!run)
|
|
9832
|
+
throw new Error(`workflow run not found: ${id}`);
|
|
9833
|
+
return run;
|
|
9834
|
+
}
|
|
9835
|
+
async listWorkflowRuns(opts = {}) {
|
|
9836
|
+
const raw = await this.t.get("/workflow-runs", { query: clean({ ...opts }) });
|
|
9837
|
+
return pickArray(raw, "workflowRuns");
|
|
9838
|
+
}
|
|
9839
|
+
async listWorkflowStepRuns(workflowRunId) {
|
|
9840
|
+
const raw = await this.t.get(`/workflow-runs/${encodeURIComponent(workflowRunId)}/steps`);
|
|
9841
|
+
return pickArray(raw, "steps");
|
|
9842
|
+
}
|
|
9843
|
+
async listWorkflowEvents(workflowRunId, limit) {
|
|
9844
|
+
const raw = await this.t.get(`/workflow-runs/${encodeURIComponent(workflowRunId)}/events`, { query: clean({ limit }) });
|
|
9845
|
+
return pickArray(raw, "events");
|
|
9846
|
+
}
|
|
9847
|
+
async recoverWorkflowRun(workflowRunId, reason) {
|
|
9848
|
+
const raw = await this.t.post(`/workflow-runs/${encodeURIComponent(workflowRunId)}/recover`, { reason });
|
|
9849
|
+
return {
|
|
9850
|
+
run: pickObject(raw, "workflowRun"),
|
|
9851
|
+
recoveredSteps: pickArray(raw, "recoveredSteps")
|
|
9852
|
+
};
|
|
9853
|
+
}
|
|
9854
|
+
async cancelWorkflowRun(_workflowRunId, _reason) {
|
|
9855
|
+
throw new CloudUnsupportedError("workflows cancel");
|
|
9856
|
+
}
|
|
9857
|
+
async listWorkflowWorkItems(opts = {}) {
|
|
9858
|
+
const raw = await this.t.get("/work-items", { query: clean({ ...opts }) });
|
|
9859
|
+
return pickArray(raw, "workItems");
|
|
9860
|
+
}
|
|
9861
|
+
async getWorkflowWorkItem(id) {
|
|
9862
|
+
try {
|
|
9863
|
+
return pickObject(await this.t.get(`/work-items/${encodeURIComponent(id)}`), "workItem");
|
|
9864
|
+
} catch {
|
|
9865
|
+
return;
|
|
7683
9866
|
}
|
|
7684
|
-
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
7685
|
-
deps.onRun?.(skipped);
|
|
7686
|
-
return skipped;
|
|
7687
9867
|
}
|
|
7688
|
-
|
|
7689
|
-
|
|
7690
|
-
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
7691
|
-
} catch (error) {
|
|
7692
|
-
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
7693
|
-
return;
|
|
7694
|
-
throw error;
|
|
9868
|
+
async requeueWorkflowWorkItem(_id, _patch = {}) {
|
|
9869
|
+
throw new CloudUnsupportedError("routes requeue");
|
|
7695
9870
|
}
|
|
7696
|
-
|
|
7697
|
-
|
|
7698
|
-
return;
|
|
9871
|
+
async listWorkflowInvocations(opts = {}) {
|
|
9872
|
+
const raw = await this.t.get("/invocations", { query: clean({ ...opts }) });
|
|
9873
|
+
return pickArray(raw, "invocations");
|
|
7699
9874
|
}
|
|
7700
|
-
|
|
7701
|
-
|
|
7702
|
-
|
|
7703
|
-
|
|
7704
|
-
runnerId: deps.runnerId,
|
|
7705
|
-
loop: claim.loop,
|
|
7706
|
-
run: claim.run,
|
|
7707
|
-
now: deps.now,
|
|
7708
|
-
execute: deps.execute,
|
|
7709
|
-
beforeFinalize: deps.beforeFinalize,
|
|
7710
|
-
daemonLeaseId: deps.daemonLeaseId,
|
|
7711
|
-
onError: deps.onError
|
|
7712
|
-
});
|
|
7713
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
|
|
7714
|
-
deps.onRun?.(finalRun);
|
|
7715
|
-
return finalRun;
|
|
7716
|
-
}
|
|
7717
|
-
function claimSlot(deps, loop, scheduledFor) {
|
|
7718
|
-
const now = deps.now?.() ?? new Date;
|
|
7719
|
-
deps.beforeRun?.(loop, scheduledFor);
|
|
7720
|
-
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
7721
|
-
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
9875
|
+
async getWorkflowInvocation(id) {
|
|
9876
|
+
try {
|
|
9877
|
+
return pickObject(await this.t.get(`/invocations/${encodeURIComponent(id)}`), "invocation");
|
|
9878
|
+
} catch {
|
|
7722
9879
|
return;
|
|
7723
|
-
|
|
9880
|
+
}
|
|
9881
|
+
}
|
|
9882
|
+
async getGoal(id) {
|
|
7724
9883
|
try {
|
|
7725
|
-
|
|
7726
|
-
|
|
7727
|
-
|
|
7728
|
-
} catch (error) {
|
|
7729
|
-
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
7730
|
-
return;
|
|
7731
|
-
throw error;
|
|
9884
|
+
return pickObject(await this.t.get(`/goals/${encodeURIComponent(id)}`), "goal");
|
|
9885
|
+
} catch {
|
|
9886
|
+
return;
|
|
7732
9887
|
}
|
|
7733
|
-
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
7734
|
-
deps.onRun?.(skipped);
|
|
7735
|
-
return skipped;
|
|
7736
9888
|
}
|
|
7737
|
-
|
|
7738
|
-
|
|
7739
|
-
|
|
7740
|
-
|
|
7741
|
-
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
9889
|
+
async findGoalByLoop(idOrName) {
|
|
9890
|
+
try {
|
|
9891
|
+
return pickObject(await this.t.get("/goals", { query: clean({ loop: idOrName }) }), "goal");
|
|
9892
|
+
} catch {
|
|
7742
9893
|
return;
|
|
7743
|
-
|
|
9894
|
+
}
|
|
7744
9895
|
}
|
|
7745
|
-
|
|
7746
|
-
|
|
7747
|
-
|
|
9896
|
+
async findGoalByRunId(id) {
|
|
9897
|
+
try {
|
|
9898
|
+
return pickObject(await this.t.get("/goals", { query: clean({ runId: id }) }), "goal");
|
|
9899
|
+
} catch {
|
|
9900
|
+
return;
|
|
9901
|
+
}
|
|
7748
9902
|
}
|
|
7749
|
-
|
|
7750
|
-
|
|
7751
|
-
|
|
7752
|
-
}
|
|
7753
|
-
function recoverAndExpire(deps, now) {
|
|
7754
|
-
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
7755
|
-
const recoveredByLoop = new Map;
|
|
7756
|
-
for (const run of recovered) {
|
|
7757
|
-
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
9903
|
+
async listGoals(opts = {}) {
|
|
9904
|
+
const raw = await this.t.get("/goals", { query: clean({ ...opts }) });
|
|
9905
|
+
return pickArray(raw, "goals");
|
|
7758
9906
|
}
|
|
7759
|
-
|
|
7760
|
-
const
|
|
7761
|
-
|
|
7762
|
-
continue;
|
|
7763
|
-
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
7764
|
-
if (retryable) {
|
|
7765
|
-
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
|
|
7766
|
-
continue;
|
|
7767
|
-
}
|
|
7768
|
-
for (const run of runs) {
|
|
7769
|
-
const current = deps.store.getLoop(run.loopId);
|
|
7770
|
-
if (current) {
|
|
7771
|
-
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
|
|
7772
|
-
}
|
|
7773
|
-
}
|
|
9907
|
+
async listGoalPlanNodes(goalIdOrPlanId) {
|
|
9908
|
+
const raw = await this.t.get(`/goals/${encodeURIComponent(goalIdOrPlanId)}/plan-nodes`);
|
|
9909
|
+
return pickArray(raw, "nodes");
|
|
7774
9910
|
}
|
|
7775
|
-
|
|
7776
|
-
|
|
7777
|
-
|
|
7778
|
-
|
|
7779
|
-
|
|
7780
|
-
|
|
7781
|
-
|
|
7782
|
-
|
|
7783
|
-
|
|
7784
|
-
|
|
7785
|
-
|
|
7786
|
-
|
|
7787
|
-
|
|
7788
|
-
const laneClaims = { command: 0, agent: 0 };
|
|
7789
|
-
const laneCap = (lane) => laneLimits === undefined ? Number.POSITIVE_INFINITY : Math.max(0, laneLimits[lane] ?? Number.POSITIVE_INFINITY);
|
|
7790
|
-
const laneFull = (lane) => laneClaims[lane] >= laneCap(lane);
|
|
7791
|
-
for (const loop of deps.store.dueLoops(now)) {
|
|
7792
|
-
if (claims.length >= maxClaims)
|
|
7793
|
-
break;
|
|
7794
|
-
const lane = loopLane(loop);
|
|
7795
|
-
if (laneFull(lane))
|
|
7796
|
-
continue;
|
|
7797
|
-
const plan = dueSlots(loop, now);
|
|
7798
|
-
let loopSkips = 0;
|
|
7799
|
-
for (const slot of plan.slots) {
|
|
7800
|
-
if (claims.length >= maxClaims)
|
|
7801
|
-
break;
|
|
7802
|
-
if (laneFull(lane))
|
|
7803
|
-
break;
|
|
7804
|
-
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
7805
|
-
break;
|
|
7806
|
-
const run = claimSlot(deps, loop, slot);
|
|
7807
|
-
if (!run)
|
|
7808
|
-
continue;
|
|
7809
|
-
if ("loop" in run) {
|
|
7810
|
-
claims.push(run);
|
|
7811
|
-
claimed.push(run.run);
|
|
7812
|
-
laneClaims[lane] += 1;
|
|
7813
|
-
} else if (run.status === "skipped") {
|
|
7814
|
-
skipped.push(run);
|
|
7815
|
-
loopSkips += 1;
|
|
7816
|
-
}
|
|
9911
|
+
async listGoalRuns(opts = {}) {
|
|
9912
|
+
const raw = await this.t.get("/goal-runs", { query: clean({ ...opts }) });
|
|
9913
|
+
return pickArray(raw, "goalRuns");
|
|
9914
|
+
}
|
|
9915
|
+
async listRuns(opts = {}) {
|
|
9916
|
+
const raw = await this.t.get("/runs", { query: clean({ ...opts, showOutput: true }) });
|
|
9917
|
+
return pickArray(raw, "runs");
|
|
9918
|
+
}
|
|
9919
|
+
async getRun(id) {
|
|
9920
|
+
try {
|
|
9921
|
+
return pickObject(await this.t.get(`/runs/${encodeURIComponent(id)}`, { query: { showOutput: true } }), "run");
|
|
9922
|
+
} catch {
|
|
9923
|
+
return;
|
|
7817
9924
|
}
|
|
7818
9925
|
}
|
|
7819
|
-
|
|
7820
|
-
|
|
7821
|
-
|
|
7822
|
-
|
|
7823
|
-
|
|
7824
|
-
|
|
7825
|
-
|
|
7826
|
-
|
|
7827
|
-
for (const loop of deps.store.dueLoops(now)) {
|
|
7828
|
-
const plan = dueSlots(loop, now);
|
|
7829
|
-
let loopSkips = 0;
|
|
7830
|
-
for (const slot of plan.slots) {
|
|
7831
|
-
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
7832
|
-
break;
|
|
7833
|
-
const run = await runSlot(deps, loop, slot);
|
|
7834
|
-
if (!run)
|
|
7835
|
-
continue;
|
|
7836
|
-
if (run.status === "running")
|
|
7837
|
-
claimed.push(run);
|
|
7838
|
-
else if (run.status === "skipped") {
|
|
7839
|
-
skipped.push(run);
|
|
7840
|
-
loopSkips += 1;
|
|
7841
|
-
} else
|
|
7842
|
-
completed.push(run);
|
|
7843
|
-
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
7844
|
-
break;
|
|
9926
|
+
async writeRunReceipt(input) {
|
|
9927
|
+
return pickObject(await this.t.post("/receipts", input), "receipt");
|
|
9928
|
+
}
|
|
9929
|
+
async getRunReceipt(runId) {
|
|
9930
|
+
try {
|
|
9931
|
+
return pickObject(await this.t.get(`/receipts/${encodeURIComponent(runId)}`), "receipt");
|
|
9932
|
+
} catch {
|
|
9933
|
+
return;
|
|
7845
9934
|
}
|
|
7846
9935
|
}
|
|
7847
|
-
|
|
9936
|
+
async listRunReceipts(opts = {}) {
|
|
9937
|
+
const raw = await this.t.get("/receipts", { query: clean({ ...opts }) });
|
|
9938
|
+
return pickArray(raw, "receipts");
|
|
9939
|
+
}
|
|
9940
|
+
async pruneHistory(opts) {
|
|
9941
|
+
const raw = await this.t.post("/history/prune", {
|
|
9942
|
+
maxAgeDays: opts.maxAgeDays,
|
|
9943
|
+
keepPerLoop: opts.keepPerLoop,
|
|
9944
|
+
dryRun: opts.dryRun
|
|
9945
|
+
});
|
|
9946
|
+
return pickObject(raw, "history");
|
|
9947
|
+
}
|
|
9948
|
+
}
|
|
9949
|
+
function getStore(env = process.env) {
|
|
9950
|
+
const resolution = resolveCloudStorage("loops", env);
|
|
9951
|
+
if (resolution.transport === "cloud-http")
|
|
9952
|
+
return new ApiStore(resolution.client, resolution.baseUrl);
|
|
9953
|
+
return new LocalStore;
|
|
9954
|
+
}
|
|
9955
|
+
function isCloudStore(env = process.env) {
|
|
9956
|
+
return resolveCloudStorage("loops", env).transport === "cloud-http";
|
|
7848
9957
|
}
|
|
7849
9958
|
|
|
7850
9959
|
// src/mcp/index.ts
|
|
@@ -7862,6 +9971,34 @@ var loopIdOrNameSchema = z2.string().min(1).describe("Loop id or exact loop name
|
|
|
7862
9971
|
var workflowIdOrNameSchema = z2.string().min(1).describe("Workflow id or exact workflow name.");
|
|
7863
9972
|
var showOutputSchema = z2.boolean().optional().describe("Include raw stdout/stderr (default false: only redacted lengths are returned).");
|
|
7864
9973
|
var limitSchema = z2.number().int().min(1).max(MAX_LIMIT).optional().describe(`Maximum entries to return (1-${MAX_LIMIT}).`);
|
|
9974
|
+
var runReceiptSummarySchema = z2.object({
|
|
9975
|
+
text: z2.string().optional().describe("Short human summary. It is scrubbed and bounded before storage."),
|
|
9976
|
+
stdout_bytes: z2.number().int().min(0).optional().describe("Original stdout byte count."),
|
|
9977
|
+
stderr_bytes: z2.number().int().min(0).optional().describe("Original stderr byte count."),
|
|
9978
|
+
stdout_excerpt: z2.string().optional().describe("Bounded stdout excerpt. Raw unbounded stdout is never required."),
|
|
9979
|
+
stderr_excerpt: z2.string().optional().describe("Bounded stderr excerpt. Raw unbounded stderr is never required."),
|
|
9980
|
+
error: z2.string().optional().describe("Bounded error excerpt."),
|
|
9981
|
+
duration_ms: z2.number().int().min(0).optional().describe("Run duration in milliseconds.")
|
|
9982
|
+
});
|
|
9983
|
+
var runReceiptInputSchema = {
|
|
9984
|
+
loop_id: z2.string().min(1).optional().describe("OpenLoops loop id. Optional when run_id references an existing loop run."),
|
|
9985
|
+
run_id: z2.string().min(1).describe("Scheduler-neutral run id. Existing values are updated idempotently."),
|
|
9986
|
+
machine: z2.union([z2.string().min(1), z2.record(z2.string(), z2.unknown())]).optional().describe("Machine id/name or machine metadata object."),
|
|
9987
|
+
repo: z2.string().min(1).optional().describe("Repository path or owner/repo string. Defaults from the loop target cwd when possible."),
|
|
9988
|
+
task_ids: z2.array(z2.string().min(1)).optional().describe("Task ids associated with this run."),
|
|
9989
|
+
knowledge_ids: z2.array(z2.string().min(1)).optional().describe("Knowledge record ids associated with this run."),
|
|
9990
|
+
digest_id: z2.string().min(1).optional().describe("Stable digest id. Computed from normalized receipt content when omitted."),
|
|
9991
|
+
started_at: z2.string().nullable().optional().describe("Run start timestamp."),
|
|
9992
|
+
finished_at: z2.string().nullable().optional().describe("Run finish timestamp."),
|
|
9993
|
+
status: z2.string().min(1).optional().describe("Run status."),
|
|
9994
|
+
exit_code: z2.number().int().nullable().optional().describe("Process exit code."),
|
|
9995
|
+
summary: z2.union([z2.string(), runReceiptSummarySchema]).nullable().optional().describe("Bounded structured summary; may be a string shorthand."),
|
|
9996
|
+
evidence_paths: z2.array(z2.string().min(1)).optional().describe("Bounded paths to durable evidence artifacts."),
|
|
9997
|
+
stdout: z2.string().optional().describe("Optional raw stdout to summarize and bound before storage."),
|
|
9998
|
+
stderr: z2.string().optional().describe("Optional raw stderr to summarize and bound before storage."),
|
|
9999
|
+
error: z2.string().optional().describe("Optional raw error text to summarize and bound before storage."),
|
|
10000
|
+
duration_ms: z2.number().int().min(0).optional().describe("Run duration in milliseconds.")
|
|
10001
|
+
};
|
|
7865
10002
|
var optionalTimeoutSchema = z2.number().int().positive().nullable().optional().describe("Per-run timeout in milliseconds; null disables the timeout.");
|
|
7866
10003
|
var catchUpSchema = z2.enum(CATCH_UP_POLICIES).optional().describe("Missed-slot policy after downtime: none skips them, latest replays the newest slot, all replays every slot.");
|
|
7867
10004
|
var overlapSchema = z2.enum(OVERLAP_POLICIES).optional().describe("Behavior when a slot comes due while a previous run is active: skip records a skipped run, allow runs concurrently.");
|
|
@@ -7931,6 +10068,17 @@ function errorResult(error) {
|
|
|
7931
10068
|
};
|
|
7932
10069
|
}
|
|
7933
10070
|
async function withStore(fn) {
|
|
10071
|
+
const store = getStore();
|
|
10072
|
+
try {
|
|
10073
|
+
return await fn(store);
|
|
10074
|
+
} finally {
|
|
10075
|
+
await store.close();
|
|
10076
|
+
}
|
|
10077
|
+
}
|
|
10078
|
+
async function withLocalStore(operation, fn) {
|
|
10079
|
+
if (isCloudStore()) {
|
|
10080
|
+
throw new Error(`'${operation}' inspects this machine's local OpenLoops runtime and is not available while flipped to the hosted OpenLoops API. ` + `Unset HASNA_LOOPS_API_URL/HASNA_LOOPS_API_KEY (or set HASNA_LOOPS_STORAGE_MODE=local) to run it here.`);
|
|
10081
|
+
}
|
|
7934
10082
|
const store = new Store;
|
|
7935
10083
|
try {
|
|
7936
10084
|
return await fn(store);
|
|
@@ -7944,7 +10092,7 @@ function requireMutationsEnabled() {
|
|
|
7944
10092
|
throw new Error(`MCP mutation tools require ${MUTATION_ENV}=true`);
|
|
7945
10093
|
}
|
|
7946
10094
|
}
|
|
7947
|
-
function
|
|
10095
|
+
function nonEmpty2(value, label) {
|
|
7948
10096
|
const trimmed = value.trim();
|
|
7949
10097
|
if (!trimmed)
|
|
7950
10098
|
throw new Error(`${label} must be non-empty`);
|
|
@@ -7962,7 +10110,7 @@ function normalizeSchedule(input) {
|
|
|
7962
10110
|
if (input.type === "interval")
|
|
7963
10111
|
return { type: "interval", everyMs: input.everyMs, anchor: input.anchor ?? "fixed_rate" };
|
|
7964
10112
|
if (input.type === "cron")
|
|
7965
|
-
return { type: "cron", expression:
|
|
10113
|
+
return { type: "cron", expression: nonEmpty2(input.expression, "schedule.expression") };
|
|
7966
10114
|
return { type: "dynamic", minIntervalMs: input.minIntervalMs };
|
|
7967
10115
|
}
|
|
7968
10116
|
function durationLabel(ms) {
|
|
@@ -8005,7 +10153,7 @@ function defaultLoopDescription(name, schedule, target) {
|
|
|
8005
10153
|
].join(" ");
|
|
8006
10154
|
}
|
|
8007
10155
|
function commonCreateInput(input) {
|
|
8008
|
-
const name =
|
|
10156
|
+
const name = nonEmpty2(input.name, "name");
|
|
8009
10157
|
const schedule = normalizeSchedule(input.schedule);
|
|
8010
10158
|
return {
|
|
8011
10159
|
name,
|
|
@@ -8061,13 +10209,13 @@ var TOOL_REGISTRATIONS = [
|
|
|
8061
10209
|
includeArchived: z2.boolean().optional().describe("Include archived loops alongside live ones (default false)."),
|
|
8062
10210
|
archivedOnly: z2.boolean().optional().describe("Return only archived loops (default false).")
|
|
8063
10211
|
},
|
|
8064
|
-
handler: ({ status, limit, includeArchived, archivedOnly }) => withStore((store) => ({
|
|
8065
|
-
loops: store.listLoops({
|
|
10212
|
+
handler: ({ status, limit, includeArchived, archivedOnly }) => withStore(async (store) => ({
|
|
10213
|
+
loops: (await store.listLoops({
|
|
8066
10214
|
status: filteredLoopStatus(status),
|
|
8067
10215
|
limit,
|
|
8068
10216
|
includeArchived: includeArchived ?? false,
|
|
8069
10217
|
archived: archivedOnly ?? false
|
|
8070
|
-
}).map(publicLoop)
|
|
10218
|
+
})).map(publicLoop)
|
|
8071
10219
|
}))
|
|
8072
10220
|
},
|
|
8073
10221
|
{
|
|
@@ -8080,9 +10228,9 @@ var TOOL_REGISTRATIONS = [
|
|
|
8080
10228
|
includeLatestRun: z2.boolean().optional().describe("Include the most recent run record (default false)."),
|
|
8081
10229
|
showOutput: showOutputSchema
|
|
8082
10230
|
},
|
|
8083
|
-
handler: ({ idOrName, includeLatestRun, showOutput }) => withStore((store) => {
|
|
8084
|
-
const loop = store.requireLoop(idOrName);
|
|
8085
|
-
const latestRun = includeLatestRun ? store.listRuns({ loopId: loop.id, limit: 1 })[0] : undefined;
|
|
10231
|
+
handler: ({ idOrName, includeLatestRun, showOutput }) => withStore(async (store) => {
|
|
10232
|
+
const loop = await store.requireLoop(idOrName);
|
|
10233
|
+
const latestRun = includeLatestRun ? (await store.listRuns({ loopId: loop.id, limit: 1 }))[0] : undefined;
|
|
8086
10234
|
return {
|
|
8087
10235
|
loop: publicLoop(loop),
|
|
8088
10236
|
latestRun: latestRun ? publicRun(latestRun, showOutput ?? false) : undefined
|
|
@@ -8101,23 +10249,67 @@ var TOOL_REGISTRATIONS = [
|
|
|
8101
10249
|
limit: limitSchema,
|
|
8102
10250
|
showOutput: showOutputSchema
|
|
8103
10251
|
},
|
|
8104
|
-
handler: ({ idOrName, status, limit, showOutput }) => withStore((store) => {
|
|
8105
|
-
const loop = idOrName ? store.requireLoop(idOrName) : undefined;
|
|
8106
|
-
const runs = store.listRuns({
|
|
10252
|
+
handler: ({ idOrName, status, limit, showOutput }) => withStore(async (store) => {
|
|
10253
|
+
const loop = idOrName ? await store.requireLoop(idOrName) : undefined;
|
|
10254
|
+
const runs = (await store.listRuns({
|
|
8107
10255
|
loopId: loop?.id,
|
|
8108
10256
|
status,
|
|
8109
10257
|
limit
|
|
8110
|
-
}).map((run) => publicRun(run, showOutput ?? false));
|
|
10258
|
+
})).map((run) => publicRun(run, showOutput ?? false));
|
|
8111
10259
|
return { loop: loop ? publicLoop(loop) : undefined, runs };
|
|
8112
10260
|
})
|
|
8113
10261
|
},
|
|
10262
|
+
{
|
|
10263
|
+
name: "loops_receipts_list",
|
|
10264
|
+
description: "List scheduler-neutral run receipts with bounded summaries and evidence paths.",
|
|
10265
|
+
readOnly: true,
|
|
10266
|
+
annotations: READ_ONLY_ANNOTATIONS,
|
|
10267
|
+
inputSchema: {
|
|
10268
|
+
loop_id: z2.string().min(1).optional().describe("Filter by loop_id."),
|
|
10269
|
+
repo: z2.string().min(1).optional().describe("Filter by repo."),
|
|
10270
|
+
task_id: z2.string().min(1).optional().describe("Filter by task id."),
|
|
10271
|
+
knowledge_id: z2.string().min(1).optional().describe("Filter by knowledge id."),
|
|
10272
|
+
status: z2.string().min(1).optional().describe("Filter by receipt status."),
|
|
10273
|
+
limit: limitSchema
|
|
10274
|
+
},
|
|
10275
|
+
handler: ({ loop_id, repo, task_id, knowledge_id, status, limit }) => withStore(async (store) => ({
|
|
10276
|
+
receipts: (await store.listRunReceipts({ loopId: loop_id, repo, taskId: task_id, knowledgeId: knowledge_id, status, limit })).map(publicRunReceipt)
|
|
10277
|
+
}))
|
|
10278
|
+
},
|
|
10279
|
+
{
|
|
10280
|
+
name: "loops_receipt_read",
|
|
10281
|
+
description: "Read one scheduler-neutral run receipt by run id.",
|
|
10282
|
+
readOnly: true,
|
|
10283
|
+
annotations: READ_ONLY_ANNOTATIONS,
|
|
10284
|
+
inputSchema: {
|
|
10285
|
+
run_id: z2.string().min(1).describe("Run id.")
|
|
10286
|
+
},
|
|
10287
|
+
handler: ({ run_id }) => withStore(async (store) => {
|
|
10288
|
+
const receipt = await store.getRunReceipt(run_id);
|
|
10289
|
+
if (!receipt)
|
|
10290
|
+
throw new Error(`run receipt not found: ${run_id}`);
|
|
10291
|
+
return { receipt: publicRunReceipt(receipt) };
|
|
10292
|
+
})
|
|
10293
|
+
},
|
|
10294
|
+
{
|
|
10295
|
+
name: "loops_receipt_write",
|
|
10296
|
+
description: `Write a scheduler-neutral run receipt. Requires ${MUTATION_ENV}=true on the MCP server process.`,
|
|
10297
|
+
readOnly: false,
|
|
10298
|
+
guarded: true,
|
|
10299
|
+
annotations: mutationAnnotations({ idempotent: true }),
|
|
10300
|
+
inputSchema: runReceiptInputSchema,
|
|
10301
|
+
handler: (input) => {
|
|
10302
|
+
requireMutationsEnabled();
|
|
10303
|
+
return withStore(async (store) => ({ receipt: publicRunReceipt(await store.writeRunReceipt(input)) }));
|
|
10304
|
+
}
|
|
10305
|
+
},
|
|
8114
10306
|
{
|
|
8115
10307
|
name: "loops_doctor",
|
|
8116
10308
|
description: "Run OpenLoops runtime diagnostics.",
|
|
8117
10309
|
readOnly: true,
|
|
8118
10310
|
annotations: READ_ONLY_ANNOTATIONS,
|
|
8119
10311
|
inputSchema: {},
|
|
8120
|
-
handler: () =>
|
|
10312
|
+
handler: () => withLocalStore("loops_doctor", (store) => runDoctor(store))
|
|
8121
10313
|
},
|
|
8122
10314
|
{
|
|
8123
10315
|
name: "loops_health",
|
|
@@ -8129,7 +10321,33 @@ var TOOL_REGISTRATIONS = [
|
|
|
8129
10321
|
includeInactive: z2.boolean().optional().describe("Include stopped/expired loops (default false: active and paused only)."),
|
|
8130
10322
|
limit: limitSchema
|
|
8131
10323
|
},
|
|
8132
|
-
handler: ({ includeArchived, includeInactive, limit }) =>
|
|
10324
|
+
handler: ({ includeArchived, includeInactive, limit }) => withLocalStore("loops_health", (store) => buildHealthReport(store, { includeArchived, includeInactive, limit }))
|
|
10325
|
+
},
|
|
10326
|
+
{
|
|
10327
|
+
name: "loops_health_scan",
|
|
10328
|
+
description: "Build a read-only OpenLoops health scan with bounded daemon, doctor/preflight, latest-run, and stale-running findings.",
|
|
10329
|
+
readOnly: true,
|
|
10330
|
+
annotations: READ_ONLY_ANNOTATIONS,
|
|
10331
|
+
inputSchema: {
|
|
10332
|
+
includeStatuses: z2.array(z2.enum(LOOP_STATUSES)).optional().describe("Loop statuses to inventory (default active and paused)."),
|
|
10333
|
+
includeArchived: z2.boolean().optional().describe("Include archived loops in the scan (default false)."),
|
|
10334
|
+
latestRun: z2.boolean().optional().describe("Include latest-run and stale-running checks (default true)."),
|
|
10335
|
+
doctor: z2.boolean().optional().describe("Include doctor/preflight checks (default false)."),
|
|
10336
|
+
daemon: z2.boolean().optional().describe("Include daemon status checks (default false)."),
|
|
10337
|
+
staleRunningMs: z2.number().int().positive().optional().describe("Minimum age before a running latest run is stale; loop lease and 10m still apply."),
|
|
10338
|
+
maxFindings: z2.number().int().min(0).max(MAX_LIMIT).optional().describe(`Maximum findings to return (0-${MAX_LIMIT}, default 100).`),
|
|
10339
|
+
limit: limitSchema
|
|
10340
|
+
},
|
|
10341
|
+
handler: ({ includeStatuses, includeArchived, latestRun, doctor, daemon, staleRunningMs, maxFindings, limit }) => withLocalStore("loops_health_scan", (store) => buildHealthScan(store, {
|
|
10342
|
+
includeStatuses,
|
|
10343
|
+
includeArchived,
|
|
10344
|
+
latestRun,
|
|
10345
|
+
doctor: doctor ? runDoctor(store) : undefined,
|
|
10346
|
+
daemon: daemon ? daemonStatus(store) : undefined,
|
|
10347
|
+
staleRunningMs,
|
|
10348
|
+
maxFindings,
|
|
10349
|
+
limit
|
|
10350
|
+
}))
|
|
8133
10351
|
},
|
|
8134
10352
|
{
|
|
8135
10353
|
name: "loops_diagnose",
|
|
@@ -8142,7 +10360,7 @@ var TOOL_REGISTRATIONS = [
|
|
|
8142
10360
|
runLimit: z2.number().int().min(1).max(50).optional().describe("How many recent runs to classify (1-50, default 5)."),
|
|
8143
10361
|
showOutput: showOutputSchema
|
|
8144
10362
|
},
|
|
8145
|
-
handler: ({ idOrName, runLimit, showOutput }) =>
|
|
10363
|
+
handler: ({ idOrName, runLimit, showOutput }) => withLocalStore("loops_diagnose", (store) => {
|
|
8146
10364
|
const loop = store.requireLoop(idOrName);
|
|
8147
10365
|
const runs = store.listRuns({ loopId: loop.id, limit: runLimit ?? 5 });
|
|
8148
10366
|
return {
|
|
@@ -8162,7 +10380,7 @@ var TOOL_REGISTRATIONS = [
|
|
|
8162
10380
|
readOnly: true,
|
|
8163
10381
|
annotations: READ_ONLY_ANNOTATIONS,
|
|
8164
10382
|
inputSchema: {},
|
|
8165
|
-
handler: () =>
|
|
10383
|
+
handler: () => withLocalStore("loops_daemon_status", (store) => daemonStatus(store))
|
|
8166
10384
|
},
|
|
8167
10385
|
{
|
|
8168
10386
|
name: "loops_workflows_list",
|
|
@@ -8175,12 +10393,12 @@ var TOOL_REGISTRATIONS = [
|
|
|
8175
10393
|
limit: limitSchema,
|
|
8176
10394
|
offset: z2.number().int().min(0).optional().describe("Entries to skip before returning results (pagination).")
|
|
8177
10395
|
},
|
|
8178
|
-
handler: ({ status, limit, offset }) => withStore((store) => ({
|
|
8179
|
-
workflows: store.listWorkflows({
|
|
10396
|
+
handler: ({ status, limit, offset }) => withStore(async (store) => ({
|
|
10397
|
+
workflows: (await store.listWorkflows({
|
|
8180
10398
|
status: filteredWorkflowStatus(status),
|
|
8181
10399
|
limit,
|
|
8182
10400
|
offset
|
|
8183
|
-
}).map(publicWorkflow)
|
|
10401
|
+
})).map(publicWorkflow)
|
|
8184
10402
|
}))
|
|
8185
10403
|
},
|
|
8186
10404
|
{
|
|
@@ -8196,16 +10414,16 @@ var TOOL_REGISTRATIONS = [
|
|
|
8196
10414
|
runLimit: limitSchema,
|
|
8197
10415
|
showOutput: showOutputSchema
|
|
8198
10416
|
},
|
|
8199
|
-
handler: ({ idOrName, includeRuns, includeEvents, runLimit, showOutput }) => withStore((store) => {
|
|
8200
|
-
const workflow = store.requireWorkflow(idOrName);
|
|
8201
|
-
const runs = includeRuns ? store.listWorkflowRuns({ workflowId: workflow.id, limit: runLimit ?? 20 }) : [];
|
|
10417
|
+
handler: ({ idOrName, includeRuns, includeEvents, runLimit, showOutput }) => withStore(async (store) => {
|
|
10418
|
+
const workflow = await store.requireWorkflow(idOrName);
|
|
10419
|
+
const runs = includeRuns ? await store.listWorkflowRuns({ workflowId: workflow.id, limit: runLimit ?? 20 }) : [];
|
|
8202
10420
|
return {
|
|
8203
10421
|
workflow: publicWorkflow(workflow),
|
|
8204
|
-
runs: runs.map((run) => ({
|
|
10422
|
+
runs: await Promise.all(runs.map(async (run) => ({
|
|
8205
10423
|
...publicWorkflowRun(run),
|
|
8206
|
-
steps: store.listWorkflowStepRuns(run.id).map((step) => publicWorkflowStepRun(step, showOutput ?? false)),
|
|
8207
|
-
events: includeEvents ? store.listWorkflowEvents(run.id).map(publicWorkflowEvent) : undefined
|
|
8208
|
-
}))
|
|
10424
|
+
steps: (await store.listWorkflowStepRuns(run.id)).map((step) => publicWorkflowStepRun(step, showOutput ?? false)),
|
|
10425
|
+
events: includeEvents ? (await store.listWorkflowEvents(run.id)).map(publicWorkflowEvent) : undefined
|
|
10426
|
+
})))
|
|
8209
10427
|
};
|
|
8210
10428
|
})
|
|
8211
10429
|
},
|
|
@@ -8247,14 +10465,14 @@ var TOOL_REGISTRATIONS = [
|
|
|
8247
10465
|
includeEvents: z2.boolean().optional().describe("Include the workflow event log (default true)."),
|
|
8248
10466
|
showOutput: showOutputSchema
|
|
8249
10467
|
},
|
|
8250
|
-
handler: ({ runId, includeEvents, showOutput }) => withStore((store) => {
|
|
8251
|
-
const run = store.getWorkflowRun(runId);
|
|
10468
|
+
handler: ({ runId, includeEvents, showOutput }) => withStore(async (store) => {
|
|
10469
|
+
const run = await store.getWorkflowRun(runId);
|
|
8252
10470
|
if (!run)
|
|
8253
10471
|
throw new Error(`workflow run not found: ${runId}`);
|
|
8254
10472
|
return {
|
|
8255
10473
|
run: publicWorkflowRun(run),
|
|
8256
|
-
steps: store.listWorkflowStepRuns(run.id).map((step) => publicWorkflowStepRun(step, showOutput ?? false)),
|
|
8257
|
-
events: includeEvents ?? true ? store.listWorkflowEvents(run.id).map(publicWorkflowEvent) : undefined
|
|
10474
|
+
steps: (await store.listWorkflowStepRuns(run.id)).map((step) => publicWorkflowStepRun(step, showOutput ?? false)),
|
|
10475
|
+
events: includeEvents ?? true ? (await store.listWorkflowEvents(run.id)).map(publicWorkflowEvent) : undefined
|
|
8258
10476
|
};
|
|
8259
10477
|
})
|
|
8260
10478
|
},
|
|
@@ -8266,7 +10484,12 @@ var TOOL_REGISTRATIONS = [
|
|
|
8266
10484
|
guarded: true,
|
|
8267
10485
|
annotations: mutationAnnotations({ idempotent: true }),
|
|
8268
10486
|
inputSchema: { idOrName: loopIdOrNameSchema },
|
|
8269
|
-
handler: ({ idOrName }) => withStore((store) =>
|
|
10487
|
+
handler: ({ idOrName }) => withStore(async (store) => {
|
|
10488
|
+
const loop = await store.requireUniqueLoop(idOrName);
|
|
10489
|
+
if (loop.archivedAt)
|
|
10490
|
+
throw new LoopArchivedError(idOrName);
|
|
10491
|
+
return { loop: publicLoop(await store.updateLoop(loop.id, { status: "paused" })) };
|
|
10492
|
+
})
|
|
8270
10493
|
},
|
|
8271
10494
|
{
|
|
8272
10495
|
name: "loops_resume",
|
|
@@ -8276,14 +10499,16 @@ var TOOL_REGISTRATIONS = [
|
|
|
8276
10499
|
guarded: true,
|
|
8277
10500
|
annotations: mutationAnnotations({ idempotent: true }),
|
|
8278
10501
|
inputSchema: { idOrName: loopIdOrNameSchema },
|
|
8279
|
-
handler: ({ idOrName }) => withStore((store) => {
|
|
8280
|
-
const loop = store.requireUniqueLoop(idOrName);
|
|
10502
|
+
handler: ({ idOrName }) => withStore(async (store) => {
|
|
10503
|
+
const loop = await store.requireUniqueLoop(idOrName);
|
|
10504
|
+
if (loop.archivedAt)
|
|
10505
|
+
throw new LoopArchivedError(idOrName);
|
|
8281
10506
|
let nextRunAt = loop.nextRunAt;
|
|
8282
10507
|
if (!nextRunAt) {
|
|
8283
10508
|
const now = new Date;
|
|
8284
10509
|
nextRunAt = computeNextAfter(loop.schedule, now, now);
|
|
8285
10510
|
}
|
|
8286
|
-
return { loop: publicLoop(store.updateLoop(loop.id, { status: "active", nextRunAt })) };
|
|
10511
|
+
return { loop: publicLoop(await store.updateLoop(loop.id, { status: "active", nextRunAt })) };
|
|
8287
10512
|
})
|
|
8288
10513
|
},
|
|
8289
10514
|
{
|
|
@@ -8294,9 +10519,12 @@ var TOOL_REGISTRATIONS = [
|
|
|
8294
10519
|
guarded: true,
|
|
8295
10520
|
annotations: mutationAnnotations({ idempotent: true }),
|
|
8296
10521
|
inputSchema: { idOrName: loopIdOrNameSchema },
|
|
8297
|
-
handler: ({ idOrName }) => withStore((store) =>
|
|
8298
|
-
loop
|
|
8299
|
-
|
|
10522
|
+
handler: ({ idOrName }) => withStore(async (store) => {
|
|
10523
|
+
const loop = await store.requireUniqueLoop(idOrName);
|
|
10524
|
+
if (loop.archivedAt)
|
|
10525
|
+
throw new LoopArchivedError(idOrName);
|
|
10526
|
+
return { loop: publicLoop(await store.updateLoop(loop.id, { status: "stopped", nextRunAt: undefined })) };
|
|
10527
|
+
})
|
|
8300
10528
|
},
|
|
8301
10529
|
{
|
|
8302
10530
|
name: "loops_run_now",
|
|
@@ -8307,8 +10535,22 @@ var TOOL_REGISTRATIONS = [
|
|
|
8307
10535
|
annotations: mutationAnnotations(),
|
|
8308
10536
|
inputSchema: { idOrName: loopIdOrNameSchema },
|
|
8309
10537
|
handler: ({ idOrName }) => withStore(async (store) => {
|
|
8310
|
-
|
|
8311
|
-
|
|
10538
|
+
if (store.transport === "cloud-http") {
|
|
10539
|
+
const loop = await store.requireUniqueLoop(idOrName);
|
|
10540
|
+
if (loop.archivedAt)
|
|
10541
|
+
throw new LoopArchivedError(idOrName);
|
|
10542
|
+
const now = new Date().toISOString();
|
|
10543
|
+
const updated = await store.updateLoop(loop.id, { status: "active", nextRunAt: now });
|
|
10544
|
+
return {
|
|
10545
|
+
scheduledFor: now,
|
|
10546
|
+
loop: publicLoop(updated),
|
|
10547
|
+
daemon: undefined,
|
|
10548
|
+
warning: "loops is flipped to self_hosted: the loop is marked due on the hosted control plane; a self-hosted runner must execute it."
|
|
10549
|
+
};
|
|
10550
|
+
}
|
|
10551
|
+
const raw = store.raw;
|
|
10552
|
+
const daemon = daemonStatus(raw);
|
|
10553
|
+
const result = await runLoopNow({ store: raw, idOrName: raw.requireUniqueLoop(idOrName).id, runnerId: `mcp:${process.pid}`, mode: "schedule" });
|
|
8312
10554
|
return {
|
|
8313
10555
|
scheduledFor: result.scheduledFor,
|
|
8314
10556
|
loop: publicLoop(result.loop),
|
|
@@ -8324,7 +10566,7 @@ var TOOL_REGISTRATIONS = [
|
|
|
8324
10566
|
guarded: true,
|
|
8325
10567
|
annotations: mutationAnnotations({ idempotent: true }),
|
|
8326
10568
|
inputSchema: { idOrName: loopIdOrNameSchema },
|
|
8327
|
-
handler: ({ idOrName }) => withStore((store) => ({ loop: publicLoop(store.archiveLoop(idOrName)) }))
|
|
10569
|
+
handler: ({ idOrName }) => withStore(async (store) => ({ loop: publicLoop(await store.archiveLoop(idOrName)) }))
|
|
8328
10570
|
},
|
|
8329
10571
|
{
|
|
8330
10572
|
name: "loops_unarchive",
|
|
@@ -8333,7 +10575,7 @@ var TOOL_REGISTRATIONS = [
|
|
|
8333
10575
|
guarded: true,
|
|
8334
10576
|
annotations: mutationAnnotations({ idempotent: true }),
|
|
8335
10577
|
inputSchema: { idOrName: loopIdOrNameSchema },
|
|
8336
|
-
handler: ({ idOrName }) => withStore((store) => ({ loop: publicLoop(store.unarchiveLoop(idOrName)) }))
|
|
10578
|
+
handler: ({ idOrName }) => withStore(async (store) => ({ loop: publicLoop(await store.unarchiveLoop(idOrName)) }))
|
|
8337
10579
|
},
|
|
8338
10580
|
{
|
|
8339
10581
|
name: "loops_create_command",
|
|
@@ -8349,21 +10591,21 @@ var TOOL_REGISTRATIONS = [
|
|
|
8349
10591
|
cwd: z2.string().optional().describe("Working directory for the command."),
|
|
8350
10592
|
shell: z2.never().optional().describe("Not supported over MCP: shell execution is rejected. Use 'command' plus 'args' instead.")
|
|
8351
10593
|
},
|
|
8352
|
-
handler: (input) =>
|
|
10594
|
+
handler: (input) => {
|
|
8353
10595
|
const timeoutMs = input.timeoutMs;
|
|
8354
|
-
const
|
|
10596
|
+
const createInput = commonCreateInput({
|
|
8355
10597
|
...input,
|
|
8356
10598
|
target: {
|
|
8357
10599
|
type: "command",
|
|
8358
|
-
command:
|
|
10600
|
+
command: nonEmpty2(input.command, "command"),
|
|
8359
10601
|
args: input.args,
|
|
8360
10602
|
cwd: input.cwd,
|
|
8361
10603
|
shell: false,
|
|
8362
10604
|
timeoutMs
|
|
8363
10605
|
}
|
|
8364
|
-
})
|
|
8365
|
-
return { loop: publicLoop(
|
|
8366
|
-
}
|
|
10606
|
+
});
|
|
10607
|
+
return withStore(async (store) => ({ loop: publicLoop(await store.createLoop(createInput)) }));
|
|
10608
|
+
}
|
|
8367
10609
|
},
|
|
8368
10610
|
{
|
|
8369
10611
|
name: "loops_create_workflow",
|
|
@@ -8377,20 +10619,17 @@ var TOOL_REGISTRATIONS = [
|
|
|
8377
10619
|
workflow: workflowIdOrNameSchema,
|
|
8378
10620
|
workflowInput: z2.record(z2.string(), z2.string()).optional().describe("String key/value input passed to the workflow on each run.")
|
|
8379
10621
|
},
|
|
8380
|
-
handler: (input) =>
|
|
8381
|
-
const workflow = store.requireWorkflow(input.workflow);
|
|
10622
|
+
handler: (input) => {
|
|
8382
10623
|
const timeoutMs = input.timeoutMs;
|
|
8383
|
-
|
|
8384
|
-
|
|
8385
|
-
|
|
8386
|
-
|
|
8387
|
-
workflowId:
|
|
8388
|
-
|
|
8389
|
-
|
|
8390
|
-
|
|
8391
|
-
|
|
8392
|
-
return { workflow: publicWorkflow(workflow), loop: publicLoop(loop) };
|
|
8393
|
-
})
|
|
10624
|
+
return withStore(async (store) => {
|
|
10625
|
+
const wf = await store.requireWorkflow(input.workflow);
|
|
10626
|
+
const createInput = commonCreateInput({
|
|
10627
|
+
...input,
|
|
10628
|
+
target: { type: "workflow", workflowId: wf.id, input: input.workflowInput, timeoutMs }
|
|
10629
|
+
});
|
|
10630
|
+
return { workflow: publicWorkflow(wf), loop: publicLoop(await store.createLoop(createInput)) };
|
|
10631
|
+
});
|
|
10632
|
+
}
|
|
8394
10633
|
}
|
|
8395
10634
|
];
|
|
8396
10635
|
function toolDescription(tool) {
|
|
@@ -8467,8 +10706,20 @@ async function main() {
|
|
|
8467
10706
|
console.log(JSON.stringify(listToolsForCli(), null, 2));
|
|
8468
10707
|
return;
|
|
8469
10708
|
}
|
|
8470
|
-
|
|
8471
|
-
|
|
10709
|
+
if (isStdioMode()) {
|
|
10710
|
+
const server = createLoopsMcpServer();
|
|
10711
|
+
await server.connect(new StdioServerTransport);
|
|
10712
|
+
return;
|
|
10713
|
+
}
|
|
10714
|
+
const handle = await startMcpHttpServer(() => createLoopsMcpServer(), {
|
|
10715
|
+
port: resolveMcpHttpPort()
|
|
10716
|
+
});
|
|
10717
|
+
process.on("SIGINT", () => {
|
|
10718
|
+
handle.close().finally(() => process.exit(0));
|
|
10719
|
+
});
|
|
10720
|
+
process.on("SIGTERM", () => {
|
|
10721
|
+
handle.close().finally(() => process.exit(0));
|
|
10722
|
+
});
|
|
8472
10723
|
}
|
|
8473
10724
|
if (import.meta.main) {
|
|
8474
10725
|
main().catch((error) => {
|