@hasna/loops 0.4.13 → 0.4.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +99 -3
- package/README.md +171 -39
- package/dist/api/index.d.ts +36 -0
- package/dist/api/index.js +1518 -15
- package/dist/cli/index.js +7280 -3213
- package/dist/cli/ui.d.ts +44 -0
- package/dist/daemon/index.js +1433 -303
- package/dist/generated/storage-kit/health.d.ts +19 -0
- package/dist/generated/storage-kit/index.d.ts +7 -0
- package/dist/generated/storage-kit/migrations.d.ts +47 -0
- package/dist/generated/storage-kit/mode.d.ts +47 -0
- package/dist/generated/storage-kit/pool.d.ts +33 -0
- package/dist/generated/storage-kit/query.d.ts +35 -0
- package/dist/generated/storage-kit/tls.d.ts +25 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +8689 -4837
- package/dist/lib/cloud/mode.d.ts +17 -0
- package/dist/lib/cloud/resolve.d.ts +16 -0
- package/dist/lib/cloud/storage.d.ts +82 -0
- package/dist/lib/cloud/transport.d.ts +148 -0
- package/dist/lib/format.d.ts +2 -1
- package/dist/lib/health.d.ts +83 -3
- package/dist/lib/migration.d.ts +40 -0
- package/dist/lib/mode.js +15 -3
- package/dist/lib/route/index.d.ts +2 -0
- package/dist/lib/route/policies.d.ts +51 -0
- package/dist/lib/route/provider-admission.d.ts +37 -0
- package/dist/lib/route/throttle.d.ts +4 -0
- package/dist/lib/route/types.d.ts +8 -0
- package/dist/lib/run-receipts.d.ts +14 -0
- package/dist/lib/scheduler.d.ts +5 -2
- package/dist/lib/storage/contract.d.ts +7 -1
- package/dist/lib/storage/index.d.ts +1 -0
- package/dist/lib/storage/index.js +2028 -231
- package/dist/lib/storage/pg-executor.d.ts +27 -0
- package/dist/lib/storage/pg-runner-claim.d.ts +40 -0
- package/dist/lib/storage/postgres-loop-storage.d.ts +120 -0
- package/dist/lib/storage/postgres-schema.js +29 -0
- package/dist/lib/storage/postgres.js +29 -0
- package/dist/lib/storage/sqlite.d.ts +6 -0
- package/dist/lib/storage/sqlite.js +748 -26
- package/dist/lib/store/index.d.ts +268 -0
- package/dist/lib/store.d.ts +282 -1
- package/dist/lib/store.js +746 -26
- package/dist/lib/template-kit.d.ts +25 -0
- package/dist/lib/templates.d.ts +16 -1
- package/dist/mcp/http.d.ts +16 -0
- package/dist/mcp/index.js +2885 -634
- package/dist/runner/index.js +198 -32
- package/dist/sdk/http.d.ts +182 -0
- package/dist/sdk/http.js +166 -0
- package/dist/sdk/index.d.ts +55 -18
- package/dist/sdk/index.js +2734 -617
- package/dist/serve/index.d.ts +4 -0
- package/dist/serve/index.js +9095 -0
- package/dist/types.d.ts +52 -0
- package/docs/AUTOMATION_RUNTIME_DESIGN.md +442 -1
- package/docs/CUTOVER-RUNBOOK.md +78 -0
- package/docs/DEPLOYMENT_MODES.md +35 -18
- package/docs/RUNTIME_BOUNDARY.md +203 -0
- package/docs/USAGE.md +195 -38
- package/package.json +15 -3
package/dist/sdk/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/lib/store.ts
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as
|
|
5
|
-
import { tmpdir } from "os";
|
|
6
|
-
import { basename as basename2, dirname as dirname2, join as
|
|
4
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync as mkdtempSync2, rmSync as rmSync3 } from "fs";
|
|
5
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
6
|
+
import { basename as basename2, dirname as dirname2, join as join4 } from "path";
|
|
7
7
|
|
|
8
8
|
// src/lib/errors.ts
|
|
9
9
|
class CodedError extends Error {
|
|
@@ -1202,15 +1202,378 @@ function discardWorkflowRunManifest(staged) {
|
|
|
1202
1202
|
} catch {}
|
|
1203
1203
|
}
|
|
1204
1204
|
|
|
1205
|
+
// src/lib/run-receipts.ts
|
|
1206
|
+
import { createHash as createHash2 } from "crypto";
|
|
1207
|
+
import { hostname } from "os";
|
|
1208
|
+
|
|
1209
|
+
// src/lib/run-envelope.ts
|
|
1210
|
+
var ENVELOPE_EXCERPT_CHARS = 2048;
|
|
1211
|
+
var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
|
|
1212
|
+
function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
|
|
1213
|
+
if (!text)
|
|
1214
|
+
return;
|
|
1215
|
+
const scrubbed = scrubSecrets(text);
|
|
1216
|
+
if (scrubbed.length <= limit * 2)
|
|
1217
|
+
return scrubbed;
|
|
1218
|
+
const omitted = scrubbed.length - limit * 2;
|
|
1219
|
+
return `${scrubbed.slice(0, limit)}
|
|
1220
|
+
[... ${omitted} chars omitted ...]
|
|
1221
|
+
${scrubbed.slice(-limit)}`;
|
|
1222
|
+
}
|
|
1223
|
+
function summarizeOutput(stdout, stderr) {
|
|
1224
|
+
return {
|
|
1225
|
+
stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
|
|
1226
|
+
stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
|
|
1227
|
+
stdoutExcerpt: boundedExcerpt(stdout),
|
|
1228
|
+
stderrExcerpt: boundedExcerpt(stderr)
|
|
1229
|
+
};
|
|
1230
|
+
}
|
|
1231
|
+
function summarizeExecutorResult(result) {
|
|
1232
|
+
return {
|
|
1233
|
+
...summarizeOutput(result.stdout, result.stderr),
|
|
1234
|
+
status: result.status,
|
|
1235
|
+
exitCode: result.exitCode,
|
|
1236
|
+
durationMs: result.durationMs,
|
|
1237
|
+
error: boundedExcerpt(result.error)
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
function isBlockedStepRun(step) {
|
|
1241
|
+
return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
|
|
1242
|
+
}
|
|
1243
|
+
function summarizeWorkflowStepRun(step) {
|
|
1244
|
+
return {
|
|
1245
|
+
...summarizeOutput(step.stdout, step.stderr),
|
|
1246
|
+
stepId: step.stepId,
|
|
1247
|
+
status: step.status,
|
|
1248
|
+
exitCode: step.exitCode,
|
|
1249
|
+
durationMs: step.durationMs,
|
|
1250
|
+
error: boundedExcerpt(step.error),
|
|
1251
|
+
blocked: isBlockedStepRun(step)
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
function workflowRunEnvelope(run, steps) {
|
|
1255
|
+
return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
// src/lib/run-receipts.ts
|
|
1259
|
+
var RUN_RECEIPT_SUMMARY_TEXT_CHARS = 4096;
|
|
1260
|
+
var RUN_RECEIPT_MAX_IDS = 100;
|
|
1261
|
+
var RUN_RECEIPT_MAX_EVIDENCE_PATHS = 100;
|
|
1262
|
+
var RUN_RECEIPT_MAX_PATH_CHARS = 1024;
|
|
1263
|
+
function canonicalJson(value) {
|
|
1264
|
+
if (Array.isArray(value))
|
|
1265
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
1266
|
+
if (value && typeof value === "object") {
|
|
1267
|
+
return `{${Object.entries(value).filter(([, entry]) => entry !== undefined).sort(([a], [b]) => a.localeCompare(b)).map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
|
|
1268
|
+
}
|
|
1269
|
+
return JSON.stringify(value);
|
|
1270
|
+
}
|
|
1271
|
+
function digestReceipt(receipt) {
|
|
1272
|
+
return `sha256:${createHash2("sha256").update(canonicalJson(receipt)).digest("hex")}`;
|
|
1273
|
+
}
|
|
1274
|
+
function isoOrNull(value, label) {
|
|
1275
|
+
if (value === undefined || value === null || value === "")
|
|
1276
|
+
return null;
|
|
1277
|
+
const date = new Date(value);
|
|
1278
|
+
if (Number.isNaN(date.getTime()))
|
|
1279
|
+
throw new Error(`${label} must be a valid ISO date/time`);
|
|
1280
|
+
return date.toISOString();
|
|
1281
|
+
}
|
|
1282
|
+
function nonEmpty(value, label) {
|
|
1283
|
+
const trimmed = value?.trim();
|
|
1284
|
+
if (!trimmed)
|
|
1285
|
+
throw new Error(`${label} must be non-empty`);
|
|
1286
|
+
return trimmed;
|
|
1287
|
+
}
|
|
1288
|
+
function normalizedStringArray(values, opts) {
|
|
1289
|
+
const out = [];
|
|
1290
|
+
const seen = new Set;
|
|
1291
|
+
for (const raw of values ?? []) {
|
|
1292
|
+
const value = String(raw).trim();
|
|
1293
|
+
if (!value || seen.has(value))
|
|
1294
|
+
continue;
|
|
1295
|
+
seen.add(value);
|
|
1296
|
+
out.push(opts.itemMax ? value.slice(0, opts.itemMax) : value);
|
|
1297
|
+
if (out.length >= opts.max)
|
|
1298
|
+
break;
|
|
1299
|
+
}
|
|
1300
|
+
if ((values?.length ?? 0) > opts.max) {
|
|
1301
|
+
out.push(`[truncated ${values.length - opts.max} ${opts.label}]`);
|
|
1302
|
+
}
|
|
1303
|
+
return out;
|
|
1304
|
+
}
|
|
1305
|
+
function receiptMachine(input, opts) {
|
|
1306
|
+
if (typeof input.machine === "string")
|
|
1307
|
+
return nonEmpty(input.machine, "machine");
|
|
1308
|
+
if (input.machine && typeof input.machine === "object")
|
|
1309
|
+
return input.machine;
|
|
1310
|
+
if (opts.loop?.machine)
|
|
1311
|
+
return { ...opts.loop.machine };
|
|
1312
|
+
if (typeof opts.defaultMachine === "string")
|
|
1313
|
+
return nonEmpty(opts.defaultMachine, "machine");
|
|
1314
|
+
if (opts.defaultMachine && typeof opts.defaultMachine === "object")
|
|
1315
|
+
return opts.defaultMachine;
|
|
1316
|
+
return hostname();
|
|
1317
|
+
}
|
|
1318
|
+
function normalizeSummary(input, run) {
|
|
1319
|
+
const stdout = input.stdout ?? run?.stdout;
|
|
1320
|
+
const stderr = input.stderr ?? run?.stderr;
|
|
1321
|
+
const output = summarizeOutput(stdout, stderr);
|
|
1322
|
+
const summaryInput = input.summary;
|
|
1323
|
+
const provided = typeof summaryInput === "object" && summaryInput !== null ? summaryInput : {};
|
|
1324
|
+
const text = typeof summaryInput === "string" ? boundedExcerpt(summaryInput, Math.floor(RUN_RECEIPT_SUMMARY_TEXT_CHARS / 2)) : typeof provided.text === "string" ? boundedExcerpt(provided.text, Math.floor(RUN_RECEIPT_SUMMARY_TEXT_CHARS / 2)) : undefined;
|
|
1325
|
+
return {
|
|
1326
|
+
text,
|
|
1327
|
+
stdout_bytes: Number.isFinite(provided.stdout_bytes) ? Number(provided.stdout_bytes) : output.stdoutBytes,
|
|
1328
|
+
stderr_bytes: Number.isFinite(provided.stderr_bytes) ? Number(provided.stderr_bytes) : output.stderrBytes,
|
|
1329
|
+
stdout_excerpt: typeof provided.stdout_excerpt === "string" ? boundedExcerpt(provided.stdout_excerpt) : output.stdoutExcerpt,
|
|
1330
|
+
stderr_excerpt: typeof provided.stderr_excerpt === "string" ? boundedExcerpt(provided.stderr_excerpt) : output.stderrExcerpt,
|
|
1331
|
+
error: typeof provided.error === "string" ? boundedExcerpt(provided.error) : boundedExcerpt(input.error ?? run?.error),
|
|
1332
|
+
duration_ms: Number.isFinite(provided.duration_ms) ? Number(provided.duration_ms) : input.duration_ms ?? run?.durationMs
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
function normalizeRunReceipt(input, opts = {}) {
|
|
1336
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
1337
|
+
const run = opts.run;
|
|
1338
|
+
const loopId = input.loop_id ?? run?.loopId ?? opts.loop?.id;
|
|
1339
|
+
const normalized = {
|
|
1340
|
+
loop_id: nonEmpty(loopId, "loop_id"),
|
|
1341
|
+
run_id: nonEmpty(input.run_id ?? run?.id, "run_id"),
|
|
1342
|
+
machine: receiptMachine(input, opts),
|
|
1343
|
+
repo: nonEmpty(input.repo ?? opts.defaultRepo ?? targetRepo(opts.loop) ?? process.cwd(), "repo"),
|
|
1344
|
+
task_ids: normalizedStringArray(input.task_ids, { max: RUN_RECEIPT_MAX_IDS, label: "task_ids" }),
|
|
1345
|
+
knowledge_ids: normalizedStringArray(input.knowledge_ids, { max: RUN_RECEIPT_MAX_IDS, label: "knowledge_ids" }),
|
|
1346
|
+
started_at: isoOrNull(input.started_at ?? run?.startedAt, "started_at"),
|
|
1347
|
+
finished_at: isoOrNull(input.finished_at ?? run?.finishedAt, "finished_at"),
|
|
1348
|
+
status: nonEmpty(input.status ?? run?.status, "status"),
|
|
1349
|
+
exit_code: input.exit_code === undefined ? run?.exitCode ?? null : input.exit_code,
|
|
1350
|
+
summary: normalizeSummary(input, run),
|
|
1351
|
+
evidence_paths: normalizedStringArray(input.evidence_paths, {
|
|
1352
|
+
max: RUN_RECEIPT_MAX_EVIDENCE_PATHS,
|
|
1353
|
+
label: "evidence_paths",
|
|
1354
|
+
itemMax: RUN_RECEIPT_MAX_PATH_CHARS
|
|
1355
|
+
})
|
|
1356
|
+
};
|
|
1357
|
+
return {
|
|
1358
|
+
...normalized,
|
|
1359
|
+
digest_id: input.digest_id?.trim() || digestReceipt(normalized),
|
|
1360
|
+
created_at: opts.existing?.created_at ?? now,
|
|
1361
|
+
updated_at: now
|
|
1362
|
+
};
|
|
1363
|
+
}
|
|
1364
|
+
function targetRepo(loop) {
|
|
1365
|
+
if (!loop)
|
|
1366
|
+
return;
|
|
1367
|
+
if (loop.target.type === "command")
|
|
1368
|
+
return loop.target.cwd;
|
|
1369
|
+
if (loop.target.type === "agent")
|
|
1370
|
+
return loop.target.cwd;
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
// src/lib/route/todos-cli.ts
|
|
1375
|
+
import { closeSync, mkdtempSync, openSync, readFileSync as readFileSync3, rmSync as rmSync2 } from "fs";
|
|
1376
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
1377
|
+
import { join as join3 } from "path";
|
|
1378
|
+
import { homedir as homedir2, tmpdir } from "os";
|
|
1379
|
+
|
|
1380
|
+
// src/lib/format.ts
|
|
1381
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
1382
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
1383
|
+
function redact(value, visible = 0) {
|
|
1384
|
+
if (!value)
|
|
1385
|
+
return value;
|
|
1386
|
+
if (value.length <= visible)
|
|
1387
|
+
return value;
|
|
1388
|
+
if (visible <= 0)
|
|
1389
|
+
return `[redacted ${value.length} chars]`;
|
|
1390
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
1391
|
+
}
|
|
1392
|
+
function truncateTextOutput(value) {
|
|
1393
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
1394
|
+
return value;
|
|
1395
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
1396
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
1397
|
+
}
|
|
1398
|
+
function redactSensitivePayload(value, key) {
|
|
1399
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
1400
|
+
if (typeof value === "string")
|
|
1401
|
+
return redact(value);
|
|
1402
|
+
if (value === undefined || value === null)
|
|
1403
|
+
return value;
|
|
1404
|
+
return "[redacted]";
|
|
1405
|
+
}
|
|
1406
|
+
if (Array.isArray(value))
|
|
1407
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
1408
|
+
if (value && typeof value === "object") {
|
|
1409
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
1410
|
+
}
|
|
1411
|
+
return value;
|
|
1412
|
+
}
|
|
1413
|
+
function textOutputBlocks(value, opts = {}) {
|
|
1414
|
+
const indent = opts.indent ?? "";
|
|
1415
|
+
const nested = `${indent} `;
|
|
1416
|
+
const blocks = [];
|
|
1417
|
+
for (const [label, output] of [
|
|
1418
|
+
["stdout", value.stdout],
|
|
1419
|
+
["stderr", value.stderr]
|
|
1420
|
+
]) {
|
|
1421
|
+
if (!output)
|
|
1422
|
+
continue;
|
|
1423
|
+
blocks.push(`${indent}${label}:`);
|
|
1424
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
1425
|
+
blocks.push(`${nested}${line}`);
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
return blocks;
|
|
1429
|
+
}
|
|
1430
|
+
function publicLoop(loop) {
|
|
1431
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
1432
|
+
return {
|
|
1433
|
+
...loop,
|
|
1434
|
+
target
|
|
1435
|
+
};
|
|
1436
|
+
}
|
|
1437
|
+
function publicRun(run, showOutput = false, opts = {}) {
|
|
1438
|
+
return {
|
|
1439
|
+
...run,
|
|
1440
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1441
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1442
|
+
error: opts.redactError ? redact(run.error) : run.error
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
function publicRunReceipt(receipt) {
|
|
1446
|
+
return { ...receipt };
|
|
1447
|
+
}
|
|
1448
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
1449
|
+
return {
|
|
1450
|
+
...result,
|
|
1451
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
1452
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
1453
|
+
error: redact(result.error)
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
function publicWorkflow(workflow) {
|
|
1457
|
+
return {
|
|
1458
|
+
...workflow,
|
|
1459
|
+
steps: workflow.steps.map((step) => ({
|
|
1460
|
+
...step,
|
|
1461
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
1462
|
+
}))
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
function publicWorkflowRun(run) {
|
|
1466
|
+
return { ...run, error: redact(run.error) };
|
|
1467
|
+
}
|
|
1468
|
+
function publicWorkflowInvocation(invocation) {
|
|
1469
|
+
return redactSensitivePayload(invocation);
|
|
1470
|
+
}
|
|
1471
|
+
function publicWorkflowWorkItem(item) {
|
|
1472
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
1473
|
+
}
|
|
1474
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
1475
|
+
return {
|
|
1476
|
+
...run,
|
|
1477
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
1478
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
1479
|
+
error: redact(run.error)
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1482
|
+
function publicWorkflowEvent(event) {
|
|
1483
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
1484
|
+
}
|
|
1485
|
+
function publicGoal(goal) {
|
|
1486
|
+
return {
|
|
1487
|
+
...goal,
|
|
1488
|
+
objective: redact(goal.objective, 120)
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
function publicGoalRun(run) {
|
|
1492
|
+
return {
|
|
1493
|
+
...run,
|
|
1494
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
1495
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
1496
|
+
};
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
// src/lib/route/todos-cli.ts
|
|
1500
|
+
function defaultLoopsProject() {
|
|
1501
|
+
return process.env.LOOPS_TASK_PROJECT || process.env.LOOPS_DATA_DIR || `${process.env.HOME || homedir2()}/.hasna/loops`;
|
|
1502
|
+
}
|
|
1503
|
+
function runLocalCommand(command, args, opts = {}) {
|
|
1504
|
+
const result = spawnSync2(command, args, {
|
|
1505
|
+
input: opts.input,
|
|
1506
|
+
encoding: "utf8",
|
|
1507
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1508
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1509
|
+
env: process.env
|
|
1510
|
+
});
|
|
1511
|
+
return {
|
|
1512
|
+
ok: result.status === 0,
|
|
1513
|
+
status: result.status,
|
|
1514
|
+
stdout: result.stdout || "",
|
|
1515
|
+
stderr: result.stderr || "",
|
|
1516
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1517
|
+
};
|
|
1518
|
+
}
|
|
1519
|
+
function runLocalCommandWithStdoutFile(command, args, opts = {}) {
|
|
1520
|
+
const tempDir = mkdtempSync(join3(tmpdir(), "loops-command-output-"));
|
|
1521
|
+
const stdoutPath = join3(tempDir, "stdout");
|
|
1522
|
+
const stdoutFd = openSync(stdoutPath, "w");
|
|
1523
|
+
let result;
|
|
1524
|
+
try {
|
|
1525
|
+
result = spawnSync2(command, args, {
|
|
1526
|
+
input: opts.input,
|
|
1527
|
+
encoding: "utf8",
|
|
1528
|
+
timeout: opts.timeoutMs ?? 30000,
|
|
1529
|
+
maxBuffer: opts.maxBuffer ?? 8 * 1024 * 1024,
|
|
1530
|
+
env: process.env,
|
|
1531
|
+
stdio: ["pipe", stdoutFd, "pipe"]
|
|
1532
|
+
});
|
|
1533
|
+
} finally {
|
|
1534
|
+
closeSync(stdoutFd);
|
|
1535
|
+
}
|
|
1536
|
+
try {
|
|
1537
|
+
return {
|
|
1538
|
+
ok: result.status === 0,
|
|
1539
|
+
status: result.status,
|
|
1540
|
+
stdout: readFileSync3(stdoutPath, "utf8"),
|
|
1541
|
+
stderr: typeof result.stderr === "string" ? result.stderr : result.stderr?.toString() || "",
|
|
1542
|
+
error: result.error ? String(result.error.message || result.error) : ""
|
|
1543
|
+
};
|
|
1544
|
+
} finally {
|
|
1545
|
+
rmSync2(tempDir, { recursive: true, force: true });
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
function ensureTodosTaskList(project, slug, name, description) {
|
|
1549
|
+
runLocalCommand("todos", ["--project", project, "task-lists", "--add", name, "--slug", slug, "-d", description]);
|
|
1550
|
+
const list = runLocalCommand("todos", ["--project", project, "--json", "task-lists"]);
|
|
1551
|
+
if (!list.ok)
|
|
1552
|
+
throw new Error(list.stderr || list.error || "failed to list todos task lists");
|
|
1553
|
+
const values = JSON.parse(list.stdout || "[]");
|
|
1554
|
+
const found = values.find((entry) => entry.slug === slug);
|
|
1555
|
+
if (!found)
|
|
1556
|
+
throw new Error(`todos task list not found after ensure: ${slug}`);
|
|
1557
|
+
return found.id;
|
|
1558
|
+
}
|
|
1559
|
+
function todosMutationSummary(result) {
|
|
1560
|
+
return {
|
|
1561
|
+
ok: result.ok,
|
|
1562
|
+
status: result.status,
|
|
1563
|
+
error: result.ok ? undefined : redact(result.stderr || result.error || "todos mutation failed", 320)
|
|
1564
|
+
};
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1205
1567
|
// src/lib/store.ts
|
|
1206
1568
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
1207
1569
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
1208
1570
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1209
|
-
var SCHEMA_USER_VERSION =
|
|
1571
|
+
var SCHEMA_USER_VERSION = 8;
|
|
1210
1572
|
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1211
1573
|
var PRUNE_BATCH_SIZE = 400;
|
|
1212
1574
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
1213
1575
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
1576
|
+
var TASK_LIFECYCLE_TEMPLATE_ID = "task-lifecycle";
|
|
1214
1577
|
function rowToLoop(row) {
|
|
1215
1578
|
return {
|
|
1216
1579
|
id: row.id,
|
|
@@ -1261,6 +1624,28 @@ function rowToRun(row) {
|
|
|
1261
1624
|
updatedAt: row.updated_at
|
|
1262
1625
|
};
|
|
1263
1626
|
}
|
|
1627
|
+
function rowToRunReceipt(row) {
|
|
1628
|
+
return {
|
|
1629
|
+
loop_id: row.loop_id,
|
|
1630
|
+
run_id: row.run_id,
|
|
1631
|
+
machine: JSON.parse(row.machine_json),
|
|
1632
|
+
repo: row.repo,
|
|
1633
|
+
task_ids: JSON.parse(row.task_ids_json),
|
|
1634
|
+
knowledge_ids: JSON.parse(row.knowledge_ids_json),
|
|
1635
|
+
digest_id: row.digest_id,
|
|
1636
|
+
started_at: row.started_at,
|
|
1637
|
+
finished_at: row.finished_at,
|
|
1638
|
+
status: row.status,
|
|
1639
|
+
exit_code: row.exit_code,
|
|
1640
|
+
summary: JSON.parse(row.summary_json),
|
|
1641
|
+
evidence_paths: JSON.parse(row.evidence_paths_json),
|
|
1642
|
+
created_at: row.created_at,
|
|
1643
|
+
updated_at: row.updated_at
|
|
1644
|
+
};
|
|
1645
|
+
}
|
|
1646
|
+
function latestRunTime(row) {
|
|
1647
|
+
return row.finished_at ?? row.started_at ?? row.created_at;
|
|
1648
|
+
}
|
|
1264
1649
|
function rowToWorkflow(row) {
|
|
1265
1650
|
return {
|
|
1266
1651
|
id: row.id,
|
|
@@ -1321,6 +1706,7 @@ function rowToWorkflowWorkItem(row) {
|
|
|
1321
1706
|
subjectRef: row.subject_ref,
|
|
1322
1707
|
projectKey: row.project_key ?? undefined,
|
|
1323
1708
|
projectGroup: row.project_group ?? undefined,
|
|
1709
|
+
machineId: row.machine_id ?? undefined,
|
|
1324
1710
|
routeScope: row.route_scope ?? undefined,
|
|
1325
1711
|
priority: row.priority,
|
|
1326
1712
|
status: row.status,
|
|
@@ -1478,6 +1864,7 @@ function scrubbedOrNull(value) {
|
|
|
1478
1864
|
return value == null ? null : scrubSecrets(value);
|
|
1479
1865
|
}
|
|
1480
1866
|
var MAX_PERSISTED_RUN_OUTPUT_CHARS = 64 * 1024;
|
|
1867
|
+
var MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS = 64 * 1024;
|
|
1481
1868
|
function clampPersistedRunOutput(value) {
|
|
1482
1869
|
if (value == null || value.length <= MAX_PERSISTED_RUN_OUTPUT_CHARS)
|
|
1483
1870
|
return value;
|
|
@@ -1492,6 +1879,42 @@ ${tail}`;
|
|
|
1492
1879
|
function persistedRunOutput(value) {
|
|
1493
1880
|
return clampPersistedRunOutput(scrubbedOrNull(value));
|
|
1494
1881
|
}
|
|
1882
|
+
function clampTextToChars(value, maxChars, reason) {
|
|
1883
|
+
if (value.length <= maxChars)
|
|
1884
|
+
return value;
|
|
1885
|
+
const marker = `
|
|
1886
|
+
...[truncated by ${reason}]...
|
|
1887
|
+
`;
|
|
1888
|
+
const budget = Math.max(0, maxChars - marker.length);
|
|
1889
|
+
const headLength = Math.ceil(budget / 2);
|
|
1890
|
+
const tailLength = Math.floor(budget / 2);
|
|
1891
|
+
return `${value.slice(0, headLength)}${marker}${value.slice(value.length - tailLength)}`;
|
|
1892
|
+
}
|
|
1893
|
+
function boundedWorkflowEventPayloadJson(scrubbedJson) {
|
|
1894
|
+
if (scrubbedJson.length <= MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS)
|
|
1895
|
+
return scrubbedJson;
|
|
1896
|
+
const base = {
|
|
1897
|
+
truncated: true,
|
|
1898
|
+
originalChars: scrubbedJson.length,
|
|
1899
|
+
maxChars: MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS,
|
|
1900
|
+
preview: ""
|
|
1901
|
+
};
|
|
1902
|
+
const baseChars = JSON.stringify(base).length;
|
|
1903
|
+
let previewBudget = Math.max(0, MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS - baseChars - 64);
|
|
1904
|
+
while (true) {
|
|
1905
|
+
const preview = clampTextToChars(scrubbedJson, previewBudget, "loops workflow-event payload retention");
|
|
1906
|
+
const bounded = JSON.stringify({ ...base, preview });
|
|
1907
|
+
if (bounded.length <= MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS || previewBudget === 0)
|
|
1908
|
+
return bounded;
|
|
1909
|
+
previewBudget = Math.max(0, previewBudget - (bounded.length - MAX_PERSISTED_WORKFLOW_EVENT_PAYLOAD_CHARS) - 64);
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
function persistedWorkflowEventPayload(payload) {
|
|
1913
|
+
if (payload == null)
|
|
1914
|
+
return null;
|
|
1915
|
+
const scrubbed = scrubSecretsDeep(payload);
|
|
1916
|
+
return boundedWorkflowEventPayloadJson(scrubSecrets(JSON.stringify(scrubbed)));
|
|
1917
|
+
}
|
|
1495
1918
|
function chmodIfExists(path, mode) {
|
|
1496
1919
|
try {
|
|
1497
1920
|
if (existsSync(path))
|
|
@@ -1515,7 +1938,7 @@ class Store {
|
|
|
1515
1938
|
const file = path ?? dbPath();
|
|
1516
1939
|
if (file !== ":memory:")
|
|
1517
1940
|
ensurePrivateStorePath(file);
|
|
1518
|
-
this.rootDir = file === ":memory:" ?
|
|
1941
|
+
this.rootDir = file === ":memory:" ? mkdtempSync2(join4(tmpdir2(), "open-loops-store-")) : dirname2(file);
|
|
1519
1942
|
if (file === ":memory:")
|
|
1520
1943
|
this.memoryRootDir = this.rootDir;
|
|
1521
1944
|
this.db = new Database(file);
|
|
@@ -1609,6 +2032,17 @@ class Store {
|
|
|
1609
2032
|
this.addColumnIfMissing("workflow_work_items", "route_scope", "TEXT");
|
|
1610
2033
|
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_scope ON workflow_work_items(route_scope, status)");
|
|
1611
2034
|
}
|
|
2035
|
+
},
|
|
2036
|
+
{
|
|
2037
|
+
id: "0009_run_receipts",
|
|
2038
|
+
apply: () => this.createRunReceiptsSchema()
|
|
2039
|
+
},
|
|
2040
|
+
{
|
|
2041
|
+
id: "0010_work_item_machine_id",
|
|
2042
|
+
apply: () => {
|
|
2043
|
+
this.addColumnIfMissing("workflow_work_items", "machine_id", "TEXT");
|
|
2044
|
+
this.db.exec("CREATE INDEX IF NOT EXISTS idx_workflow_work_items_machine ON workflow_work_items(machine_id, status)");
|
|
2045
|
+
}
|
|
1612
2046
|
}
|
|
1613
2047
|
];
|
|
1614
2048
|
}
|
|
@@ -1670,6 +2104,28 @@ class Store {
|
|
|
1670
2104
|
CREATE INDEX IF NOT EXISTS idx_runs_status_lease ON loop_runs(status, lease_expires_at);
|
|
1671
2105
|
CREATE INDEX IF NOT EXISTS idx_runs_scheduled ON loop_runs(scheduled_for);
|
|
1672
2106
|
|
|
2107
|
+
CREATE TABLE IF NOT EXISTS run_receipts (
|
|
2108
|
+
run_id TEXT PRIMARY KEY,
|
|
2109
|
+
loop_id TEXT NOT NULL,
|
|
2110
|
+
machine_json TEXT NOT NULL,
|
|
2111
|
+
repo TEXT NOT NULL,
|
|
2112
|
+
task_ids_json TEXT NOT NULL,
|
|
2113
|
+
knowledge_ids_json TEXT NOT NULL,
|
|
2114
|
+
digest_id TEXT NOT NULL,
|
|
2115
|
+
started_at TEXT,
|
|
2116
|
+
finished_at TEXT,
|
|
2117
|
+
status TEXT NOT NULL,
|
|
2118
|
+
exit_code INTEGER,
|
|
2119
|
+
summary_json TEXT NOT NULL,
|
|
2120
|
+
evidence_paths_json TEXT NOT NULL,
|
|
2121
|
+
created_at TEXT NOT NULL,
|
|
2122
|
+
updated_at TEXT NOT NULL
|
|
2123
|
+
);
|
|
2124
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_loop ON run_receipts(loop_id, created_at DESC);
|
|
2125
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_repo ON run_receipts(repo, created_at DESC);
|
|
2126
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_digest ON run_receipts(digest_id);
|
|
2127
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_status ON run_receipts(status, created_at DESC);
|
|
2128
|
+
|
|
1673
2129
|
CREATE TABLE IF NOT EXISTS daemon_lease (
|
|
1674
2130
|
id TEXT PRIMARY KEY,
|
|
1675
2131
|
pid INTEGER NOT NULL,
|
|
@@ -1756,6 +2212,7 @@ class Store {
|
|
|
1756
2212
|
subject_ref TEXT NOT NULL,
|
|
1757
2213
|
project_key TEXT,
|
|
1758
2214
|
project_group TEXT,
|
|
2215
|
+
machine_id TEXT,
|
|
1759
2216
|
route_scope TEXT,
|
|
1760
2217
|
priority INTEGER NOT NULL,
|
|
1761
2218
|
status TEXT NOT NULL,
|
|
@@ -1774,10 +2231,10 @@ class Store {
|
|
|
1774
2231
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_project ON workflow_work_items(project_key, status);
|
|
1775
2232
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_group ON workflow_work_items(project_group, status);
|
|
1776
2233
|
CREATE INDEX IF NOT EXISTS idx_workflow_work_items_invocation ON workflow_work_items(invocation_id);
|
|
1777
|
-
--
|
|
1778
|
-
--
|
|
2234
|
+
-- New-column indexes (route_scope, machine_id, etc.) are created ONLY by
|
|
2235
|
+
-- their additive migrations, never here: this baseline DDL
|
|
1779
2236
|
-- re-runs on EVERY open (0001 is not skip-guarded), and on a pre-0008
|
|
1780
|
-
-- database the CREATE TABLE above is a no-op, so an index on
|
|
2237
|
+
-- database the CREATE TABLE above is a no-op, so an index on a new column
|
|
1781
2238
|
-- here would execute before the column exists and crash the open
|
|
1782
2239
|
-- ("no such column: route_scope"). New columns may be folded into the
|
|
1783
2240
|
-- CREATE TABLE (fresh-db only); their indexes must live in the migration.
|
|
@@ -1888,10 +2345,35 @@ class Store {
|
|
|
1888
2345
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
1889
2346
|
`);
|
|
1890
2347
|
}
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
2348
|
+
createRunReceiptsSchema() {
|
|
2349
|
+
this.db.exec(`
|
|
2350
|
+
CREATE TABLE IF NOT EXISTS run_receipts (
|
|
2351
|
+
run_id TEXT PRIMARY KEY,
|
|
2352
|
+
loop_id TEXT NOT NULL,
|
|
2353
|
+
machine_json TEXT NOT NULL,
|
|
2354
|
+
repo TEXT NOT NULL,
|
|
2355
|
+
task_ids_json TEXT NOT NULL,
|
|
2356
|
+
knowledge_ids_json TEXT NOT NULL,
|
|
2357
|
+
digest_id TEXT NOT NULL,
|
|
2358
|
+
started_at TEXT,
|
|
2359
|
+
finished_at TEXT,
|
|
2360
|
+
status TEXT NOT NULL,
|
|
2361
|
+
exit_code INTEGER,
|
|
2362
|
+
summary_json TEXT NOT NULL,
|
|
2363
|
+
evidence_paths_json TEXT NOT NULL,
|
|
2364
|
+
created_at TEXT NOT NULL,
|
|
2365
|
+
updated_at TEXT NOT NULL
|
|
2366
|
+
);
|
|
2367
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_loop ON run_receipts(loop_id, created_at DESC);
|
|
2368
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_repo ON run_receipts(repo, created_at DESC);
|
|
2369
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_digest ON run_receipts(digest_id);
|
|
2370
|
+
CREATE INDEX IF NOT EXISTS idx_run_receipts_status ON run_receipts(status, created_at DESC);
|
|
2371
|
+
`);
|
|
2372
|
+
}
|
|
2373
|
+
addColumnIfMissing(table, column, definition) {
|
|
2374
|
+
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
2375
|
+
if (columns.some((c) => c.name === column))
|
|
2376
|
+
return;
|
|
1895
2377
|
this.db.query(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`).run();
|
|
1896
2378
|
}
|
|
1897
2379
|
createWorkflowRunBackfillIndexes() {
|
|
@@ -1999,21 +2481,51 @@ class Store {
|
|
|
1999
2481
|
}
|
|
2000
2482
|
listLoops(opts = {}) {
|
|
2001
2483
|
const limit = opts.limit ?? 200;
|
|
2484
|
+
const offset = Math.max(0, Math.floor(opts.offset ?? 0));
|
|
2485
|
+
if (opts.name != null) {
|
|
2486
|
+
const rows2 = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.name, limit, offset);
|
|
2487
|
+
return this.withLatestRunSummaries(rows2.map(rowToLoop));
|
|
2488
|
+
}
|
|
2002
2489
|
let rows;
|
|
2003
2490
|
if (opts.status && opts.archived) {
|
|
2004
|
-
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NOT NULL ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
|
|
2491
|
+
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NOT NULL ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
2005
2492
|
} else if (opts.status && opts.includeArchived) {
|
|
2006
|
-
rows = this.db.query("SELECT * FROM loops WHERE status = ? ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
|
|
2493
|
+
rows = this.db.query("SELECT * FROM loops WHERE status = ? ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
2007
2494
|
} else if (opts.status) {
|
|
2008
|
-
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NULL ORDER BY next_run_at ASC LIMIT ?").all(opts.status, limit);
|
|
2495
|
+
rows = this.db.query("SELECT * FROM loops WHERE status = ? AND archived_at IS NULL ORDER BY next_run_at ASC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
2009
2496
|
} else if (opts.archived) {
|
|
2010
|
-
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NOT NULL ORDER BY archived_at DESC LIMIT ?").all(limit);
|
|
2497
|
+
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NOT NULL ORDER BY archived_at DESC LIMIT ? OFFSET ?").all(limit, offset);
|
|
2011
2498
|
} else if (opts.includeArchived) {
|
|
2012
|
-
rows = this.db.query("SELECT * FROM loops ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2499
|
+
rows = this.db.query("SELECT * FROM loops ORDER BY status ASC, next_run_at ASC LIMIT ? OFFSET ?").all(limit, offset);
|
|
2013
2500
|
} else {
|
|
2014
|
-
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ?").all(limit);
|
|
2015
|
-
}
|
|
2016
|
-
return rows.map(rowToLoop);
|
|
2501
|
+
rows = this.db.query("SELECT * FROM loops WHERE archived_at IS NULL ORDER BY status ASC, next_run_at ASC LIMIT ? OFFSET ?").all(limit, offset);
|
|
2502
|
+
}
|
|
2503
|
+
return this.withLatestRunSummaries(rows.map(rowToLoop));
|
|
2504
|
+
}
|
|
2505
|
+
withLatestRunSummaries(loops) {
|
|
2506
|
+
if (loops.length === 0)
|
|
2507
|
+
return loops;
|
|
2508
|
+
const placeholders = loops.map(() => "?").join(",");
|
|
2509
|
+
const rows = this.db.query(`SELECT loop_id, id, status, started_at, finished_at, created_at
|
|
2510
|
+
FROM (
|
|
2511
|
+
SELECT loop_id, id, status, started_at, finished_at, created_at,
|
|
2512
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS rn
|
|
2513
|
+
FROM loop_runs
|
|
2514
|
+
WHERE loop_id IN (${placeholders})
|
|
2515
|
+
)
|
|
2516
|
+
WHERE rn = 1`).all(...loops.map((loop) => loop.id));
|
|
2517
|
+
const latestByLoopId = new Map(rows.map((row) => [row.loop_id, row]));
|
|
2518
|
+
return loops.map((loop) => {
|
|
2519
|
+
const latest = latestByLoopId.get(loop.id);
|
|
2520
|
+
if (!latest)
|
|
2521
|
+
return loop;
|
|
2522
|
+
return {
|
|
2523
|
+
...loop,
|
|
2524
|
+
latestRunId: latest.id,
|
|
2525
|
+
latestRunStatus: latest.status,
|
|
2526
|
+
lastRunAt: latestRunTime(latest)
|
|
2527
|
+
};
|
|
2528
|
+
});
|
|
2017
2529
|
}
|
|
2018
2530
|
dueLoops(now, limit = 500) {
|
|
2019
2531
|
const rows = this.db.query(`SELECT * FROM loops
|
|
@@ -2132,6 +2644,45 @@ class Store {
|
|
|
2132
2644
|
throw error;
|
|
2133
2645
|
}
|
|
2134
2646
|
}
|
|
2647
|
+
updateAgentLoopTimeout(idOrName, timeoutMs, opts = {}) {
|
|
2648
|
+
const updated = (opts.now ?? new Date).toISOString();
|
|
2649
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
2650
|
+
try {
|
|
2651
|
+
const current = this.requireUniqueLoop(idOrName);
|
|
2652
|
+
if (current.archivedAt)
|
|
2653
|
+
throw new LoopArchivedError(current.name || current.id);
|
|
2654
|
+
if (current.target.type !== "agent")
|
|
2655
|
+
throw new Error(`loop is not an agent loop: ${idOrName}`);
|
|
2656
|
+
if (this.hasRunningRun(current.id))
|
|
2657
|
+
throw new Error(`refusing to update running loop: ${current.id}`);
|
|
2658
|
+
const target = { ...current.target, timeoutMs };
|
|
2659
|
+
if (timeoutMs === null && target.idleTimeoutMs !== undefined)
|
|
2660
|
+
delete target.idleTimeoutMs;
|
|
2661
|
+
const res = this.db.query(`UPDATE loops SET target_json=$target, updated_at=$updated
|
|
2662
|
+
WHERE id=$id
|
|
2663
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2664
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2665
|
+
))`).run({
|
|
2666
|
+
$id: current.id,
|
|
2667
|
+
$target: JSON.stringify(target),
|
|
2668
|
+
$updated: updated,
|
|
2669
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2670
|
+
$now: updated
|
|
2671
|
+
});
|
|
2672
|
+
if (res.changes !== 1)
|
|
2673
|
+
throw new Error("daemon lease lost");
|
|
2674
|
+
this.db.exec("COMMIT");
|
|
2675
|
+
const after = this.getLoop(current.id);
|
|
2676
|
+
if (!after)
|
|
2677
|
+
throw new Error(`loop not found after timeout update: ${current.id}`);
|
|
2678
|
+
return after;
|
|
2679
|
+
} catch (error) {
|
|
2680
|
+
try {
|
|
2681
|
+
this.db.exec("ROLLBACK");
|
|
2682
|
+
} catch {}
|
|
2683
|
+
throw error;
|
|
2684
|
+
}
|
|
2685
|
+
}
|
|
2135
2686
|
createAndRetargetWorkflowLoop(idOrName, workflowInput, opts = {}) {
|
|
2136
2687
|
const normalized = normalizeCreateWorkflowInput(workflowInput);
|
|
2137
2688
|
const updated = (opts.now ?? new Date).toISOString();
|
|
@@ -2462,6 +3013,59 @@ class Store {
|
|
|
2462
3013
|
updated
|
|
2463
3014
|
});
|
|
2464
3015
|
}
|
|
3016
|
+
taskLifecycleTodosPointerContext(workflowRunId) {
|
|
3017
|
+
const run = this.getWorkflowRun(workflowRunId);
|
|
3018
|
+
if (!run || run.status !== "succeeded" || !run.invocationId || !run.workItemId || !run.manifestPath)
|
|
3019
|
+
return;
|
|
3020
|
+
const workItem = this.getWorkflowWorkItem(run.workItemId);
|
|
3021
|
+
if (!workItem || workItem.routeKey !== "todos-task")
|
|
3022
|
+
return;
|
|
3023
|
+
const invocation = this.getWorkflowInvocation(run.invocationId);
|
|
3024
|
+
if (!invocation || invocation.templateId !== TASK_LIFECYCLE_TEMPLATE_ID)
|
|
3025
|
+
return;
|
|
3026
|
+
const projectPath = workItem.projectKey ?? invocation.scope?.projectPath;
|
|
3027
|
+
const taskId = invocation.subjectRef.id ?? workItem.subjectRef;
|
|
3028
|
+
if (!projectPath || !taskId)
|
|
3029
|
+
return;
|
|
3030
|
+
return {
|
|
3031
|
+
projectPath,
|
|
3032
|
+
taskId,
|
|
3033
|
+
invocationId: invocation.id,
|
|
3034
|
+
workflowRunId: run.id,
|
|
3035
|
+
manifestPath: run.manifestPath
|
|
3036
|
+
};
|
|
3037
|
+
}
|
|
3038
|
+
syncSuccessfulTaskLifecycleTodosPointers(workflowRunId) {
|
|
3039
|
+
const context = this.taskLifecycleTodosPointerContext(workflowRunId);
|
|
3040
|
+
if (!context)
|
|
3041
|
+
return;
|
|
3042
|
+
const result = runLocalCommand("todos", [
|
|
3043
|
+
"--project",
|
|
3044
|
+
context.projectPath,
|
|
3045
|
+
"task",
|
|
3046
|
+
"workflow-pointers",
|
|
3047
|
+
context.taskId,
|
|
3048
|
+
"--clear",
|
|
3049
|
+
"--invocation",
|
|
3050
|
+
context.invocationId,
|
|
3051
|
+
"--run",
|
|
3052
|
+
context.workflowRunId,
|
|
3053
|
+
"--manifest",
|
|
3054
|
+
context.manifestPath,
|
|
3055
|
+
"--state",
|
|
3056
|
+
"succeeded",
|
|
3057
|
+
"--actor",
|
|
3058
|
+
"openloops:task-lifecycle"
|
|
3059
|
+
]);
|
|
3060
|
+
this.appendWorkflowEvent(workflowRunId, result.ok ? "todos_workflow_pointers_synced" : "todos_workflow_pointers_sync_failed", undefined, {
|
|
3061
|
+
projectPath: context.projectPath,
|
|
3062
|
+
taskId: context.taskId,
|
|
3063
|
+
invocationId: context.invocationId,
|
|
3064
|
+
workflowRunId: context.workflowRunId,
|
|
3065
|
+
manifestPath: context.manifestPath,
|
|
3066
|
+
mutation: todosMutationSummary(result)
|
|
3067
|
+
});
|
|
3068
|
+
}
|
|
2465
3069
|
createWorkflowInvocation(input) {
|
|
2466
3070
|
const now = nowIso();
|
|
2467
3071
|
const sourceDedupeKey = input.sourceRef.dedupeKey ?? undefined;
|
|
@@ -2572,10 +3176,10 @@ class Store {
|
|
|
2572
3176
|
const id = genId();
|
|
2573
3177
|
const status = input.status ?? "queued";
|
|
2574
3178
|
this.db.query(`INSERT INTO workflow_work_items (id, route_key, idempotency_key, invocation_id, source_type, source_ref,
|
|
2575
|
-
subject_ref, project_key, project_group, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
3179
|
+
subject_ref, project_key, project_group, machine_id, route_scope, priority, status, attempts, next_attempt_at, lease_expires_at,
|
|
2576
3180
|
workflow_id, loop_id, workflow_run_id, last_reason, created_at, updated_at)
|
|
2577
3181
|
VALUES ($id, $routeKey, $idempotencyKey, $invocationId, $sourceType, $sourceRef, $subjectRef,
|
|
2578
|
-
$projectKey, $projectGroup, $routeScope, $priority, $status, 0, $nextAttemptAt, NULL, NULL, NULL, NULL,
|
|
3182
|
+
$projectKey, $projectGroup, $machineId, $routeScope, $priority, $status, 0, $nextAttemptAt, NULL, NULL, NULL, NULL,
|
|
2579
3183
|
$lastReason, $created, $updated)
|
|
2580
3184
|
ON CONFLICT(route_key, idempotency_key) DO UPDATE SET
|
|
2581
3185
|
invocation_id=excluded.invocation_id,
|
|
@@ -2584,6 +3188,10 @@ class Store {
|
|
|
2584
3188
|
subject_ref=excluded.subject_ref,
|
|
2585
3189
|
project_key=excluded.project_key,
|
|
2586
3190
|
project_group=excluded.project_group,
|
|
3191
|
+
machine_id=CASE
|
|
3192
|
+
WHEN workflow_work_items.status IN ('succeeded', 'admitted', 'running', 'failed', 'dead_letter', 'cancelled') THEN workflow_work_items.machine_id
|
|
3193
|
+
ELSE excluded.machine_id
|
|
3194
|
+
END,
|
|
2587
3195
|
route_scope=excluded.route_scope,
|
|
2588
3196
|
priority=excluded.priority,
|
|
2589
3197
|
status=CASE
|
|
@@ -2626,6 +3234,7 @@ class Store {
|
|
|
2626
3234
|
$subjectRef: input.subjectRef,
|
|
2627
3235
|
$projectKey: input.projectKey ?? null,
|
|
2628
3236
|
$projectGroup: input.projectGroup ?? null,
|
|
3237
|
+
$machineId: input.machineId ?? null,
|
|
2629
3238
|
$routeScope: input.routeScope ?? null,
|
|
2630
3239
|
$priority: input.priority ?? 0,
|
|
2631
3240
|
$status: status,
|
|
@@ -3135,7 +3744,7 @@ class Store {
|
|
|
3135
3744
|
VALUES ($id, $workflowRunId, 1, 'created', NULL, $payload, $created)`).run({
|
|
3136
3745
|
$id: genId(),
|
|
3137
3746
|
$workflowRunId: runId,
|
|
3138
|
-
$payload:
|
|
3747
|
+
$payload: persistedWorkflowEventPayload({
|
|
3139
3748
|
workflowId: input.workflow.id,
|
|
3140
3749
|
workflowName: input.workflow.name,
|
|
3141
3750
|
stepCount: input.workflow.steps.length,
|
|
@@ -3424,6 +4033,8 @@ class Store {
|
|
|
3424
4033
|
const run = this.getWorkflowRun(workflowRunId);
|
|
3425
4034
|
if (!run)
|
|
3426
4035
|
throw new Error(`workflow run not found after finalize: ${workflowRunId}`);
|
|
4036
|
+
if (changed && status === "succeeded")
|
|
4037
|
+
this.syncSuccessfulTaskLifecycleTodosPointers(workflowRunId);
|
|
3427
4038
|
return run;
|
|
3428
4039
|
}
|
|
3429
4040
|
cancelWorkflowRun(workflowRunId, reason = "cancelled by user") {
|
|
@@ -3464,7 +4075,7 @@ class Store {
|
|
|
3464
4075
|
$sequence: sequence,
|
|
3465
4076
|
$eventType: eventType,
|
|
3466
4077
|
$stepId: stepId ?? null,
|
|
3467
|
-
$payload:
|
|
4078
|
+
$payload: persistedWorkflowEventPayload(payload),
|
|
3468
4079
|
$created: now
|
|
3469
4080
|
});
|
|
3470
4081
|
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
@@ -3485,6 +4096,17 @@ class Store {
|
|
|
3485
4096
|
const row = this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE loop_id = ? AND scheduled_for = ? AND status = 'running'").get(loopId, scheduledFor);
|
|
3486
4097
|
return (row?.count ?? 0) > 0;
|
|
3487
4098
|
}
|
|
4099
|
+
hasBlockingRunningRunForOtherSlot(loopId, scheduledFor, nowIso2) {
|
|
4100
|
+
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
4101
|
+
WHERE loop_id = ? AND scheduled_for <> ? AND status = 'running'`).all(loopId, scheduledFor);
|
|
4102
|
+
return rows.some((row) => {
|
|
4103
|
+
if (!row.lease_expires_at || row.lease_expires_at > nowIso2)
|
|
4104
|
+
return true;
|
|
4105
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at))
|
|
4106
|
+
return true;
|
|
4107
|
+
return this.hasLiveWorkflowStepProcesses(row.id);
|
|
4108
|
+
});
|
|
4109
|
+
}
|
|
3488
4110
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
3489
4111
|
const now = (opts.now ?? new Date).toISOString();
|
|
3490
4112
|
const startedMs = processStartTimeMs(pid);
|
|
@@ -3617,6 +4239,10 @@ class Store {
|
|
|
3617
4239
|
loop = currentLoop;
|
|
3618
4240
|
const leaseExpiresAt = new Date(now.getTime() + loop.leaseMs).toISOString();
|
|
3619
4241
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
4242
|
+
if (loop.overlap === "skip" && this.hasBlockingRunningRunForOtherSlot(loop.id, scheduledFor, startedAt)) {
|
|
4243
|
+
this.db.exec("COMMIT");
|
|
4244
|
+
return;
|
|
4245
|
+
}
|
|
3620
4246
|
if (existing) {
|
|
3621
4247
|
if (existing.status === "running") {
|
|
3622
4248
|
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
@@ -3772,18 +4398,93 @@ class Store {
|
|
|
3772
4398
|
}
|
|
3773
4399
|
listRuns(opts = {}) {
|
|
3774
4400
|
const limit = opts.limit ?? 100;
|
|
4401
|
+
const offset = Math.max(0, Math.floor(opts.offset ?? 0));
|
|
3775
4402
|
let rows;
|
|
3776
4403
|
if (opts.loopId && opts.status) {
|
|
3777
|
-
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND status = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopId, opts.status, limit);
|
|
4404
|
+
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? AND status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.loopId, opts.status, limit, offset);
|
|
3778
4405
|
} else if (opts.loopId) {
|
|
3779
|
-
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? ORDER BY created_at DESC LIMIT ?").all(opts.loopId, limit);
|
|
4406
|
+
rows = this.db.query("SELECT * FROM loop_runs WHERE loop_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.loopId, limit, offset);
|
|
3780
4407
|
} else if (opts.status) {
|
|
3781
|
-
rows = this.db.query("SELECT * FROM loop_runs WHERE status = ? ORDER BY created_at DESC LIMIT ?").all(opts.status, limit);
|
|
4408
|
+
rows = this.db.query("SELECT * FROM loop_runs WHERE status = ? ORDER BY created_at DESC LIMIT ? OFFSET ?").all(opts.status, limit, offset);
|
|
3782
4409
|
} else {
|
|
3783
|
-
rows = this.db.query("SELECT * FROM loop_runs ORDER BY created_at DESC LIMIT ?").all(limit);
|
|
4410
|
+
rows = this.db.query("SELECT * FROM loop_runs ORDER BY created_at DESC LIMIT ? OFFSET ?").all(limit, offset);
|
|
3784
4411
|
}
|
|
3785
4412
|
return rows.map(rowToRun);
|
|
3786
4413
|
}
|
|
4414
|
+
writeRunReceipt(input, opts = {}) {
|
|
4415
|
+
const inputRunId = typeof input.run_id === "string" && input.run_id.trim() ? input.run_id : undefined;
|
|
4416
|
+
const existing = inputRunId ? this.getRunReceipt(inputRunId) : undefined;
|
|
4417
|
+
const run = inputRunId ? this.getRun(inputRunId) : undefined;
|
|
4418
|
+
const loop = input.loop_id ? this.getLoop(input.loop_id) : run ? this.getLoop(run.loopId) : undefined;
|
|
4419
|
+
const receipt = normalizeRunReceipt(input, { now: opts.now, run, loop, existing });
|
|
4420
|
+
this.db.query(`INSERT INTO run_receipts (run_id, loop_id, machine_json, repo, task_ids_json, knowledge_ids_json, digest_id,
|
|
4421
|
+
started_at, finished_at, status, exit_code, summary_json, evidence_paths_json, created_at, updated_at)
|
|
4422
|
+
VALUES ($runId, $loopId, $machineJson, $repo, $taskIdsJson, $knowledgeIdsJson, $digestId,
|
|
4423
|
+
$startedAt, $finishedAt, $status, $exitCode, $summaryJson, $evidencePathsJson, $createdAt, $updatedAt)
|
|
4424
|
+
ON CONFLICT(run_id) DO UPDATE SET
|
|
4425
|
+
loop_id=excluded.loop_id,
|
|
4426
|
+
machine_json=excluded.machine_json,
|
|
4427
|
+
repo=excluded.repo,
|
|
4428
|
+
task_ids_json=excluded.task_ids_json,
|
|
4429
|
+
knowledge_ids_json=excluded.knowledge_ids_json,
|
|
4430
|
+
digest_id=excluded.digest_id,
|
|
4431
|
+
started_at=excluded.started_at,
|
|
4432
|
+
finished_at=excluded.finished_at,
|
|
4433
|
+
status=excluded.status,
|
|
4434
|
+
exit_code=excluded.exit_code,
|
|
4435
|
+
summary_json=excluded.summary_json,
|
|
4436
|
+
evidence_paths_json=excluded.evidence_paths_json,
|
|
4437
|
+
updated_at=excluded.updated_at`).run({
|
|
4438
|
+
$runId: receipt.run_id,
|
|
4439
|
+
$loopId: receipt.loop_id,
|
|
4440
|
+
$machineJson: JSON.stringify(receipt.machine),
|
|
4441
|
+
$repo: receipt.repo,
|
|
4442
|
+
$taskIdsJson: JSON.stringify(receipt.task_ids),
|
|
4443
|
+
$knowledgeIdsJson: JSON.stringify(receipt.knowledge_ids),
|
|
4444
|
+
$digestId: receipt.digest_id,
|
|
4445
|
+
$startedAt: receipt.started_at,
|
|
4446
|
+
$finishedAt: receipt.finished_at,
|
|
4447
|
+
$status: receipt.status,
|
|
4448
|
+
$exitCode: receipt.exit_code,
|
|
4449
|
+
$summaryJson: JSON.stringify(receipt.summary),
|
|
4450
|
+
$evidencePathsJson: JSON.stringify(receipt.evidence_paths),
|
|
4451
|
+
$createdAt: receipt.created_at,
|
|
4452
|
+
$updatedAt: receipt.updated_at
|
|
4453
|
+
});
|
|
4454
|
+
return this.getRunReceipt(receipt.run_id) ?? receipt;
|
|
4455
|
+
}
|
|
4456
|
+
getRunReceipt(runId) {
|
|
4457
|
+
const row = this.db.query("SELECT * FROM run_receipts WHERE run_id = ?").get(runId);
|
|
4458
|
+
return row ? rowToRunReceipt(row) : undefined;
|
|
4459
|
+
}
|
|
4460
|
+
listRunReceipts(opts = {}) {
|
|
4461
|
+
const limit = opts.limit ?? 100;
|
|
4462
|
+
const filters = [];
|
|
4463
|
+
const params = [];
|
|
4464
|
+
if (opts.loopId) {
|
|
4465
|
+
filters.push("loop_id = ?");
|
|
4466
|
+
params.push(opts.loopId);
|
|
4467
|
+
}
|
|
4468
|
+
if (opts.repo) {
|
|
4469
|
+
filters.push("repo = ?");
|
|
4470
|
+
params.push(opts.repo);
|
|
4471
|
+
}
|
|
4472
|
+
if (opts.status) {
|
|
4473
|
+
filters.push("status = ?");
|
|
4474
|
+
params.push(opts.status);
|
|
4475
|
+
}
|
|
4476
|
+
if (opts.taskId) {
|
|
4477
|
+
filters.push("EXISTS (SELECT 1 FROM json_each(run_receipts.task_ids_json) WHERE value = ?)");
|
|
4478
|
+
params.push(opts.taskId);
|
|
4479
|
+
}
|
|
4480
|
+
if (opts.knowledgeId) {
|
|
4481
|
+
filters.push("EXISTS (SELECT 1 FROM json_each(run_receipts.knowledge_ids_json) WHERE value = ?)");
|
|
4482
|
+
params.push(opts.knowledgeId);
|
|
4483
|
+
}
|
|
4484
|
+
const where = filters.length ? `WHERE ${filters.join(" AND ")}` : "";
|
|
4485
|
+
const rows = this.db.query(`SELECT * FROM run_receipts ${where} ORDER BY created_at DESC LIMIT ?`).all(...params, limit);
|
|
4486
|
+
return rows.map(rowToRunReceipt);
|
|
4487
|
+
}
|
|
3787
4488
|
deferLiveExpiredRun(id, now, opts = {}) {
|
|
3788
4489
|
const updated = now.toISOString();
|
|
3789
4490
|
const deferredUntil = new Date(now.getTime() + LIVE_EXPIRED_RUN_GRACE_MS).toISOString();
|
|
@@ -3942,6 +4643,9 @@ class Store {
|
|
|
3942
4643
|
const runs = includeRuns ? this.db.query("SELECT * FROM loop_runs ORDER BY created_at ASC, id ASC").all().map(rowToRun) : [];
|
|
3943
4644
|
return { schemaVersion: SCHEMA_USER_VERSION, workflows, loops, runs, checks: this.migrationChecks() };
|
|
3944
4645
|
}
|
|
4646
|
+
exportMigrationRunPage(opts) {
|
|
4647
|
+
return this.db.query("SELECT * FROM loop_runs ORDER BY created_at ASC, id ASC LIMIT ? OFFSET ?").all(opts.limit, opts.offset).map(rowToRun);
|
|
4648
|
+
}
|
|
3945
4649
|
countTable(table) {
|
|
3946
4650
|
const row = this.db.query(`SELECT COUNT(*) AS count FROM ${table}`).get();
|
|
3947
4651
|
return row?.count ?? 0;
|
|
@@ -4188,9 +4892,9 @@ class Store {
|
|
|
4188
4892
|
const runDir = dirname2(manifestPath);
|
|
4189
4893
|
try {
|
|
4190
4894
|
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
4191
|
-
|
|
4895
|
+
rmSync3(runDir, { recursive: true, force: true });
|
|
4192
4896
|
} else {
|
|
4193
|
-
|
|
4897
|
+
rmSync3(manifestPath, { force: true });
|
|
4194
4898
|
}
|
|
4195
4899
|
} catch {}
|
|
4196
4900
|
}
|
|
@@ -4257,7 +4961,7 @@ class Store {
|
|
|
4257
4961
|
this.db.close();
|
|
4258
4962
|
if (this.memoryRootDir) {
|
|
4259
4963
|
try {
|
|
4260
|
-
|
|
4964
|
+
rmSync3(this.memoryRootDir, { recursive: true, force: true });
|
|
4261
4965
|
} catch {}
|
|
4262
4966
|
this.memoryRootDir = undefined;
|
|
4263
4967
|
}
|
|
@@ -4266,7 +4970,7 @@ class Store {
|
|
|
4266
4970
|
// package.json
|
|
4267
4971
|
var package_default = {
|
|
4268
4972
|
name: "@hasna/loops",
|
|
4269
|
-
version: "0.4.
|
|
4973
|
+
version: "0.4.22",
|
|
4270
4974
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4271
4975
|
type: "module",
|
|
4272
4976
|
main: "dist/index.js",
|
|
@@ -4275,6 +4979,7 @@ var package_default = {
|
|
|
4275
4979
|
loops: "dist/cli/index.js",
|
|
4276
4980
|
"loops-daemon": "dist/daemon/index.js",
|
|
4277
4981
|
"loops-api": "dist/api/index.js",
|
|
4982
|
+
"loops-serve": "dist/serve/index.js",
|
|
4278
4983
|
"loops-runner": "dist/runner/index.js",
|
|
4279
4984
|
"loops-mcp": "dist/mcp/index.js"
|
|
4280
4985
|
},
|
|
@@ -4287,6 +4992,14 @@ var package_default = {
|
|
|
4287
4992
|
types: "./dist/sdk/index.d.ts",
|
|
4288
4993
|
import: "./dist/sdk/index.js"
|
|
4289
4994
|
},
|
|
4995
|
+
"./sdk/http": {
|
|
4996
|
+
types: "./dist/sdk/http.d.ts",
|
|
4997
|
+
import: "./dist/sdk/http.js"
|
|
4998
|
+
},
|
|
4999
|
+
"./serve": {
|
|
5000
|
+
types: "./dist/serve/index.d.ts",
|
|
5001
|
+
import: "./dist/serve/index.js"
|
|
5002
|
+
},
|
|
4290
5003
|
"./mcp": {
|
|
4291
5004
|
types: "./dist/mcp/index.d.ts",
|
|
4292
5005
|
import: "./dist/mcp/index.js"
|
|
@@ -4332,7 +5045,7 @@ var package_default = {
|
|
|
4332
5045
|
"LICENSE"
|
|
4333
5046
|
],
|
|
4334
5047
|
scripts: {
|
|
4335
|
-
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
5048
|
+
build: "rm -rf dist && bun build src/cli/index.ts src/daemon/index.ts src/api/index.ts src/serve/index.ts src/runner/index.ts src/mcp/index.ts src/index.ts src/sdk/index.ts src/sdk/http.ts src/lib/store.ts src/lib/mode.ts src/lib/storage/index.ts src/lib/storage/contract.ts src/lib/storage/sqlite.ts src/lib/storage/postgres.ts src/lib/storage/postgres-schema.ts --root src --outdir dist --target bun --packages external && chmod +x dist/cli/index.js dist/daemon/index.js dist/api/index.js dist/serve/index.js dist/runner/index.js dist/mcp/index.js && tsc -p tsconfig.build.json --emitDeclarationOnly --outDir dist",
|
|
4336
5049
|
"build:bin": "bun build src/cli/index.ts --compile --outfile dist/loops",
|
|
4337
5050
|
typecheck: "tsc --noEmit",
|
|
4338
5051
|
test: "bun test",
|
|
@@ -4372,13 +5085,16 @@ var package_default = {
|
|
|
4372
5085
|
"@openrouter/ai-sdk-provider": "2.9.1",
|
|
4373
5086
|
ai: "6.0.204",
|
|
4374
5087
|
commander: "^13.1.0",
|
|
5088
|
+
pg: "^8.13.1",
|
|
4375
5089
|
zod: "4.4.3"
|
|
4376
5090
|
},
|
|
4377
5091
|
optionalDependencies: {
|
|
4378
|
-
"@hasna/machines": "0.0.49"
|
|
5092
|
+
"@hasna/machines": "0.0.49",
|
|
5093
|
+
"@hasna/contracts": "^0.4.2"
|
|
4379
5094
|
},
|
|
4380
5095
|
devDependencies: {
|
|
4381
5096
|
"@types/bun": "latest",
|
|
5097
|
+
"@types/pg": "^8.11.10",
|
|
4382
5098
|
typescript: "^5.7.3"
|
|
4383
5099
|
},
|
|
4384
5100
|
publishConfig: {
|
|
@@ -4586,15 +5302,11 @@ function deploymentStatusLine(status) {
|
|
|
4586
5302
|
].join(" ");
|
|
4587
5303
|
}
|
|
4588
5304
|
|
|
4589
|
-
// src/lib/doctor.ts
|
|
4590
|
-
import { spawnSync as spawnSync5 } from "child_process";
|
|
4591
|
-
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
4592
|
-
|
|
4593
5305
|
// src/daemon/control.ts
|
|
4594
|
-
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as
|
|
4595
|
-
import { spawnSync as
|
|
4596
|
-
import { hostname } from "os";
|
|
4597
|
-
import { dirname as dirname3, join as
|
|
5306
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync4, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
5307
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
5308
|
+
import { hostname as hostname2 } from "os";
|
|
5309
|
+
import { dirname as dirname3, join as join5 } from "path";
|
|
4598
5310
|
|
|
4599
5311
|
// src/daemon/loop.ts
|
|
4600
5312
|
function realSleep(ms) {
|
|
@@ -4624,7 +5336,7 @@ function readPidRecord(path = pidFilePath()) {
|
|
|
4624
5336
|
return;
|
|
4625
5337
|
let raw;
|
|
4626
5338
|
try {
|
|
4627
|
-
raw =
|
|
5339
|
+
raw = readFileSync4(path, "utf8").trim();
|
|
4628
5340
|
} catch {
|
|
4629
5341
|
return;
|
|
4630
5342
|
}
|
|
@@ -4650,7 +5362,7 @@ function writePid(pid = process.pid, path = pidFilePath()) {
|
|
|
4650
5362
|
writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
4651
5363
|
}
|
|
4652
5364
|
function removePid(path = pidFilePath()) {
|
|
4653
|
-
|
|
5365
|
+
rmSync4(path, { force: true });
|
|
4654
5366
|
}
|
|
4655
5367
|
function isAlive(pid, startedAt) {
|
|
4656
5368
|
try {
|
|
@@ -4680,7 +5392,7 @@ function ownProcessGroupId() {
|
|
|
4680
5392
|
return pgrp;
|
|
4681
5393
|
}
|
|
4682
5394
|
try {
|
|
4683
|
-
const run =
|
|
5395
|
+
const run = spawnSync3("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
4684
5396
|
const pgid = Number(run.stdout.trim());
|
|
4685
5397
|
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
4686
5398
|
return pgid;
|
|
@@ -4746,7 +5458,7 @@ function daemonStatus(store, path = pidFilePath()) {
|
|
|
4746
5458
|
return {
|
|
4747
5459
|
...isDaemonRunning(path),
|
|
4748
5460
|
lease: store.getDaemonLease(),
|
|
4749
|
-
host:
|
|
5461
|
+
host: hostname2(),
|
|
4750
5462
|
loops: {
|
|
4751
5463
|
total: store.countLoops(),
|
|
4752
5464
|
active: store.countLoops("active"),
|
|
@@ -4776,7 +5488,7 @@ async function stopDaemon(opts = {}) {
|
|
|
4776
5488
|
if (!state.running || !state.pid)
|
|
4777
5489
|
return { wasRunning: false, stopped: false, forced: false };
|
|
4778
5490
|
const sleep = opts.sleep ?? realSleep;
|
|
4779
|
-
const store = new Store(
|
|
5491
|
+
const store = new Store(join5(dirname3(path), "loops.db"));
|
|
4780
5492
|
try {
|
|
4781
5493
|
const lease = store.getDaemonLease();
|
|
4782
5494
|
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
@@ -4813,15 +5525,19 @@ async function stopDaemon(opts = {}) {
|
|
|
4813
5525
|
}
|
|
4814
5526
|
}
|
|
4815
5527
|
|
|
5528
|
+
// src/lib/doctor.ts
|
|
5529
|
+
import { spawnSync as spawnSync6 } from "child_process";
|
|
5530
|
+
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
5531
|
+
|
|
4816
5532
|
// src/lib/executor.ts
|
|
4817
|
-
import { spawn as spawn2, spawnSync as
|
|
5533
|
+
import { spawn as spawn2, spawnSync as spawnSync5 } from "child_process";
|
|
4818
5534
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
4819
5535
|
import { once } from "events";
|
|
4820
5536
|
import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
|
|
4821
5537
|
import { dirname as dirname4, resolve as resolve2 } from "path";
|
|
4822
5538
|
|
|
4823
5539
|
// src/lib/accounts.ts
|
|
4824
|
-
import { spawnSync as
|
|
5540
|
+
import { spawnSync as spawnSync4 } from "child_process";
|
|
4825
5541
|
import { existsSync as existsSync3 } from "fs";
|
|
4826
5542
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4827
5543
|
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
@@ -4926,7 +5642,7 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4926
5642
|
if (!account)
|
|
4927
5643
|
return {};
|
|
4928
5644
|
const tool = requiredAccountTool(account, toolHint);
|
|
4929
|
-
const result =
|
|
5645
|
+
const result = spawnSync4("accounts", ["env", account.profile, "--tool", tool], {
|
|
4930
5646
|
encoding: "utf8",
|
|
4931
5647
|
env,
|
|
4932
5648
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -4942,8 +5658,8 @@ function resolveAccountEnvSync(account, toolHint, env) {
|
|
|
4942
5658
|
|
|
4943
5659
|
// src/lib/env.ts
|
|
4944
5660
|
import { accessSync, constants } from "fs";
|
|
4945
|
-
import { homedir as
|
|
4946
|
-
import { delimiter, join as
|
|
5661
|
+
import { homedir as homedir3 } from "os";
|
|
5662
|
+
import { delimiter, join as join6 } from "path";
|
|
4947
5663
|
function compactPathParts(parts) {
|
|
4948
5664
|
const seen = new Set;
|
|
4949
5665
|
const result = [];
|
|
@@ -4957,16 +5673,16 @@ function compactPathParts(parts) {
|
|
|
4957
5673
|
return result;
|
|
4958
5674
|
}
|
|
4959
5675
|
function commonExecutableDirs(env = process.env) {
|
|
4960
|
-
const home = env.HOME ||
|
|
5676
|
+
const home = env.HOME || homedir3();
|
|
4961
5677
|
return compactPathParts([
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
env.BUN_INSTALL ?
|
|
5678
|
+
join6(home, ".local", "bin"),
|
|
5679
|
+
join6(home, ".bun", "bin"),
|
|
5680
|
+
join6(home, ".cargo", "bin"),
|
|
5681
|
+
join6(home, ".npm-global", "bin"),
|
|
5682
|
+
join6(home, "bin"),
|
|
5683
|
+
env.BUN_INSTALL ? join6(env.BUN_INSTALL, "bin") : undefined,
|
|
4968
5684
|
env.PNPM_HOME,
|
|
4969
|
-
env.NPM_CONFIG_PREFIX ?
|
|
5685
|
+
env.NPM_CONFIG_PREFIX ? join6(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
4970
5686
|
"/opt/homebrew/bin",
|
|
4971
5687
|
"/usr/local/bin",
|
|
4972
5688
|
"/usr/bin",
|
|
@@ -4990,7 +5706,7 @@ function executableExists(command, env = process.env) {
|
|
|
4990
5706
|
if (command.includes("/"))
|
|
4991
5707
|
return isExecutable(command);
|
|
4992
5708
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
4993
|
-
if (dir && isExecutable(
|
|
5709
|
+
if (dir && isExecutable(join6(dir, command)))
|
|
4994
5710
|
return true;
|
|
4995
5711
|
}
|
|
4996
5712
|
return false;
|
|
@@ -5099,6 +5815,7 @@ var TRANSPORT_ENV_KEYS = new Set([
|
|
|
5099
5815
|
"LOGNAME",
|
|
5100
5816
|
"PATH",
|
|
5101
5817
|
"SHELL",
|
|
5818
|
+
"SHLVL",
|
|
5102
5819
|
"SSH_AGENT_PID",
|
|
5103
5820
|
"SSH_AUTH_SOCK",
|
|
5104
5821
|
"TERM",
|
|
@@ -5131,6 +5848,32 @@ function timeoutResult(startedAt, error, fields = {}) {
|
|
|
5131
5848
|
function successResult(startedAt, fields = {}) {
|
|
5132
5849
|
return buildResult("succeeded", startedAt, fields);
|
|
5133
5850
|
}
|
|
5851
|
+
function codewithJsonlHasTerminalSuccess(stdout) {
|
|
5852
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
5853
|
+
const trimmed = line.trim();
|
|
5854
|
+
if (!trimmed || !trimmed.startsWith("{"))
|
|
5855
|
+
continue;
|
|
5856
|
+
let event;
|
|
5857
|
+
try {
|
|
5858
|
+
event = JSON.parse(trimmed);
|
|
5859
|
+
} catch {
|
|
5860
|
+
continue;
|
|
5861
|
+
}
|
|
5862
|
+
if (!event || typeof event !== "object")
|
|
5863
|
+
continue;
|
|
5864
|
+
const record = event;
|
|
5865
|
+
if (record.type === "task_complete")
|
|
5866
|
+
return true;
|
|
5867
|
+
const payload = record.payload;
|
|
5868
|
+
if (payload && typeof payload === "object" && payload.type === "task_complete") {
|
|
5869
|
+
return true;
|
|
5870
|
+
}
|
|
5871
|
+
}
|
|
5872
|
+
return false;
|
|
5873
|
+
}
|
|
5874
|
+
function codewithJsonlReconciledSuccess(spec, fields) {
|
|
5875
|
+
return spec.agentProvider === "codewith" && codewithJsonlHasTerminalSuccess(fields.stdout ?? "");
|
|
5876
|
+
}
|
|
5134
5877
|
function notifySpawn(pid, opts) {
|
|
5135
5878
|
if (!pid)
|
|
5136
5879
|
return;
|
|
@@ -5146,10 +5889,48 @@ function shellQuote(value) {
|
|
|
5146
5889
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
5147
5890
|
}
|
|
5148
5891
|
function codewithProfileCandidateFromLine(line) {
|
|
5149
|
-
const
|
|
5150
|
-
if (
|
|
5151
|
-
return
|
|
5152
|
-
|
|
5892
|
+
const trimmed = line.trim();
|
|
5893
|
+
if (!trimmed || trimmed === "No auth profiles saved.")
|
|
5894
|
+
return;
|
|
5895
|
+
const cols = trimmed.split(/\s+/);
|
|
5896
|
+
const candidate = cols[0] === "*" ? cols[1] : cols[0];
|
|
5897
|
+
if (candidate === "NAME" && (cols[1] === "ACCOUNT" || cols[2] === "ACCOUNT"))
|
|
5898
|
+
return;
|
|
5899
|
+
if (candidate === '"name":') {
|
|
5900
|
+
const jsonName = trimmed.match(/"name"\s*:\s*"([^"]+)"/)?.[1];
|
|
5901
|
+
if (jsonName)
|
|
5902
|
+
return jsonName;
|
|
5903
|
+
}
|
|
5904
|
+
return candidate;
|
|
5905
|
+
}
|
|
5906
|
+
function codewithProfileCandidatesFromJson(value) {
|
|
5907
|
+
const candidates = [];
|
|
5908
|
+
const root = value && typeof value === "object" ? value : undefined;
|
|
5909
|
+
const addProfile = (entry) => {
|
|
5910
|
+
if (!entry || typeof entry !== "object")
|
|
5911
|
+
return;
|
|
5912
|
+
const name = entry.name;
|
|
5913
|
+
if (typeof name === "string" && name)
|
|
5914
|
+
candidates.push(name);
|
|
5915
|
+
};
|
|
5916
|
+
const data = root?.data;
|
|
5917
|
+
if (Array.isArray(data))
|
|
5918
|
+
data.forEach(addProfile);
|
|
5919
|
+
const profiles = root?.profiles;
|
|
5920
|
+
if (Array.isArray(profiles))
|
|
5921
|
+
profiles.forEach(addProfile);
|
|
5922
|
+
return candidates;
|
|
5923
|
+
}
|
|
5924
|
+
function codewithProfileCandidatesFromOutput(output) {
|
|
5925
|
+
const trimmed = output.trim();
|
|
5926
|
+
const jsonStart = trimmed.indexOf("{");
|
|
5927
|
+
const jsonEnd = trimmed.lastIndexOf("}");
|
|
5928
|
+
if (jsonStart >= 0 && jsonEnd >= jsonStart) {
|
|
5929
|
+
try {
|
|
5930
|
+
return codewithProfileCandidatesFromJson(JSON.parse(trimmed.slice(jsonStart, jsonEnd + 1)));
|
|
5931
|
+
} catch {}
|
|
5932
|
+
}
|
|
5933
|
+
return output.split(/\r?\n/).map(codewithProfileCandidateFromLine).filter(Boolean);
|
|
5153
5934
|
}
|
|
5154
5935
|
function codewithProfileForError(profile) {
|
|
5155
5936
|
return /[\u0000-\u001F\u007F]/.test(profile) ? JSON.stringify(profile) ?? profile : profile;
|
|
@@ -5242,6 +6023,7 @@ function commandSpec(target, opts) {
|
|
|
5242
6023
|
preflightAnyOf: invocation.preflightAnyOf,
|
|
5243
6024
|
stdin: invocation.stdin,
|
|
5244
6025
|
allowlist: agentTarget.allowlist,
|
|
6026
|
+
agentProvider: agentTarget.provider,
|
|
5245
6027
|
worktree: agentTarget.worktree
|
|
5246
6028
|
};
|
|
5247
6029
|
}
|
|
@@ -5255,6 +6037,7 @@ function composeExecutionEnv(spec, metadata, opts, accountEnv) {
|
|
|
5255
6037
|
Object.assign(env, spec.env ?? {});
|
|
5256
6038
|
Object.assign(env, allowlistEnv(spec.allowlist));
|
|
5257
6039
|
env.PATH = normalizeExecutionPath(env);
|
|
6040
|
+
env.SHLVL ||= "1";
|
|
5258
6041
|
Object.assign(env, metadataEnv(metadata));
|
|
5259
6042
|
return env;
|
|
5260
6043
|
}
|
|
@@ -5326,7 +6109,7 @@ function remoteWorktreePrepareLines(worktree) {
|
|
|
5326
6109
|
return [
|
|
5327
6110
|
"__openloops_prepare_worktree() {",
|
|
5328
6111
|
` local repo=${shellQuote(repoRoot)} path=${shellQuote(path)} branch=${shellQuote(branch)}`,
|
|
5329
|
-
" local top expected_common actual_common current",
|
|
6112
|
+
" local top expected_common actual_common current status recovered",
|
|
5330
6113
|
' if [ -L "$path" ]; then echo "refusing symlinked worktree path $path" >&2; return 1; fi',
|
|
5331
6114
|
' if [ -e "$path" ]; then',
|
|
5332
6115
|
' top="$(git -C "$path" rev-parse --show-toplevel 2>/dev/null)" || { echo "refusing to reuse non-worktree path: $path" >&2; return 1; }',
|
|
@@ -5335,7 +6118,19 @@ function remoteWorktreePrepareLines(worktree) {
|
|
|
5335
6118
|
' actual_common="$(cd "$path" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
|
|
5336
6119
|
' if [ -z "$expected_common" ] || [ "$expected_common" != "$actual_common" ]; then echo "existing worktree $path belongs to a different git common dir" >&2; return 1; fi',
|
|
5337
6120
|
' current="$(git -C "$path" branch --show-current 2>/dev/null)"',
|
|
5338
|
-
' if [ "$current" != "$branch" ]; then
|
|
6121
|
+
' if [ "$current" != "$branch" ]; then',
|
|
6122
|
+
' status="$(git -C "$path" status --porcelain=v1 --untracked-files=all 2>/dev/null)" || { echo "existing worktree $path is on branch ${current:-unknown}, expected $branch, and cleanliness could not be checked; inspect or remove the stale worktree" >&2; return 1; }',
|
|
6123
|
+
' if [ -n "$status" ]; then echo "existing worktree $path is on branch ${current:-unknown}, expected $branch, and has local changes; commit/stash them or remove the stale worktree before retrying" >&2; return 1; fi',
|
|
6124
|
+
' if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
|
|
6125
|
+
' git -C "$path" checkout "$branch" 1>&2 || { echo "existing worktree $path is clean but could not switch from branch ${current:-unknown} to expected $branch; remove/prune the stale worktree or free the branch before retrying" >&2; return 1; }',
|
|
6126
|
+
' elif [ -z "$current" ]; then',
|
|
6127
|
+
' git -C "$path" checkout -b "$branch" 1>&2 || { echo "existing worktree $path is clean but could not recreate expected branch $branch at the current detached HEAD; remove/prune the stale worktree before retrying" >&2; return 1; }',
|
|
6128
|
+
" else",
|
|
6129
|
+
' echo "existing worktree $path is on branch $current, expected $branch, but expected branch does not exist; remove/prune the stale worktree or recreate the expected branch before retrying" >&2; return 1',
|
|
6130
|
+
" fi",
|
|
6131
|
+
' recovered="$(git -C "$path" branch --show-current 2>/dev/null)"',
|
|
6132
|
+
' if [ "$recovered" != "$branch" ]; then echo "existing worktree $path branch recovery ended on ${recovered:-unknown}, expected $branch; remove/prune the stale worktree before retrying" >&2; return 1; fi',
|
|
6133
|
+
" fi",
|
|
5339
6134
|
" return 0",
|
|
5340
6135
|
" fi",
|
|
5341
6136
|
' git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
|
|
@@ -5401,7 +6196,7 @@ function remotePreflightScript(spec, metadata) {
|
|
|
5401
6196
|
}
|
|
5402
6197
|
if (spec.nativeAuthProfile?.provider === "codewith") {
|
|
5403
6198
|
const profileForError = codewithProfileForError(spec.nativeAuthProfile.profile);
|
|
5404
|
-
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE",
|
|
6199
|
+
lines.push(`__OPENLOOPS_CODEWITH_PROFILE=${shellQuote(spec.nativeAuthProfile.profile)}`, "export __OPENLOOPS_CODEWITH_PROFILE", "__openloops_codewith_profile_list_contains() {", ` printf '%s\\n' "$__OPENLOOPS_CODEWITH_PROFILES" | awk '{ line = $0; gsub(/^[[:space:]]+|[[:space:]]+$/, "", line); if (line == "" || line == "No auth profiles saved.") next; if (line ~ /"(data|profiles)"[[:space:]]*:[[:space:]]*\\[/) { in_profiles = 1; next } if (in_profiles && line ~ /^\\]/) { in_profiles = 0; next } json = line; if (in_profiles && json ~ /"name"[[:space:]]*:[[:space:]]*"/) { sub(/^.*"name"[[:space:]]*:[[:space:]]*"/, "", json); sub(/".*$/, "", json); if (json == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1; next } split(line, cols, /[[:space:]]+/); candidate = (cols[1] == "*" ? cols[2] : cols[1]); if (candidate == "NAME" && (cols[2] == "ACCOUNT" || cols[3] == "ACCOUNT")) next; if (candidate == ENVIRON["__OPENLOOPS_CODEWITH_PROFILE"]) found = 1 } END { exit(found ? 0 : 1) }'`, "}", `if __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list --json 2>/dev/null)" && __openloops_codewith_profile_list_contains; then`, " :", "else", ` __OPENLOOPS_CODEWITH_PROFILES="$(${shellQuote(spec.command)} profile list)" || {`, ` printf '%s\\n' ${shellQuote("codewith auth profile preflight failed")} >&2`, " exit 1", " }", " if ! __openloops_codewith_profile_list_contains; then", ` printf '%s\\n' ${shellQuote(`codewith auth profile not found: ${profileForError}`)} >&2`, " exit 1", " fi", "fi");
|
|
5405
6200
|
}
|
|
5406
6201
|
return lines.join(`
|
|
5407
6202
|
`);
|
|
@@ -5416,9 +6211,28 @@ function transportEnv(opts) {
|
|
|
5416
6211
|
env[key] = value;
|
|
5417
6212
|
}
|
|
5418
6213
|
env.PATH = normalizeExecutionPath(env);
|
|
6214
|
+
env.SHLVL ||= "1";
|
|
5419
6215
|
return env;
|
|
5420
6216
|
}
|
|
6217
|
+
function remotePreflightFailureMessage(machineId, detail) {
|
|
6218
|
+
const normalized = detail.trim();
|
|
6219
|
+
if (/Host key verification failed|REMOTE HOST IDENTIFICATION HAS CHANGED|Offending .*key in .*known_hosts/i.test(normalized)) {
|
|
6220
|
+
return [
|
|
6221
|
+
`remote preflight failed on ${machineId}: SSH host key verification failed.`,
|
|
6222
|
+
`Verify ${machineId}'s host identity and repair SSH known_hosts/trust material outside OpenLoops;`,
|
|
6223
|
+
"OpenLoops will not disable host-key checking or modify known_hosts automatically.",
|
|
6224
|
+
normalized ? `Transport detail: ${normalized}` : ""
|
|
6225
|
+
].filter(Boolean).join(" ");
|
|
6226
|
+
}
|
|
6227
|
+
return `remote preflight failed on ${machineId}${normalized ? `: ${normalized}` : ""}`;
|
|
6228
|
+
}
|
|
5421
6229
|
function assertCodewithProfileListed(profile, result) {
|
|
6230
|
+
const profiles = codewithProfileSetFromResult(result);
|
|
6231
|
+
if (!profiles.has(profile)) {
|
|
6232
|
+
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
6233
|
+
}
|
|
6234
|
+
}
|
|
6235
|
+
function codewithProfileSetFromResult(result) {
|
|
5422
6236
|
if (result.error) {
|
|
5423
6237
|
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
5424
6238
|
}
|
|
@@ -5426,32 +6240,45 @@ function assertCodewithProfileListed(profile, result) {
|
|
|
5426
6240
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5427
6241
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
5428
6242
|
}
|
|
5429
|
-
|
|
5430
|
-
if (!profiles.has(profile)) {
|
|
5431
|
-
throw new Error(`codewith auth profile not found: ${codewithProfileForError(profile)}`);
|
|
5432
|
-
}
|
|
6243
|
+
return new Set(codewithProfileCandidatesFromOutput(result.stdout || ""));
|
|
5433
6244
|
}
|
|
5434
6245
|
function preflightNativeAuthProfileSync(spec, env) {
|
|
5435
6246
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5436
6247
|
return;
|
|
5437
|
-
const
|
|
5438
|
-
|
|
5439
|
-
|
|
5440
|
-
|
|
5441
|
-
|
|
5442
|
-
|
|
5443
|
-
|
|
5444
|
-
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5448
|
-
|
|
6248
|
+
const runProfileList = (args) => {
|
|
6249
|
+
const result = spawnSync5(spec.command, args, {
|
|
6250
|
+
encoding: "utf8",
|
|
6251
|
+
env,
|
|
6252
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
6253
|
+
timeout: 15000
|
|
6254
|
+
});
|
|
6255
|
+
return {
|
|
6256
|
+
status: result.status,
|
|
6257
|
+
stdout: result.stdout ?? "",
|
|
6258
|
+
stderr: result.stderr ?? "",
|
|
6259
|
+
error: result.error?.message
|
|
6260
|
+
};
|
|
6261
|
+
};
|
|
6262
|
+
const jsonResult = runProfileList(["profile", "list", "--json"]);
|
|
6263
|
+
try {
|
|
6264
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
6265
|
+
return;
|
|
6266
|
+
} catch {}
|
|
6267
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, runProfileList(["profile", "list"]));
|
|
5449
6268
|
}
|
|
5450
6269
|
async function preflightNativeAuthProfile(spec, env) {
|
|
5451
6270
|
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
5452
6271
|
return;
|
|
5453
|
-
const
|
|
5454
|
-
|
|
6272
|
+
const jsonResult = await spawnCapture(spec.command, ["profile", "list", "--json"], { env, timeoutMs: 15000 });
|
|
6273
|
+
try {
|
|
6274
|
+
if (codewithProfileSetFromResult(jsonResult).has(spec.nativeAuthProfile.profile))
|
|
6275
|
+
return;
|
|
6276
|
+
} catch {}
|
|
6277
|
+
const tableResult = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
6278
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, tableResult);
|
|
6279
|
+
}
|
|
6280
|
+
function spawnDetail(result) {
|
|
6281
|
+
return (result.stderr || result.stdout || result.error || "").toString().trim();
|
|
5455
6282
|
}
|
|
5456
6283
|
function resolvedDirEquals(left, right) {
|
|
5457
6284
|
try {
|
|
@@ -5502,8 +6329,45 @@ async function ensureLocalWorktree(worktree, env) {
|
|
|
5502
6329
|
}
|
|
5503
6330
|
const current = await git(["-C", path, "branch", "--show-current"]);
|
|
5504
6331
|
const actualBranch = current.stdout.trim();
|
|
5505
|
-
if (current.error || (current.status ?? 1) !== 0
|
|
5506
|
-
return { error: `existing worktree ${path}
|
|
6332
|
+
if (current.error || (current.status ?? 1) !== 0) {
|
|
6333
|
+
return { error: `existing worktree ${path} branch could not be inspected${spawnDetail(current) ? `: ${spawnDetail(current)}` : ""}` };
|
|
6334
|
+
}
|
|
6335
|
+
if (actualBranch !== branch) {
|
|
6336
|
+
const actualLabel = actualBranch || "unknown";
|
|
6337
|
+
const status = await git(["-C", path, "status", "--porcelain=v1", "--untracked-files=all"]);
|
|
6338
|
+
if (status.error || (status.status ?? 1) !== 0) {
|
|
6339
|
+
const detail = spawnDetail(status);
|
|
6340
|
+
return {
|
|
6341
|
+
error: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, and cleanliness could not be checked; inspect or remove the stale worktree${detail ? `: ${detail}` : ""}`
|
|
6342
|
+
};
|
|
6343
|
+
}
|
|
6344
|
+
if (status.stdout.trim()) {
|
|
6345
|
+
return {
|
|
6346
|
+
error: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, and has local changes; commit/stash them or remove the stale worktree before retrying`
|
|
6347
|
+
};
|
|
6348
|
+
}
|
|
6349
|
+
const hasBranch2 = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
6350
|
+
const checkout = hasBranch2.status === 0 ? await git(["-C", path, "checkout", branch]) : actualBranch ? {
|
|
6351
|
+
status: 1,
|
|
6352
|
+
stdout: "",
|
|
6353
|
+
stderr: `existing worktree ${path} is on branch ${actualLabel}, expected ${branch}, but expected branch does not exist; remove/prune the stale worktree or recreate the expected branch before retrying`,
|
|
6354
|
+
signal: null,
|
|
6355
|
+
timedOut: false
|
|
6356
|
+
} : await git(["-C", path, "checkout", "-b", branch]);
|
|
6357
|
+
if (checkout.error || (checkout.status ?? 1) !== 0) {
|
|
6358
|
+
const detail = spawnDetail(checkout);
|
|
6359
|
+
return {
|
|
6360
|
+
error: `existing worktree ${path} is clean but could not recover branch ${branch} from ${actualLabel}; remove/prune the stale worktree before retrying${detail ? `: ${detail}` : ""}`
|
|
6361
|
+
};
|
|
6362
|
+
}
|
|
6363
|
+
const recovered = await git(["-C", path, "branch", "--show-current"]);
|
|
6364
|
+
if (recovered.error || (recovered.status ?? 1) !== 0 || recovered.stdout.trim() !== branch) {
|
|
6365
|
+
const recoveredBranch = recovered.stdout.trim() || "unknown";
|
|
6366
|
+
const detail = spawnDetail(recovered);
|
|
6367
|
+
return {
|
|
6368
|
+
error: `existing worktree ${path} branch recovery ended on ${recoveredBranch}, expected ${branch}; remove/prune the stale worktree before retrying${detail ? `: ${detail}` : ""}`
|
|
6369
|
+
};
|
|
6370
|
+
}
|
|
5507
6371
|
}
|
|
5508
6372
|
return { cwd: worktree.cwd };
|
|
5509
6373
|
}
|
|
@@ -5519,7 +6383,7 @@ async function ensureLocalWorktree(worktree, env) {
|
|
|
5519
6383
|
const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
5520
6384
|
const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
|
|
5521
6385
|
if (add.error || (add.status ?? 1) !== 0) {
|
|
5522
|
-
const detail = (add
|
|
6386
|
+
const detail = spawnDetail(add);
|
|
5523
6387
|
return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
|
|
5524
6388
|
}
|
|
5525
6389
|
return { cwd: worktree.cwd };
|
|
@@ -5554,7 +6418,7 @@ function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
|
5554
6418
|
}
|
|
5555
6419
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
5556
6420
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
5557
|
-
const result =
|
|
6421
|
+
const result = spawnSync5(plan.command, plan.args, {
|
|
5558
6422
|
encoding: "utf8",
|
|
5559
6423
|
env: transportEnv(opts),
|
|
5560
6424
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -5565,7 +6429,7 @@ function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
|
5565
6429
|
throw new Error(`remote preflight failed on ${machine.id}: ${result.error.message}`);
|
|
5566
6430
|
if ((result.status ?? 1) !== 0) {
|
|
5567
6431
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
5568
|
-
throw new Error(
|
|
6432
|
+
throw new Error(remotePreflightFailureMessage(machine.id, detail));
|
|
5569
6433
|
}
|
|
5570
6434
|
}
|
|
5571
6435
|
async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
@@ -5653,6 +6517,9 @@ async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
|
5653
6517
|
if (timedOut || idleTimedOut) {
|
|
5654
6518
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5655
6519
|
}
|
|
6520
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6521
|
+
return successResult(startedAt, fields);
|
|
6522
|
+
}
|
|
5656
6523
|
if (error || exitCode !== 0) {
|
|
5657
6524
|
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5658
6525
|
}
|
|
@@ -5788,6 +6655,9 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
5788
6655
|
if (timedOut || idleTimedOut) {
|
|
5789
6656
|
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5790
6657
|
}
|
|
6658
|
+
if (!error && exitCode !== 0 && codewithJsonlReconciledSuccess(spec, fields)) {
|
|
6659
|
+
return successResult(startedAt, fields);
|
|
6660
|
+
}
|
|
5791
6661
|
if (error || exitCode !== 0) {
|
|
5792
6662
|
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5793
6663
|
}
|
|
@@ -5818,322 +6688,392 @@ async function executeLoop(loop, run, opts = {}) {
|
|
|
5818
6688
|
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5819
6689
|
}
|
|
5820
6690
|
|
|
5821
|
-
// src/lib/
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
|
|
5827
|
-
|
|
5828
|
-
|
|
6691
|
+
// src/lib/health.ts
|
|
6692
|
+
import { createHash as createHash3 } from "crypto";
|
|
6693
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync6, readFileSync as readFileSync5, writeFileSync as writeFileSync3 } from "fs";
|
|
6694
|
+
import { join as join7 } from "path";
|
|
6695
|
+
var EVIDENCE_CHARS = 2000;
|
|
6696
|
+
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6697
|
+
var DEFAULT_SCAN_LIMIT = 200;
|
|
6698
|
+
var DEFAULT_SCAN_STATUSES = ["active", "paused"];
|
|
6699
|
+
var DEFAULT_MAX_FINDINGS = 100;
|
|
6700
|
+
var MIN_STALE_RUNNING_MS = 10 * 60000;
|
|
6701
|
+
var CLASSIFICATIONS = [
|
|
6702
|
+
"rate_limit",
|
|
6703
|
+
"auth",
|
|
6704
|
+
"provider_unavailable",
|
|
6705
|
+
"model_not_found",
|
|
6706
|
+
"context_length",
|
|
6707
|
+
"schema_response_format",
|
|
6708
|
+
"node_init",
|
|
6709
|
+
"preflight",
|
|
6710
|
+
"route_functional",
|
|
6711
|
+
"timeout",
|
|
6712
|
+
"sigsegv",
|
|
6713
|
+
"restart_interrupted",
|
|
6714
|
+
"skipped_previous_active",
|
|
6715
|
+
"circuit_breaker",
|
|
6716
|
+
"unknown"
|
|
5829
6717
|
];
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
6718
|
+
var RESTART_INTERRUPTED_RUN_PREFIX = "daemon restart interrupted active run";
|
|
6719
|
+
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
6720
|
+
if (!value)
|
|
6721
|
+
return;
|
|
6722
|
+
if (value.length <= limit)
|
|
6723
|
+
return value;
|
|
6724
|
+
return `${value.slice(0, limit)}
|
|
6725
|
+
[truncated ${value.length - limit} chars]`;
|
|
5833
6726
|
}
|
|
5834
|
-
function
|
|
5835
|
-
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
|
|
5839
|
-
|
|
6727
|
+
function redactedEvidence(value) {
|
|
6728
|
+
return redact(bounded(value));
|
|
6729
|
+
}
|
|
6730
|
+
function searchableText(run) {
|
|
6731
|
+
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
6732
|
+
`);
|
|
6733
|
+
}
|
|
6734
|
+
function isRecord(value) {
|
|
6735
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
6736
|
+
}
|
|
6737
|
+
function stringValue(value) {
|
|
6738
|
+
return typeof value === "string" && value.trim() ? value : undefined;
|
|
6739
|
+
}
|
|
6740
|
+
function objectField(value, key) {
|
|
6741
|
+
if (!value)
|
|
5840
6742
|
return;
|
|
5841
|
-
|
|
6743
|
+
const field = value[key];
|
|
6744
|
+
return isRecord(field) ? field : undefined;
|
|
5842
6745
|
}
|
|
5843
|
-
function
|
|
5844
|
-
|
|
5845
|
-
|
|
5846
|
-
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
|
|
5850
|
-
|
|
5851
|
-
|
|
5852
|
-
|
|
5853
|
-
|
|
5854
|
-
|
|
5855
|
-
|
|
6746
|
+
function tagsFromValue(value) {
|
|
6747
|
+
if (Array.isArray(value))
|
|
6748
|
+
return value.map((entry) => String(entry).trim()).filter(Boolean);
|
|
6749
|
+
if (typeof value === "string")
|
|
6750
|
+
return value.split(",").map((entry) => entry.trim()).filter(Boolean);
|
|
6751
|
+
return [];
|
|
6752
|
+
}
|
|
6753
|
+
function stableFingerprint(parts) {
|
|
6754
|
+
return createHash3("sha256").update(parts.join(`
|
|
6755
|
+
`)).digest("hex").slice(0, 16);
|
|
6756
|
+
}
|
|
6757
|
+
function stableScanFingerprint(parts) {
|
|
6758
|
+
return `openloops:health-scan:${stableFingerprint(parts)}`;
|
|
6759
|
+
}
|
|
6760
|
+
function safeHost(value) {
|
|
6761
|
+
if (!value)
|
|
6762
|
+
return;
|
|
6763
|
+
let host = value.trim().replace(/^[a-z]+:\/\//i, "").split(/[/:?#\s)"'\\]+/)[0] ?? "";
|
|
6764
|
+
host = host.replace(/^\[|\]$/g, "");
|
|
6765
|
+
return /^[a-z0-9.-]+$/i.test(host) ? host.toLowerCase() : undefined;
|
|
6766
|
+
}
|
|
6767
|
+
function isCursorHost(host) {
|
|
6768
|
+
return host === "cursor.sh" || Boolean(host?.endsWith(".cursor.sh"));
|
|
6769
|
+
}
|
|
6770
|
+
function providerUnavailableSummary(rawText) {
|
|
6771
|
+
const dns = /\bgetaddrinfo\s+(EAI_AGAIN|ENOTFOUND)\s+([a-z0-9.-]+)/i.exec(rawText);
|
|
6772
|
+
if (dns) {
|
|
6773
|
+
const host = safeHost(dns[2]);
|
|
6774
|
+
if (isCursorHost(host))
|
|
6775
|
+
return `provider DNS lookup failed: ${dns[1]} ${host}`;
|
|
5856
6776
|
}
|
|
5857
|
-
|
|
5858
|
-
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
5859
|
-
const accountsVersion = commandVersion("accounts");
|
|
5860
|
-
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
5861
|
-
try {
|
|
5862
|
-
const machines = listOpenMachines();
|
|
5863
|
-
const local = machines.find((machine) => machine.local);
|
|
5864
|
-
checks.push({
|
|
5865
|
-
id: "machines",
|
|
5866
|
-
status: "ok",
|
|
5867
|
-
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
5868
|
-
detail: local ? `local=${local.id}` : undefined
|
|
5869
|
-
});
|
|
5870
|
-
} catch (error) {
|
|
5871
|
-
checks.push({
|
|
5872
|
-
id: "machines",
|
|
5873
|
-
status: "warn",
|
|
5874
|
-
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
5875
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5876
|
-
});
|
|
5877
|
-
}
|
|
5878
|
-
for (const command of PROVIDER_COMMANDS) {
|
|
5879
|
-
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
5880
|
-
}
|
|
5881
|
-
const status = daemonStatus(store);
|
|
5882
|
-
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
5883
|
-
const failedRuns = store.countRuns("failed");
|
|
5884
|
-
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
5885
|
-
const deployment = buildDeploymentStatus();
|
|
5886
|
-
const schedulerState = deployment.schedulerState;
|
|
5887
|
-
checks.push({
|
|
5888
|
-
id: "scheduler-state",
|
|
5889
|
-
status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
|
|
5890
|
-
message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
|
|
5891
|
-
detail: [
|
|
5892
|
-
`route_state=${schedulerState.routeAdmission.stateStore}`,
|
|
5893
|
-
`active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
|
|
5894
|
-
`gates=${schedulerState.routeAdmission.gates.join(",")}`,
|
|
5895
|
-
`artifacts=${schedulerState.localStore.runArtifacts}`,
|
|
5896
|
-
`remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
|
|
5897
|
-
`remote_apply=${String(schedulerState.remoteStore.applySupported)}`
|
|
5898
|
-
].join(" ")
|
|
5899
|
-
});
|
|
5900
|
-
for (const loop of store.listLoops({ status: "active" })) {
|
|
5901
|
-
try {
|
|
5902
|
-
if (loop.target.type === "workflow") {
|
|
5903
|
-
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
5904
|
-
for (const step of workflowExecutionOrder(workflow)) {
|
|
5905
|
-
preflightTarget({
|
|
5906
|
-
...step.target,
|
|
5907
|
-
account: step.account ?? step.target.account,
|
|
5908
|
-
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
5909
|
-
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
5910
|
-
}
|
|
5911
|
-
} else {
|
|
5912
|
-
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
5913
|
-
}
|
|
5914
|
-
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
5915
|
-
} catch (error) {
|
|
5916
|
-
checks.push({
|
|
5917
|
-
id: `loop:${loop.id}:preflight`,
|
|
5918
|
-
status: "fail",
|
|
5919
|
-
message: `active loop target preflight failed: ${loop.name}`,
|
|
5920
|
-
detail: error instanceof Error ? error.message : String(error)
|
|
5921
|
-
});
|
|
5922
|
-
}
|
|
5923
|
-
}
|
|
5924
|
-
return {
|
|
5925
|
-
ok: checks.every((check) => check.status !== "fail"),
|
|
5926
|
-
checks
|
|
5927
|
-
};
|
|
6777
|
+
return;
|
|
5928
6778
|
}
|
|
5929
|
-
|
|
5930
|
-
|
|
5931
|
-
|
|
5932
|
-
|
|
5933
|
-
|
|
5934
|
-
|
|
5935
|
-
|
|
5936
|
-
|
|
5937
|
-
|
|
5938
|
-
if (!value)
|
|
5939
|
-
return value;
|
|
5940
|
-
if (value.length <= visible)
|
|
5941
|
-
return value;
|
|
5942
|
-
if (visible <= 0)
|
|
5943
|
-
return `[redacted ${value.length} chars]`;
|
|
5944
|
-
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
6779
|
+
function stableFailureFingerprint(run, classification, summary) {
|
|
6780
|
+
const evidence = summary ?? (classification === "provider_unavailable" ? providerUnavailableSummary(searchableText(run)) : undefined) ?? run.error ?? run.stderr ?? run.stdout ?? "";
|
|
6781
|
+
return stableFingerprint([
|
|
6782
|
+
run.loopId,
|
|
6783
|
+
classification,
|
|
6784
|
+
String(run.status),
|
|
6785
|
+
String(run.exitCode ?? ""),
|
|
6786
|
+
evidence.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6787
|
+
]);
|
|
5945
6788
|
}
|
|
5946
|
-
function
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
6789
|
+
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
6790
|
+
return stableFingerprint([
|
|
6791
|
+
loop.id,
|
|
6792
|
+
"route_functional",
|
|
6793
|
+
reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6794
|
+
]);
|
|
5951
6795
|
}
|
|
5952
|
-
function
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
}
|
|
5960
|
-
if (Array.isArray(value))
|
|
5961
|
-
return value.map((item) => redactSensitivePayload(item));
|
|
5962
|
-
if (value && typeof value === "object") {
|
|
5963
|
-
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
5964
|
-
}
|
|
5965
|
-
return value;
|
|
6796
|
+
function healthRun(run) {
|
|
6797
|
+
return {
|
|
6798
|
+
...run,
|
|
6799
|
+
error: redactedEvidence(run.error),
|
|
6800
|
+
stdout: redactedEvidence(run.stdout),
|
|
6801
|
+
stderr: redactedEvidence(run.stderr)
|
|
6802
|
+
};
|
|
5966
6803
|
}
|
|
5967
|
-
function
|
|
5968
|
-
const
|
|
5969
|
-
|
|
5970
|
-
const blocks = [];
|
|
5971
|
-
for (const [label, output] of [
|
|
5972
|
-
["stdout", value.stdout],
|
|
5973
|
-
["stderr", value.stderr]
|
|
5974
|
-
]) {
|
|
5975
|
-
if (!output)
|
|
5976
|
-
continue;
|
|
5977
|
-
blocks.push(`${indent}${label}:`);
|
|
5978
|
-
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
5979
|
-
blocks.push(`${nested}${line}`);
|
|
5980
|
-
}
|
|
5981
|
-
}
|
|
5982
|
-
return blocks;
|
|
6804
|
+
function compactText(value, limit = 500) {
|
|
6805
|
+
const text = redact(bounded(value, limit));
|
|
6806
|
+
return text?.replace(/\s+/g, " ").trim();
|
|
5983
6807
|
}
|
|
5984
|
-
function
|
|
5985
|
-
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
6808
|
+
function publicDoctorCheck(check) {
|
|
5986
6809
|
return {
|
|
5987
|
-
|
|
5988
|
-
|
|
6810
|
+
id: check.id,
|
|
6811
|
+
status: check.status,
|
|
6812
|
+
message: redact(bounded(check.message, 500)) ?? "",
|
|
6813
|
+
detail: redact(bounded(check.detail, 800))
|
|
5989
6814
|
};
|
|
5990
6815
|
}
|
|
5991
|
-
function
|
|
6816
|
+
function publicDoctorReport(report) {
|
|
5992
6817
|
return {
|
|
5993
|
-
|
|
5994
|
-
|
|
5995
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
5996
|
-
error: opts.redactError ? redact(run.error) : run.error
|
|
6818
|
+
ok: report.ok,
|
|
6819
|
+
checks: report.checks.map(publicDoctorCheck)
|
|
5997
6820
|
};
|
|
5998
6821
|
}
|
|
5999
|
-
function
|
|
6822
|
+
function includedStatusSet(statuses) {
|
|
6823
|
+
const values = statuses?.length ? statuses : DEFAULT_SCAN_STATUSES;
|
|
6824
|
+
return new Set(values);
|
|
6825
|
+
}
|
|
6826
|
+
function statusCounts(loops) {
|
|
6000
6827
|
return {
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6828
|
+
loops: loops.length,
|
|
6829
|
+
active: loops.filter((loop) => loop.status === "active").length,
|
|
6830
|
+
paused: loops.filter((loop) => loop.status === "paused").length,
|
|
6831
|
+
stopped: loops.filter((loop) => loop.status === "stopped").length,
|
|
6832
|
+
expired: loops.filter((loop) => loop.status === "expired").length
|
|
6005
6833
|
};
|
|
6006
6834
|
}
|
|
6007
|
-
function
|
|
6835
|
+
function compareLoopsForScan(left, right) {
|
|
6836
|
+
const statusOrder = left.status.localeCompare(right.status);
|
|
6837
|
+
if (statusOrder !== 0)
|
|
6838
|
+
return statusOrder;
|
|
6839
|
+
return (left.nextRunAt ?? "").localeCompare(right.nextRunAt ?? "");
|
|
6840
|
+
}
|
|
6841
|
+
function scanLoops(store, statuses, opts) {
|
|
6842
|
+
const limit = opts.limit ?? DEFAULT_SCAN_LIMIT;
|
|
6843
|
+
return statuses.flatMap((status) => store.listLoops({ includeArchived: opts.includeArchived, status, limit })).sort(compareLoopsForScan).slice(0, limit);
|
|
6844
|
+
}
|
|
6845
|
+
function healthReportForLoops(store, loops, generatedAt) {
|
|
6846
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
6847
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
6848
|
+
for (const expectation of expectations) {
|
|
6849
|
+
if (expectation.failure)
|
|
6850
|
+
classifications[expectation.failure.classification] += 1;
|
|
6851
|
+
}
|
|
6852
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
6853
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
6008
6854
|
return {
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6855
|
+
ok: unhealthy === 0,
|
|
6856
|
+
generatedAt,
|
|
6857
|
+
summary: {
|
|
6858
|
+
loops: expectations.length,
|
|
6859
|
+
healthy: expectations.length - unhealthy,
|
|
6860
|
+
unhealthy,
|
|
6861
|
+
warnings
|
|
6862
|
+
},
|
|
6863
|
+
classifications,
|
|
6864
|
+
expectations
|
|
6014
6865
|
};
|
|
6015
6866
|
}
|
|
6016
|
-
function
|
|
6017
|
-
|
|
6867
|
+
function ageMs(run, now) {
|
|
6868
|
+
const stamp = run.startedAt ?? run.createdAt ?? run.scheduledFor;
|
|
6869
|
+
const time = Date.parse(stamp);
|
|
6870
|
+
return Number.isFinite(time) ? Math.max(0, now.getTime() - time) : 0;
|
|
6018
6871
|
}
|
|
6019
|
-
function
|
|
6020
|
-
return
|
|
6872
|
+
function shortLoop(loop) {
|
|
6873
|
+
return {
|
|
6874
|
+
id: loop.id,
|
|
6875
|
+
name: loop.name,
|
|
6876
|
+
status: loop.status,
|
|
6877
|
+
nextRunAt: loop.nextRunAt,
|
|
6878
|
+
leaseMs: loop.leaseMs
|
|
6879
|
+
};
|
|
6021
6880
|
}
|
|
6022
|
-
function
|
|
6023
|
-
|
|
6881
|
+
function priorityForSeverity(severity) {
|
|
6882
|
+
if (severity === "critical")
|
|
6883
|
+
return "critical";
|
|
6884
|
+
if (severity === "high")
|
|
6885
|
+
return "high";
|
|
6886
|
+
if (severity === "low")
|
|
6887
|
+
return "low";
|
|
6888
|
+
return "medium";
|
|
6889
|
+
}
|
|
6890
|
+
function recommendedFindingTask(finding, route) {
|
|
6891
|
+
const tags = ["bug", "openloops", "loops", "loop-health", finding.kind];
|
|
6892
|
+
if (finding.classification)
|
|
6893
|
+
tags.push(finding.classification);
|
|
6894
|
+
const description = [
|
|
6895
|
+
`OpenLoops health scan found a ${finding.kind} issue.`,
|
|
6896
|
+
finding.loop ? `Loop: ${finding.loop.name} (${finding.loop.id})` : undefined,
|
|
6897
|
+
finding.run ? `Run: ${finding.run.id}` : undefined,
|
|
6898
|
+
finding.classification ? `Classification: ${finding.classification}` : undefined,
|
|
6899
|
+
finding.ageMs !== undefined ? `AgeMs: ${finding.ageMs}` : undefined,
|
|
6900
|
+
finding.staleThresholdMs !== undefined ? `StaleThresholdMs: ${finding.staleThresholdMs}` : undefined,
|
|
6901
|
+
`Fingerprint: ${finding.fingerprint}`,
|
|
6902
|
+
`Severity: ${finding.severity}`,
|
|
6903
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6904
|
+
route?.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6905
|
+
route?.provider ? `Provider: ${route.provider}` : undefined,
|
|
6906
|
+
"",
|
|
6907
|
+
finding.message,
|
|
6908
|
+
finding.run?.error ? `Error:
|
|
6909
|
+
${finding.run.error}` : undefined,
|
|
6910
|
+
finding.run?.stderr ? `Stderr:
|
|
6911
|
+
${finding.run.stderr}` : undefined
|
|
6912
|
+
].filter(Boolean).join(`
|
|
6913
|
+
|
|
6914
|
+
`);
|
|
6915
|
+
return {
|
|
6916
|
+
title: finding.title,
|
|
6917
|
+
description,
|
|
6918
|
+
priority: priorityForSeverity(finding.severity),
|
|
6919
|
+
tags,
|
|
6920
|
+
dedupeKey: finding.fingerprint,
|
|
6921
|
+
search: { query: finding.fingerprint },
|
|
6922
|
+
compatibilityFallback: {
|
|
6923
|
+
search: ["todos", "search", finding.fingerprint, "--json"],
|
|
6924
|
+
add: ["todos", "add", finding.title, "--description", description, "--tag", tags.join(","), "--priority", priorityForSeverity(finding.severity)],
|
|
6925
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
6926
|
+
},
|
|
6927
|
+
futureNativeUpsert: {
|
|
6928
|
+
command: "todos task upsert",
|
|
6929
|
+
fields: {
|
|
6930
|
+
title: finding.title,
|
|
6931
|
+
description,
|
|
6932
|
+
priority: priorityForSeverity(finding.severity),
|
|
6933
|
+
tags,
|
|
6934
|
+
dedupeKey: finding.fingerprint,
|
|
6935
|
+
routeSource: route?.source ?? "openloops",
|
|
6936
|
+
routeKind: route?.kind ?? "health_scan",
|
|
6937
|
+
routeLoopId: finding.loop?.id ?? "",
|
|
6938
|
+
routeLoopName: finding.loop?.name ?? ""
|
|
6939
|
+
}
|
|
6940
|
+
}
|
|
6941
|
+
};
|
|
6024
6942
|
}
|
|
6025
|
-
function
|
|
6943
|
+
function daemonFinding(daemon) {
|
|
6944
|
+
if (daemon.running && !daemon.stale)
|
|
6945
|
+
return;
|
|
6946
|
+
const reason = daemon.stale ? "daemon pid file is stale" : "daemon is not running";
|
|
6947
|
+
const severity = "critical";
|
|
6948
|
+
const finding = {
|
|
6949
|
+
kind: "daemon",
|
|
6950
|
+
severity,
|
|
6951
|
+
fingerprint: `openloops:health-scan:daemon:${daemon.stale ? "stale" : "not-running"}`,
|
|
6952
|
+
title: "OpenLoops daemon health issue",
|
|
6953
|
+
message: reason
|
|
6954
|
+
};
|
|
6026
6955
|
return {
|
|
6027
|
-
...
|
|
6028
|
-
|
|
6029
|
-
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
6030
|
-
error: redact(run.error)
|
|
6956
|
+
...finding,
|
|
6957
|
+
recommendedTask: recommendedFindingTask(finding, undefined)
|
|
6031
6958
|
};
|
|
6032
6959
|
}
|
|
6033
|
-
function
|
|
6034
|
-
|
|
6960
|
+
function doctorSeverity(check) {
|
|
6961
|
+
if (check.status === "fail" && check.id === "data-dir")
|
|
6962
|
+
return "critical";
|
|
6963
|
+
if (check.status === "fail")
|
|
6964
|
+
return "high";
|
|
6965
|
+
return "medium";
|
|
6035
6966
|
}
|
|
6036
|
-
function
|
|
6967
|
+
function doctorFinding(check, loop, route) {
|
|
6968
|
+
if (check.status === "ok" || check.id === "loop-runs")
|
|
6969
|
+
return;
|
|
6970
|
+
const kind = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? "preflight" : "doctor";
|
|
6971
|
+
const severity = kind === "preflight" && check.status === "fail" ? "high" : doctorSeverity(check);
|
|
6972
|
+
const fingerprint = stableScanFingerprint(["doctor", check.id, check.status, check.message, check.detail ?? ""]);
|
|
6973
|
+
const finding = {
|
|
6974
|
+
kind,
|
|
6975
|
+
severity,
|
|
6976
|
+
fingerprint,
|
|
6977
|
+
title: kind === "preflight" && loop ? `OpenLoops preflight issue - ${loop.name}` : `OpenLoops doctor issue - ${check.id}`,
|
|
6978
|
+
message: [check.status, check.message, check.detail].filter(Boolean).join(" "),
|
|
6979
|
+
loop: loop ? shortLoop(loop) : undefined,
|
|
6980
|
+
route,
|
|
6981
|
+
doctorCheck: publicDoctorCheck(check)
|
|
6982
|
+
};
|
|
6037
6983
|
return {
|
|
6038
|
-
...
|
|
6039
|
-
|
|
6984
|
+
...finding,
|
|
6985
|
+
recommendedTask: recommendedFindingTask(finding, route)
|
|
6040
6986
|
};
|
|
6041
6987
|
}
|
|
6042
|
-
function
|
|
6988
|
+
function latestRunFinding(expectation) {
|
|
6989
|
+
if (expectation.ok || !expectation.latestRun || expectation.latestRun.status === "running")
|
|
6990
|
+
return;
|
|
6991
|
+
const failure = expectation.failure;
|
|
6992
|
+
if (!failure)
|
|
6993
|
+
return;
|
|
6994
|
+
const severity = expectation.loop.status === "active" ? "high" : "medium";
|
|
6043
6995
|
return {
|
|
6044
|
-
|
|
6045
|
-
|
|
6046
|
-
|
|
6996
|
+
kind: "latest-run",
|
|
6997
|
+
severity,
|
|
6998
|
+
fingerprint: expectation.recommendedTask?.dedupeKey ?? `openloops:${expectation.loop.id}:${failure.fingerprint}`,
|
|
6999
|
+
title: expectation.recommendedTask?.title ?? `OpenLoops latest run failed - ${expectation.loop.name}`,
|
|
7000
|
+
message: expectation.check.message,
|
|
7001
|
+
loop: expectation.loop,
|
|
7002
|
+
run: expectation.latestRun,
|
|
7003
|
+
route: expectation.route,
|
|
7004
|
+
classification: failure.classification,
|
|
7005
|
+
doctorCheck: undefined,
|
|
7006
|
+
recommendedTask: expectation.recommendedTask
|
|
6047
7007
|
};
|
|
6048
7008
|
}
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
6053
|
-
var CLASSIFICATIONS = [
|
|
6054
|
-
"rate_limit",
|
|
6055
|
-
"auth",
|
|
6056
|
-
"model_not_found",
|
|
6057
|
-
"context_length",
|
|
6058
|
-
"schema_response_format",
|
|
6059
|
-
"node_init",
|
|
6060
|
-
"preflight",
|
|
6061
|
-
"route_functional",
|
|
6062
|
-
"timeout",
|
|
6063
|
-
"sigsegv",
|
|
6064
|
-
"skipped_previous_active",
|
|
6065
|
-
"circuit_breaker",
|
|
6066
|
-
"unknown"
|
|
6067
|
-
];
|
|
6068
|
-
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
6069
|
-
if (!value)
|
|
7009
|
+
function staleRunningFinding(loop, expectation, now, staleRunningMs) {
|
|
7010
|
+
const run = expectation.latestRun;
|
|
7011
|
+
if (loop.status !== "active" || run?.status !== "running")
|
|
6070
7012
|
return;
|
|
6071
|
-
|
|
6072
|
-
|
|
6073
|
-
|
|
6074
|
-
|
|
6075
|
-
}
|
|
6076
|
-
|
|
6077
|
-
|
|
7013
|
+
const threshold = Math.max(loop.leaseMs, staleRunningMs, MIN_STALE_RUNNING_MS);
|
|
7014
|
+
const age = ageMs(run, now);
|
|
7015
|
+
if (age <= threshold)
|
|
7016
|
+
return;
|
|
7017
|
+
const fingerprint = `openloops:health-scan:stale-running:${loop.id}:${run.id}`;
|
|
7018
|
+
const message = `active loop latest run is still running after ${age}ms (threshold ${threshold}ms)`;
|
|
7019
|
+
const finding = {
|
|
7020
|
+
kind: "stale-running",
|
|
7021
|
+
severity: "critical",
|
|
7022
|
+
fingerprint,
|
|
7023
|
+
title: `OpenLoops stale running run - ${loop.name}`,
|
|
7024
|
+
message,
|
|
7025
|
+
loop: shortLoop(loop),
|
|
7026
|
+
run,
|
|
7027
|
+
route: expectation.route,
|
|
7028
|
+
ageMs: age,
|
|
7029
|
+
staleThresholdMs: threshold
|
|
7030
|
+
};
|
|
7031
|
+
return {
|
|
7032
|
+
...finding,
|
|
7033
|
+
recommendedTask: recommendedFindingTask(finding, expectation.route)
|
|
7034
|
+
};
|
|
6078
7035
|
}
|
|
6079
|
-
function
|
|
6080
|
-
|
|
6081
|
-
|
|
7036
|
+
function scanStatus(findings) {
|
|
7037
|
+
if (findings.some((finding) => finding.severity === "critical"))
|
|
7038
|
+
return "critical";
|
|
7039
|
+
if (findings.length > 0)
|
|
7040
|
+
return "degraded";
|
|
7041
|
+
return "ok";
|
|
6082
7042
|
}
|
|
6083
|
-
function
|
|
6084
|
-
|
|
7043
|
+
function timestampDir(root, generatedAt) {
|
|
7044
|
+
const stamp = generatedAt.replace(/[-:]/g, "").replace(/\./g, "");
|
|
7045
|
+
return join7(root, stamp);
|
|
6085
7046
|
}
|
|
6086
|
-
function
|
|
6087
|
-
return
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6094
|
-
}
|
|
6095
|
-
|
|
6096
|
-
|
|
6097
|
-
|
|
6098
|
-
|
|
6099
|
-
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6103
|
-
|
|
6104
|
-
`)).digest("hex").slice(0, 16);
|
|
6105
|
-
}
|
|
6106
|
-
function stableFailureFingerprint(run, classification) {
|
|
6107
|
-
return stableFingerprint([
|
|
6108
|
-
run.loopId,
|
|
6109
|
-
classification,
|
|
6110
|
-
String(run.status),
|
|
6111
|
-
String(run.exitCode ?? ""),
|
|
6112
|
-
(run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6113
|
-
]);
|
|
6114
|
-
}
|
|
6115
|
-
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
6116
|
-
return stableFingerprint([
|
|
6117
|
-
loop.id,
|
|
6118
|
-
"route_functional",
|
|
6119
|
-
reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
6120
|
-
]);
|
|
6121
|
-
}
|
|
6122
|
-
function healthRun(run) {
|
|
6123
|
-
return {
|
|
6124
|
-
...run,
|
|
6125
|
-
error: redactedEvidence(run.error),
|
|
6126
|
-
stdout: redactedEvidence(run.stdout),
|
|
6127
|
-
stderr: redactedEvidence(run.stderr)
|
|
6128
|
-
};
|
|
7047
|
+
function healthScanMarkdown(scan) {
|
|
7048
|
+
return [
|
|
7049
|
+
"# OpenLoops Health Scan",
|
|
7050
|
+
"",
|
|
7051
|
+
`- status: ${scan.status}`,
|
|
7052
|
+
`- generated_at: ${scan.generatedAt}`,
|
|
7053
|
+
`- included_statuses: ${scan.includedStatuses.join(",")}`,
|
|
7054
|
+
`- loops: total=${scan.counts.loops} active=${scan.counts.active} paused=${scan.counts.paused} stopped=${scan.counts.stopped} expired=${scan.counts.expired}`,
|
|
7055
|
+
`- findings: total=${scan.counts.findings} reported=${scan.counts.reportedFindings} truncated=${scan.counts.truncatedFindings} latest_run=${scan.counts.latestRunFindings} stale_running=${scan.counts.staleRunning} daemon=${scan.counts.daemonFindings} doctor=${scan.counts.doctorFindings} preflight=${scan.counts.preflightFindings}`,
|
|
7056
|
+
scan.daemon ? `- daemon: running=${scan.daemon.running} stale=${scan.daemon.stale} pid=${scan.daemon.pid ?? "none"}` : "- daemon: not checked",
|
|
7057
|
+
scan.doctor ? `- doctor_ok: ${scan.doctor.ok}` : "- doctor: not checked",
|
|
7058
|
+
scan.selfHeals.length ? `- self_heals: ${scan.selfHeals.map((action) => `${action.kind}:${action.attempted ? action.ok ? "ok" : "failed" : "skipped"}`).join(",")}` : "- self_heals: none",
|
|
7059
|
+
"",
|
|
7060
|
+
"## Findings",
|
|
7061
|
+
scan.findings.length ? scan.findings.map((finding) => `- ${finding.severity} ${finding.kind} ${finding.fingerprint} ${finding.loop ? `${finding.loop.name}: ` : ""}${compactText(finding.message, 240) ?? ""}`).join(`
|
|
7062
|
+
`) : "None."
|
|
7063
|
+
].join(`
|
|
7064
|
+
`);
|
|
6129
7065
|
}
|
|
6130
7066
|
function classifyRunFailure(run) {
|
|
6131
7067
|
if (run.status === "succeeded" || run.status === "running")
|
|
6132
7068
|
return;
|
|
6133
|
-
const
|
|
7069
|
+
const rawText = searchableText(run);
|
|
7070
|
+
const text = rawText.toLowerCase();
|
|
6134
7071
|
let classification = "unknown";
|
|
7072
|
+
let summary;
|
|
6135
7073
|
if (run.status === "timed_out")
|
|
6136
7074
|
classification = "timeout";
|
|
7075
|
+
else if (run.status === "skipped" && run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX))
|
|
7076
|
+
classification = "restart_interrupted";
|
|
6137
7077
|
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
6138
7078
|
classification = "circuit_breaker";
|
|
6139
7079
|
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
@@ -6142,8 +7082,10 @@ function classifyRunFailure(run) {
|
|
|
6142
7082
|
classification = "preflight";
|
|
6143
7083
|
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
6144
7084
|
classification = "rate_limit";
|
|
6145
|
-
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
7085
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b|not logged in|please run \/login|login required|not authenticated|failed to authenticate/.test(text))
|
|
6146
7086
|
classification = "auth";
|
|
7087
|
+
else if (summary = providerUnavailableSummary(rawText))
|
|
7088
|
+
classification = "provider_unavailable";
|
|
6147
7089
|
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
6148
7090
|
classification = "model_not_found";
|
|
6149
7091
|
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
@@ -6156,8 +7098,9 @@ function classifyRunFailure(run) {
|
|
|
6156
7098
|
classification = "sigsegv";
|
|
6157
7099
|
return {
|
|
6158
7100
|
classification,
|
|
6159
|
-
fingerprint: stableFailureFingerprint(run, classification),
|
|
7101
|
+
fingerprint: stableFailureFingerprint(run, classification, summary),
|
|
6160
7102
|
evidence: {
|
|
7103
|
+
summary,
|
|
6161
7104
|
error: redactedEvidence(run.error),
|
|
6162
7105
|
stdout: redactedEvidence(run.stdout),
|
|
6163
7106
|
stderr: redactedEvidence(run.stderr),
|
|
@@ -6202,7 +7145,7 @@ function routeEvidenceReport(run) {
|
|
|
6202
7145
|
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
6203
7146
|
if (evidencePath && existsSync4(evidencePath)) {
|
|
6204
7147
|
try {
|
|
6205
|
-
return parseJsonObject(
|
|
7148
|
+
return parseJsonObject(readFileSync5(evidencePath, "utf8")) ?? stdoutReport;
|
|
6206
7149
|
} catch {
|
|
6207
7150
|
return stdoutReport;
|
|
6208
7151
|
}
|
|
@@ -6334,6 +7277,12 @@ function targetRoute(loop) {
|
|
|
6334
7277
|
loopName: loop.name
|
|
6335
7278
|
};
|
|
6336
7279
|
}
|
|
7280
|
+
function expectationLoop(loop) {
|
|
7281
|
+
return { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt, retryScheduledFor: loop.retryScheduledFor };
|
|
7282
|
+
}
|
|
7283
|
+
function hasPendingRetry(loop, run) {
|
|
7284
|
+
return loop.status === "active" && loop.retryScheduledFor === run.scheduledFor && run.attempt < loop.maxAttempts;
|
|
7285
|
+
}
|
|
6337
7286
|
function recommendedTask(loop, run, failure, route) {
|
|
6338
7287
|
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
6339
7288
|
const description = [
|
|
@@ -6345,6 +7294,7 @@ function recommendedTask(loop, run, failure, route) {
|
|
|
6345
7294
|
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
6346
7295
|
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
6347
7296
|
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
7297
|
+
failure.evidence.summary ? `Summary: ${failure.evidence.summary}` : undefined,
|
|
6348
7298
|
failure.evidence.error ? `Error:
|
|
6349
7299
|
${failure.evidence.error}` : undefined,
|
|
6350
7300
|
failure.evidence.stderr ? `Stderr:
|
|
@@ -6354,7 +7304,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6354
7304
|
`);
|
|
6355
7305
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6356
7306
|
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6357
|
-
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
7307
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" || failure.classification === "provider_unavailable" ? "high" : "medium";
|
|
6358
7308
|
return {
|
|
6359
7309
|
title,
|
|
6360
7310
|
description,
|
|
@@ -6388,7 +7338,7 @@ function expectationForLoop(store, loop) {
|
|
|
6388
7338
|
const route = targetRoute(loop);
|
|
6389
7339
|
if (!latestRun) {
|
|
6390
7340
|
return {
|
|
6391
|
-
loop:
|
|
7341
|
+
loop: expectationLoop(loop),
|
|
6392
7342
|
ok: true,
|
|
6393
7343
|
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
6394
7344
|
route
|
|
@@ -6397,7 +7347,7 @@ function expectationForLoop(store, loop) {
|
|
|
6397
7347
|
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
6398
7348
|
if (routeFailure) {
|
|
6399
7349
|
return {
|
|
6400
|
-
loop:
|
|
7350
|
+
loop: expectationLoop(loop),
|
|
6401
7351
|
ok: false,
|
|
6402
7352
|
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
6403
7353
|
latestRun: healthRun(latestRun),
|
|
@@ -6408,7 +7358,7 @@ function expectationForLoop(store, loop) {
|
|
|
6408
7358
|
}
|
|
6409
7359
|
if (latestRun.status === "succeeded") {
|
|
6410
7360
|
return {
|
|
6411
|
-
loop:
|
|
7361
|
+
loop: expectationLoop(loop),
|
|
6412
7362
|
ok: true,
|
|
6413
7363
|
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
6414
7364
|
latestRun: healthRun(latestRun),
|
|
@@ -6419,7 +7369,7 @@ function expectationForLoop(store, loop) {
|
|
|
6419
7369
|
if (failure?.classification === "circuit_breaker") {
|
|
6420
7370
|
if (loop.status !== "paused") {
|
|
6421
7371
|
return {
|
|
6422
|
-
loop:
|
|
7372
|
+
loop: expectationLoop(loop),
|
|
6423
7373
|
ok: true,
|
|
6424
7374
|
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
6425
7375
|
latestRun: healthRun(latestRun),
|
|
@@ -6427,7 +7377,7 @@ function expectationForLoop(store, loop) {
|
|
|
6427
7377
|
};
|
|
6428
7378
|
}
|
|
6429
7379
|
return {
|
|
6430
|
-
loop:
|
|
7380
|
+
loop: expectationLoop(loop),
|
|
6431
7381
|
ok: false,
|
|
6432
7382
|
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
6433
7383
|
latestRun: healthRun(latestRun),
|
|
@@ -6436,8 +7386,33 @@ function expectationForLoop(store, loop) {
|
|
|
6436
7386
|
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
6437
7387
|
};
|
|
6438
7388
|
}
|
|
7389
|
+
if (failure?.classification === "restart_interrupted") {
|
|
7390
|
+
return {
|
|
7391
|
+
loop: expectationLoop(loop),
|
|
7392
|
+
ok: true,
|
|
7393
|
+
check: { id: "latest-run-succeeded", status: "warn", message: latestRun.error ?? "daemon restart interrupted latest run" },
|
|
7394
|
+
latestRun: healthRun(latestRun),
|
|
7395
|
+
failure,
|
|
7396
|
+
route
|
|
7397
|
+
};
|
|
7398
|
+
}
|
|
7399
|
+
if (failure?.classification === "provider_unavailable" && hasPendingRetry(loop, latestRun)) {
|
|
7400
|
+
const message = [
|
|
7401
|
+
"provider unavailable/network failure; retry is scheduled",
|
|
7402
|
+
loop.nextRunAt ? `next attempt at ${loop.nextRunAt}` : undefined,
|
|
7403
|
+
failure.evidence.summary
|
|
7404
|
+
].filter(Boolean).join("; ");
|
|
7405
|
+
return {
|
|
7406
|
+
loop: expectationLoop(loop),
|
|
7407
|
+
ok: true,
|
|
7408
|
+
check: { id: "latest-run-succeeded", status: "warn", message },
|
|
7409
|
+
latestRun: healthRun(latestRun),
|
|
7410
|
+
failure,
|
|
7411
|
+
route
|
|
7412
|
+
};
|
|
7413
|
+
}
|
|
6439
7414
|
return {
|
|
6440
|
-
loop:
|
|
7415
|
+
loop: expectationLoop(loop),
|
|
6441
7416
|
ok: false,
|
|
6442
7417
|
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
6443
7418
|
latestRun: healthRun(latestRun),
|
|
@@ -6469,10 +7444,210 @@ function buildHealthReport(store, opts = {}) {
|
|
|
6469
7444
|
expectations
|
|
6470
7445
|
};
|
|
6471
7446
|
}
|
|
7447
|
+
function buildHealthScan(store, opts = {}) {
|
|
7448
|
+
const generatedAt = (opts.now ?? new Date).toISOString();
|
|
7449
|
+
const now = opts.now ?? new Date(generatedAt);
|
|
7450
|
+
const includeStatuses = [...includedStatusSet(opts.includeStatuses)];
|
|
7451
|
+
const loops = scanLoops(store, includeStatuses, opts);
|
|
7452
|
+
const health = healthReportForLoops(store, loops, generatedAt);
|
|
7453
|
+
const expectationsByLoopId = new Map(health.expectations.map((expectation) => [expectation.loop.id, expectation]));
|
|
7454
|
+
const loopsById = new Map(loops.map((loop) => [loop.id, loop]));
|
|
7455
|
+
const maxFindings = Math.max(0, opts.maxFindings ?? DEFAULT_MAX_FINDINGS);
|
|
7456
|
+
const allFindings = [];
|
|
7457
|
+
const pushFinding = (finding) => {
|
|
7458
|
+
if (!finding)
|
|
7459
|
+
return;
|
|
7460
|
+
allFindings.push(finding);
|
|
7461
|
+
};
|
|
7462
|
+
if (opts.daemon)
|
|
7463
|
+
pushFinding(daemonFinding(opts.daemon));
|
|
7464
|
+
if (opts.latestRun !== false) {
|
|
7465
|
+
for (const loop of loops) {
|
|
7466
|
+
const expectation = expectationsByLoopId.get(loop.id);
|
|
7467
|
+
if (!expectation)
|
|
7468
|
+
continue;
|
|
7469
|
+
pushFinding(staleRunningFinding(loop, expectation, now, opts.staleRunningMs ?? MIN_STALE_RUNNING_MS));
|
|
7470
|
+
pushFinding(latestRunFinding(expectation));
|
|
7471
|
+
}
|
|
7472
|
+
}
|
|
7473
|
+
const doctor = opts.doctor ? publicDoctorReport(opts.doctor) : undefined;
|
|
7474
|
+
if (doctor) {
|
|
7475
|
+
for (const check of doctor.checks) {
|
|
7476
|
+
const preflightLoopId = check.id.startsWith("loop:") && check.id.endsWith(":preflight") ? check.id.slice("loop:".length, -":preflight".length) : undefined;
|
|
7477
|
+
const loop = preflightLoopId ? loopsById.get(preflightLoopId) : undefined;
|
|
7478
|
+
const route = preflightLoopId ? expectationsByLoopId.get(preflightLoopId)?.route : undefined;
|
|
7479
|
+
pushFinding(doctorFinding(check, loop, route));
|
|
7480
|
+
}
|
|
7481
|
+
}
|
|
7482
|
+
const findings = allFindings.slice(0, maxFindings);
|
|
7483
|
+
const status = scanStatus(allFindings);
|
|
7484
|
+
const baseCounts = statusCounts(loops);
|
|
7485
|
+
return {
|
|
7486
|
+
ok: status === "ok",
|
|
7487
|
+
status,
|
|
7488
|
+
generatedAt,
|
|
7489
|
+
includedStatuses: includeStatuses,
|
|
7490
|
+
counts: {
|
|
7491
|
+
...baseCounts,
|
|
7492
|
+
latestRunFindings: allFindings.filter((finding) => finding.kind === "latest-run").length,
|
|
7493
|
+
staleRunning: allFindings.filter((finding) => finding.kind === "stale-running").length,
|
|
7494
|
+
daemonFindings: allFindings.filter((finding) => finding.kind === "daemon").length,
|
|
7495
|
+
doctorFindings: allFindings.filter((finding) => finding.kind === "doctor").length,
|
|
7496
|
+
preflightFindings: allFindings.filter((finding) => finding.kind === "preflight").length,
|
|
7497
|
+
findings: allFindings.length,
|
|
7498
|
+
reportedFindings: findings.length,
|
|
7499
|
+
truncatedFindings: Math.max(0, allFindings.length - findings.length)
|
|
7500
|
+
},
|
|
7501
|
+
daemon: opts.daemon ? {
|
|
7502
|
+
running: opts.daemon.running,
|
|
7503
|
+
stale: opts.daemon.stale,
|
|
7504
|
+
pid: opts.daemon.pid,
|
|
7505
|
+
host: opts.daemon.host,
|
|
7506
|
+
loops: opts.daemon.loops,
|
|
7507
|
+
runs: opts.daemon.runs,
|
|
7508
|
+
logPath: opts.daemon.logPath
|
|
7509
|
+
} : undefined,
|
|
7510
|
+
doctor,
|
|
7511
|
+
health,
|
|
7512
|
+
selfHeals: opts.selfHeals ?? [],
|
|
7513
|
+
findings
|
|
7514
|
+
};
|
|
7515
|
+
}
|
|
7516
|
+
function writeHealthScanReports(scan, opts = {}) {
|
|
7517
|
+
const root = opts.reportDir ?? join7(dataDir(), "reports", "health-scan");
|
|
7518
|
+
const dir = timestampDir(root, scan.generatedAt);
|
|
7519
|
+
mkdirSync6(dir, { recursive: true, mode: 448 });
|
|
7520
|
+
const json = join7(dir, "summary.json");
|
|
7521
|
+
const markdown = join7(dir, "report.md");
|
|
7522
|
+
const withReports = {
|
|
7523
|
+
...scan,
|
|
7524
|
+
reports: { dir, json, markdown }
|
|
7525
|
+
};
|
|
7526
|
+
writeFileSync3(json, JSON.stringify(withReports, null, 2), { mode: 384 });
|
|
7527
|
+
writeFileSync3(markdown, healthScanMarkdown(withReports), { mode: 384 });
|
|
7528
|
+
return withReports;
|
|
7529
|
+
}
|
|
7530
|
+
|
|
7531
|
+
// src/lib/doctor.ts
|
|
7532
|
+
var PROVIDER_COMMANDS = [
|
|
7533
|
+
"claude",
|
|
7534
|
+
"agent",
|
|
7535
|
+
"codewith",
|
|
7536
|
+
"aicopilot",
|
|
7537
|
+
"opencode",
|
|
7538
|
+
"codex"
|
|
7539
|
+
];
|
|
7540
|
+
function hasCommand(command) {
|
|
7541
|
+
const result = spawnSync6("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
7542
|
+
return (result.status ?? 1) === 0;
|
|
7543
|
+
}
|
|
7544
|
+
function commandVersion(command) {
|
|
7545
|
+
const result = spawnSync6(command, ["--version"], {
|
|
7546
|
+
encoding: "utf8",
|
|
7547
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
7548
|
+
});
|
|
7549
|
+
if ((result.status ?? 1) !== 0)
|
|
7550
|
+
return;
|
|
7551
|
+
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
7552
|
+
}
|
|
7553
|
+
function runDoctor(store) {
|
|
7554
|
+
const checks = [];
|
|
7555
|
+
try {
|
|
7556
|
+
const dir = ensureDataDir();
|
|
7557
|
+
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
7558
|
+
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
7559
|
+
} catch (error) {
|
|
7560
|
+
checks.push({
|
|
7561
|
+
id: "data-dir",
|
|
7562
|
+
status: "fail",
|
|
7563
|
+
message: "data directory is not writable",
|
|
7564
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7565
|
+
});
|
|
7566
|
+
}
|
|
7567
|
+
const bunVersion = commandVersion("bun");
|
|
7568
|
+
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
7569
|
+
const accountsVersion = commandVersion("accounts");
|
|
7570
|
+
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
7571
|
+
try {
|
|
7572
|
+
const machines = listOpenMachines();
|
|
7573
|
+
const local = machines.find((machine) => machine.local);
|
|
7574
|
+
checks.push({
|
|
7575
|
+
id: "machines",
|
|
7576
|
+
status: "ok",
|
|
7577
|
+
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
7578
|
+
detail: local ? `local=${local.id}` : undefined
|
|
7579
|
+
});
|
|
7580
|
+
} catch (error) {
|
|
7581
|
+
checks.push({
|
|
7582
|
+
id: "machines",
|
|
7583
|
+
status: "warn",
|
|
7584
|
+
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
7585
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7586
|
+
});
|
|
7587
|
+
}
|
|
7588
|
+
for (const command of PROVIDER_COMMANDS) {
|
|
7589
|
+
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
7590
|
+
}
|
|
7591
|
+
const status = daemonStatus(store);
|
|
7592
|
+
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
7593
|
+
const failedRuns = store.countRuns("failed");
|
|
7594
|
+
const restartInterruptedRuns = store.listRuns({ status: "skipped", limit: 1000 }).filter((run) => run.error?.startsWith(RESTART_INTERRUPTED_RUN_PREFIX)).length;
|
|
7595
|
+
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
7596
|
+
if (restartInterruptedRuns > 0) {
|
|
7597
|
+
checks.push({
|
|
7598
|
+
id: "loop-runs:restart-interrupted",
|
|
7599
|
+
status: "warn",
|
|
7600
|
+
message: `${restartInterruptedRuns} daemon restart-interrupted loop run(s) recorded`
|
|
7601
|
+
});
|
|
7602
|
+
}
|
|
7603
|
+
const deployment = buildDeploymentStatus();
|
|
7604
|
+
const schedulerState = deployment.schedulerState;
|
|
7605
|
+
checks.push({
|
|
7606
|
+
id: "scheduler-state",
|
|
7607
|
+
status: deployment.deploymentMode === "local" || deployment.controlPlane.configured ? "ok" : "warn",
|
|
7608
|
+
message: `scheduler state authority=${schedulerState.authority} local=${schedulerState.localStore.role} remote=${schedulerState.remoteStore.backend}`,
|
|
7609
|
+
detail: [
|
|
7610
|
+
`route_state=${schedulerState.routeAdmission.stateStore}`,
|
|
7611
|
+
`active_statuses=${schedulerState.routeAdmission.activeStatuses.join(",")}`,
|
|
7612
|
+
`gates=${schedulerState.routeAdmission.gates.join(",")}`,
|
|
7613
|
+
`artifacts=${schedulerState.localStore.runArtifacts}`,
|
|
7614
|
+
`remote_artifacts=${schedulerState.remoteStore.objectArtifacts}`,
|
|
7615
|
+
`remote_apply=${String(schedulerState.remoteStore.applySupported)}`
|
|
7616
|
+
].join(" ")
|
|
7617
|
+
});
|
|
7618
|
+
for (const loop of store.listLoops({ status: "active" })) {
|
|
7619
|
+
try {
|
|
7620
|
+
if (loop.target.type === "workflow") {
|
|
7621
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
7622
|
+
for (const step of workflowExecutionOrder(workflow)) {
|
|
7623
|
+
preflightTarget({
|
|
7624
|
+
...step.target,
|
|
7625
|
+
account: step.account ?? step.target.account,
|
|
7626
|
+
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
7627
|
+
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
7628
|
+
}
|
|
7629
|
+
} else {
|
|
7630
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
7631
|
+
}
|
|
7632
|
+
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
7633
|
+
} catch (error) {
|
|
7634
|
+
checks.push({
|
|
7635
|
+
id: `loop:${loop.id}:preflight`,
|
|
7636
|
+
status: "fail",
|
|
7637
|
+
message: `active loop target preflight failed: ${loop.name}`,
|
|
7638
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
7639
|
+
});
|
|
7640
|
+
}
|
|
7641
|
+
}
|
|
7642
|
+
return {
|
|
7643
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
7644
|
+
checks
|
|
7645
|
+
};
|
|
7646
|
+
}
|
|
6472
7647
|
|
|
6473
7648
|
// src/lib/migration.ts
|
|
6474
|
-
import { createHash as
|
|
6475
|
-
import { hostname as
|
|
7649
|
+
import { createHash as createHash4 } from "crypto";
|
|
7650
|
+
import { hostname as hostname3 } from "os";
|
|
6476
7651
|
var LOOPS_MIGRATION_SCHEMA = "open-loops.migration/v1";
|
|
6477
7652
|
function canonicalize(value) {
|
|
6478
7653
|
if (Array.isArray(value))
|
|
@@ -6483,7 +7658,7 @@ function canonicalize(value) {
|
|
|
6483
7658
|
return value;
|
|
6484
7659
|
}
|
|
6485
7660
|
function migrationHash(value) {
|
|
6486
|
-
return
|
|
7661
|
+
return createHash4("sha256").update(JSON.stringify(canonicalize(value))).digest("hex");
|
|
6487
7662
|
}
|
|
6488
7663
|
function pushBlocker(rows, resource, id, reason, name) {
|
|
6489
7664
|
rows.push({ resource, id, name, action: "blocked", reason });
|
|
@@ -6559,7 +7734,7 @@ function exportLoopsMigrationBundle(store, opts = {}) {
|
|
|
6559
7734
|
source: {
|
|
6560
7735
|
backend: "sqlite",
|
|
6561
7736
|
schemaVersion: rows.schemaVersion,
|
|
6562
|
-
hostname:
|
|
7737
|
+
hostname: hostname3()
|
|
6563
7738
|
},
|
|
6564
7739
|
checks: rows.checks,
|
|
6565
7740
|
importable: sanitized.blockers.length === 0,
|
|
@@ -6940,6 +8115,85 @@ async function buildSelfHostedMigrationPlan(store, opts) {
|
|
|
6940
8115
|
});
|
|
6941
8116
|
return { ...plan, importable: false };
|
|
6942
8117
|
}
|
|
8118
|
+
async function postImportBatch(fetchImpl, config, payload) {
|
|
8119
|
+
const body = await requestJson(fetchImpl, config, "/v1/import", { method: "POST", body: JSON.stringify(payload) });
|
|
8120
|
+
const imported = body.imported ?? {};
|
|
8121
|
+
return {
|
|
8122
|
+
imported: { workflows: imported.workflows ?? 0, loops: imported.loops ?? 0, runs: imported.runs ?? 0 },
|
|
8123
|
+
skippedRunning: typeof body.skippedRunning === "number" ? body.skippedRunning : 0
|
|
8124
|
+
};
|
|
8125
|
+
}
|
|
8126
|
+
async function applySelfHostedPush(store, opts) {
|
|
8127
|
+
const resolved = resolveApiConfig(opts);
|
|
8128
|
+
if (!resolved.apiUrl)
|
|
8129
|
+
throw new ValidationError("LOOPS_API_URL or --api-url is required for self-hosted push");
|
|
8130
|
+
if (!isLocalApiUrl(resolved.apiUrl) && !resolved.token) {
|
|
8131
|
+
throw new ValidationError("non-local self-hosted APIs require LOOPS_API_TOKEN or HASNA_LOOPS_API_TOKEN");
|
|
8132
|
+
}
|
|
8133
|
+
const config = { apiUrl: resolved.apiUrl, token: resolved.token };
|
|
8134
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
8135
|
+
const includeRuns = opts.includeRuns ?? true;
|
|
8136
|
+
const replace = opts.replace ?? false;
|
|
8137
|
+
const batchRows = Math.max(1, opts.batchRows ?? 200);
|
|
8138
|
+
const runBatchBytes = Math.max(64 * 1024, opts.runBatchBytes ?? 4 * 1024 * 1024);
|
|
8139
|
+
const applied = { workflows: 0, loops: 0, runs: 0 };
|
|
8140
|
+
const skipped = { runningRuns: 0, orphanRuns: 0 };
|
|
8141
|
+
let requests = 0;
|
|
8142
|
+
const base = store.exportMigrationRows({ includeRuns: false });
|
|
8143
|
+
const loopIds = new Set(base.loops.map((loop) => loop.id));
|
|
8144
|
+
for (let i = 0;i < base.workflows.length; i += batchRows) {
|
|
8145
|
+
const batch = base.workflows.slice(i, i + batchRows);
|
|
8146
|
+
const result = await postImportBatch(fetchImpl, config, { workflows: batch, replace });
|
|
8147
|
+
applied.workflows += result.imported.workflows;
|
|
8148
|
+
requests += 1;
|
|
8149
|
+
opts.onProgress?.({ phase: "workflows", sent: applied.workflows, requests });
|
|
8150
|
+
}
|
|
8151
|
+
for (let i = 0;i < base.loops.length; i += batchRows) {
|
|
8152
|
+
const batch = base.loops.slice(i, i + batchRows);
|
|
8153
|
+
const result = await postImportBatch(fetchImpl, config, { loops: batch, replace });
|
|
8154
|
+
applied.loops += result.imported.loops;
|
|
8155
|
+
requests += 1;
|
|
8156
|
+
opts.onProgress?.({ phase: "loops", sent: applied.loops, requests });
|
|
8157
|
+
}
|
|
8158
|
+
if (includeRuns) {
|
|
8159
|
+
const pageSize = 500;
|
|
8160
|
+
let pending = [];
|
|
8161
|
+
let pendingBytes = 0;
|
|
8162
|
+
const flush = async () => {
|
|
8163
|
+
if (pending.length === 0)
|
|
8164
|
+
return;
|
|
8165
|
+
const result = await postImportBatch(fetchImpl, config, { runs: pending, replace });
|
|
8166
|
+
applied.runs += result.imported.runs;
|
|
8167
|
+
skipped.runningRuns += result.skippedRunning;
|
|
8168
|
+
requests += 1;
|
|
8169
|
+
opts.onProgress?.({ phase: "runs", sent: applied.runs, requests });
|
|
8170
|
+
pending = [];
|
|
8171
|
+
pendingBytes = 0;
|
|
8172
|
+
};
|
|
8173
|
+
for (let offset = 0;; offset += pageSize) {
|
|
8174
|
+
const page = store.exportMigrationRunPage({ limit: pageSize, offset });
|
|
8175
|
+
if (page.length === 0)
|
|
8176
|
+
break;
|
|
8177
|
+
for (const run of page) {
|
|
8178
|
+
if (!loopIds.has(run.loopId)) {
|
|
8179
|
+
skipped.orphanRuns += 1;
|
|
8180
|
+
continue;
|
|
8181
|
+
}
|
|
8182
|
+
const encoded = JSON.stringify(run).length;
|
|
8183
|
+
if (pending.length > 0 && pendingBytes + encoded > runBatchBytes)
|
|
8184
|
+
await flush();
|
|
8185
|
+
pending.push(run);
|
|
8186
|
+
pendingBytes += encoded;
|
|
8187
|
+
if (pending.length >= batchRows * 4)
|
|
8188
|
+
await flush();
|
|
8189
|
+
}
|
|
8190
|
+
if (page.length < pageSize)
|
|
8191
|
+
break;
|
|
8192
|
+
}
|
|
8193
|
+
await flush();
|
|
8194
|
+
}
|
|
8195
|
+
return { ok: true, apiUrl: config.apiUrl, applied, skipped, requests };
|
|
8196
|
+
}
|
|
6943
8197
|
async function registerSelfHostedRunner(opts) {
|
|
6944
8198
|
const config = resolveApiConfig(opts);
|
|
6945
8199
|
if (!config.apiUrl)
|
|
@@ -7064,55 +8318,6 @@ ${evidence.join(`
|
|
|
7064
8318
|
import { generateObject } from "ai";
|
|
7065
8319
|
import { z } from "zod";
|
|
7066
8320
|
|
|
7067
|
-
// src/lib/run-envelope.ts
|
|
7068
|
-
var ENVELOPE_EXCERPT_CHARS = 2048;
|
|
7069
|
-
var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
|
|
7070
|
-
function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
|
|
7071
|
-
if (!text)
|
|
7072
|
-
return;
|
|
7073
|
-
const scrubbed = scrubSecrets(text);
|
|
7074
|
-
if (scrubbed.length <= limit * 2)
|
|
7075
|
-
return scrubbed;
|
|
7076
|
-
const omitted = scrubbed.length - limit * 2;
|
|
7077
|
-
return `${scrubbed.slice(0, limit)}
|
|
7078
|
-
[... ${omitted} chars omitted ...]
|
|
7079
|
-
${scrubbed.slice(-limit)}`;
|
|
7080
|
-
}
|
|
7081
|
-
function summarizeOutput(stdout, stderr) {
|
|
7082
|
-
return {
|
|
7083
|
-
stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
|
|
7084
|
-
stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
|
|
7085
|
-
stdoutExcerpt: boundedExcerpt(stdout),
|
|
7086
|
-
stderrExcerpt: boundedExcerpt(stderr)
|
|
7087
|
-
};
|
|
7088
|
-
}
|
|
7089
|
-
function summarizeExecutorResult(result) {
|
|
7090
|
-
return {
|
|
7091
|
-
...summarizeOutput(result.stdout, result.stderr),
|
|
7092
|
-
status: result.status,
|
|
7093
|
-
exitCode: result.exitCode,
|
|
7094
|
-
durationMs: result.durationMs,
|
|
7095
|
-
error: boundedExcerpt(result.error)
|
|
7096
|
-
};
|
|
7097
|
-
}
|
|
7098
|
-
function isBlockedStepRun(step) {
|
|
7099
|
-
return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
|
|
7100
|
-
}
|
|
7101
|
-
function summarizeWorkflowStepRun(step) {
|
|
7102
|
-
return {
|
|
7103
|
-
...summarizeOutput(step.stdout, step.stderr),
|
|
7104
|
-
stepId: step.stepId,
|
|
7105
|
-
status: step.status,
|
|
7106
|
-
exitCode: step.exitCode,
|
|
7107
|
-
durationMs: step.durationMs,
|
|
7108
|
-
error: boundedExcerpt(step.error),
|
|
7109
|
-
blocked: isBlockedStepRun(step)
|
|
7110
|
-
};
|
|
7111
|
-
}
|
|
7112
|
-
function workflowRunEnvelope(run, steps) {
|
|
7113
|
-
return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
|
|
7114
|
-
}
|
|
7115
|
-
|
|
7116
8321
|
// src/lib/goal/model-factory.ts
|
|
7117
8322
|
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
|
|
7118
8323
|
var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
|
|
@@ -7977,7 +9182,7 @@ var MAX_RETRY_EXPONENT = 20;
|
|
|
7977
9182
|
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
7978
9183
|
const attempt = Math.max(1, run.attempt);
|
|
7979
9184
|
const failure = classifyRunFailure(run);
|
|
7980
|
-
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
9185
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth" || failure?.classification === "provider_unavailable";
|
|
7981
9186
|
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
7982
9187
|
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
7983
9188
|
const jitter = 0.5 + random();
|
|
@@ -8104,20 +9309,21 @@ async function executeClaimedRun(deps) {
|
|
|
8104
9309
|
daemonLeaseId: deps.daemonLeaseId,
|
|
8105
9310
|
onSpawn: (pid) => deps.store.markRunPid(run.id, pid, deps.runnerId, { daemonLeaseId: deps.daemonLeaseId })
|
|
8106
9311
|
})))(deps.loop, deps.run);
|
|
9312
|
+
const finalResult = deps.finalizeResult?.(result, deps.loop, deps.run) ?? result;
|
|
8107
9313
|
deps.beforeFinalize?.(deps.loop, deps.run);
|
|
8108
9314
|
return deps.store.finalizeRun(deps.run.id, {
|
|
8109
|
-
status:
|
|
8110
|
-
finishedAt:
|
|
8111
|
-
durationMs:
|
|
8112
|
-
stdout:
|
|
8113
|
-
stderr:
|
|
8114
|
-
exitCode:
|
|
8115
|
-
error:
|
|
8116
|
-
pid:
|
|
9315
|
+
status: finalResult.status,
|
|
9316
|
+
finishedAt: finalResult.finishedAt,
|
|
9317
|
+
durationMs: finalResult.durationMs,
|
|
9318
|
+
stdout: finalResult.stdout,
|
|
9319
|
+
stderr: finalResult.stderr,
|
|
9320
|
+
exitCode: finalResult.exitCode,
|
|
9321
|
+
error: finalResult.error,
|
|
9322
|
+
pid: finalResult.pid
|
|
8117
9323
|
}, {
|
|
8118
9324
|
claimedBy: deps.runnerId,
|
|
8119
9325
|
daemonLeaseId: deps.daemonLeaseId,
|
|
8120
|
-
now: deps.now?.() ?? new Date(
|
|
9326
|
+
now: deps.now?.() ?? new Date(finalResult.finishedAt)
|
|
8121
9327
|
});
|
|
8122
9328
|
} catch (err) {
|
|
8123
9329
|
deps.onError?.(deps.loop, err);
|
|
@@ -8171,184 +9377,1068 @@ function repairWedgedTerminalSlot(deps, loop, scheduledFor, now) {
|
|
|
8171
9377
|
throw error;
|
|
8172
9378
|
}
|
|
8173
9379
|
}
|
|
8174
|
-
async function runSlot(deps, loop, scheduledFor) {
|
|
8175
|
-
const now = deps.now?.() ?? new Date;
|
|
8176
|
-
deps.beforeRun?.(loop, scheduledFor);
|
|
8177
|
-
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
8178
|
-
let skipped;
|
|
9380
|
+
async function runSlot(deps, loop, scheduledFor) {
|
|
9381
|
+
const now = deps.now?.() ?? new Date;
|
|
9382
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
9383
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
9384
|
+
let skipped;
|
|
9385
|
+
try {
|
|
9386
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
9387
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
9388
|
+
});
|
|
9389
|
+
} catch (error) {
|
|
9390
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
9391
|
+
return;
|
|
9392
|
+
throw error;
|
|
9393
|
+
}
|
|
9394
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
9395
|
+
deps.onRun?.(skipped);
|
|
9396
|
+
return skipped;
|
|
9397
|
+
}
|
|
9398
|
+
let claim;
|
|
9399
|
+
try {
|
|
9400
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
9401
|
+
} catch (error) {
|
|
9402
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
9403
|
+
return;
|
|
9404
|
+
throw error;
|
|
9405
|
+
}
|
|
9406
|
+
if (!claim) {
|
|
9407
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
9408
|
+
return;
|
|
9409
|
+
}
|
|
9410
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
9411
|
+
deps.onRun?.(claim.run);
|
|
9412
|
+
const finalRun = await executeClaimedRun({
|
|
9413
|
+
store: deps.store,
|
|
9414
|
+
runnerId: deps.runnerId,
|
|
9415
|
+
loop: claim.loop,
|
|
9416
|
+
run: claim.run,
|
|
9417
|
+
now: deps.now,
|
|
9418
|
+
execute: deps.execute,
|
|
9419
|
+
beforeFinalize: deps.beforeFinalize,
|
|
9420
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
9421
|
+
onError: deps.onError
|
|
9422
|
+
});
|
|
9423
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
|
|
9424
|
+
deps.onRun?.(finalRun);
|
|
9425
|
+
return finalRun;
|
|
9426
|
+
}
|
|
9427
|
+
function claimSlot(deps, loop, scheduledFor) {
|
|
9428
|
+
const now = deps.now?.() ?? new Date;
|
|
9429
|
+
deps.beforeRun?.(loop, scheduledFor);
|
|
9430
|
+
if (loop.overlap === "skip" && deps.store.hasRunningRun(loop.id)) {
|
|
9431
|
+
if (deps.store.hasRunningRunForSlot(loop.id, scheduledFor))
|
|
9432
|
+
return;
|
|
9433
|
+
let skipped;
|
|
9434
|
+
try {
|
|
9435
|
+
skipped = deps.store.createSkippedRun(loop, scheduledFor, "previous run still active", {
|
|
9436
|
+
daemonLeaseId: deps.daemonLeaseId
|
|
9437
|
+
});
|
|
9438
|
+
} catch (error) {
|
|
9439
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
9440
|
+
return;
|
|
9441
|
+
throw error;
|
|
9442
|
+
}
|
|
9443
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
9444
|
+
deps.onRun?.(skipped);
|
|
9445
|
+
return skipped;
|
|
9446
|
+
}
|
|
9447
|
+
let claim;
|
|
9448
|
+
try {
|
|
9449
|
+
claim = deps.store.claimRun(loop, scheduledFor, deps.runnerId, now, { daemonLeaseId: deps.daemonLeaseId });
|
|
9450
|
+
} catch (error) {
|
|
9451
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
9452
|
+
return;
|
|
9453
|
+
throw error;
|
|
9454
|
+
}
|
|
9455
|
+
if (!claim) {
|
|
9456
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
9457
|
+
return;
|
|
9458
|
+
}
|
|
9459
|
+
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
9460
|
+
deps.onRun?.(claim.run);
|
|
9461
|
+
return claim;
|
|
9462
|
+
}
|
|
9463
|
+
function recoverAndExpire(deps, now) {
|
|
9464
|
+
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
9465
|
+
const recoveredByLoop = new Map;
|
|
9466
|
+
for (const run of recovered) {
|
|
9467
|
+
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
9468
|
+
}
|
|
9469
|
+
for (const runs of recoveredByLoop.values()) {
|
|
9470
|
+
const loop = deps.store.getLoop(runs[0].loopId);
|
|
9471
|
+
if (!loop)
|
|
9472
|
+
continue;
|
|
9473
|
+
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
9474
|
+
if (retryable) {
|
|
9475
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
|
|
9476
|
+
continue;
|
|
9477
|
+
}
|
|
9478
|
+
for (const run of runs) {
|
|
9479
|
+
const current = deps.store.getLoop(run.loopId);
|
|
9480
|
+
if (current) {
|
|
9481
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
|
|
9482
|
+
}
|
|
9483
|
+
}
|
|
9484
|
+
}
|
|
9485
|
+
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
9486
|
+
return { recovered, expired };
|
|
9487
|
+
}
|
|
9488
|
+
function claimDueRuns(deps) {
|
|
9489
|
+
const now = deps.now?.() ?? new Date;
|
|
9490
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
9491
|
+
const claims = [];
|
|
9492
|
+
const claimed = [];
|
|
9493
|
+
const skipped = [];
|
|
9494
|
+
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
9495
|
+
if (maxClaims === 0)
|
|
9496
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
9497
|
+
const laneLimits = deps.laneLimits;
|
|
9498
|
+
const laneClaims = { command: 0, agent: 0 };
|
|
9499
|
+
const laneCap = (lane) => laneLimits === undefined ? Number.POSITIVE_INFINITY : Math.max(0, laneLimits[lane] ?? Number.POSITIVE_INFINITY);
|
|
9500
|
+
const laneFull = (lane) => laneClaims[lane] >= laneCap(lane);
|
|
9501
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
9502
|
+
if (claims.length >= maxClaims)
|
|
9503
|
+
break;
|
|
9504
|
+
const lane = loopLane(loop);
|
|
9505
|
+
if (laneFull(lane))
|
|
9506
|
+
continue;
|
|
9507
|
+
const plan = dueSlots(loop, now);
|
|
9508
|
+
let loopSkips = 0;
|
|
9509
|
+
for (const slot of plan.slots) {
|
|
9510
|
+
if (claims.length >= maxClaims)
|
|
9511
|
+
break;
|
|
9512
|
+
if (laneFull(lane))
|
|
9513
|
+
break;
|
|
9514
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
9515
|
+
break;
|
|
9516
|
+
const run = claimSlot(deps, loop, slot);
|
|
9517
|
+
if (!run)
|
|
9518
|
+
continue;
|
|
9519
|
+
if ("loop" in run) {
|
|
9520
|
+
claims.push(run);
|
|
9521
|
+
claimed.push(run.run);
|
|
9522
|
+
laneClaims[lane] += 1;
|
|
9523
|
+
} else if (run.status === "skipped") {
|
|
9524
|
+
skipped.push(run);
|
|
9525
|
+
loopSkips += 1;
|
|
9526
|
+
}
|
|
9527
|
+
}
|
|
9528
|
+
}
|
|
9529
|
+
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
9530
|
+
}
|
|
9531
|
+
async function tick(deps) {
|
|
9532
|
+
const now = deps.now?.() ?? new Date;
|
|
9533
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
9534
|
+
const claimed = [];
|
|
9535
|
+
const completed = [];
|
|
9536
|
+
const skipped = [];
|
|
9537
|
+
for (const loop of deps.store.dueLoops(now)) {
|
|
9538
|
+
const plan = dueSlots(loop, now);
|
|
9539
|
+
let loopSkips = 0;
|
|
9540
|
+
for (const slot of plan.slots) {
|
|
9541
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
9542
|
+
break;
|
|
9543
|
+
const run = await runSlot(deps, loop, slot);
|
|
9544
|
+
if (!run)
|
|
9545
|
+
continue;
|
|
9546
|
+
if (run.status === "running")
|
|
9547
|
+
claimed.push(run);
|
|
9548
|
+
else if (run.status === "skipped") {
|
|
9549
|
+
skipped.push(run);
|
|
9550
|
+
loopSkips += 1;
|
|
9551
|
+
} else
|
|
9552
|
+
completed.push(run);
|
|
9553
|
+
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
9554
|
+
break;
|
|
9555
|
+
}
|
|
9556
|
+
}
|
|
9557
|
+
return { claimed, completed, skipped, recovered, expired };
|
|
9558
|
+
}
|
|
9559
|
+
|
|
9560
|
+
// src/lib/cloud/mode.ts
|
|
9561
|
+
var DEPRECATED_STORAGE_MODE_ALIASES = ["remote", "hybrid", "self_hosted"];
|
|
9562
|
+
function normalizeStorageMode(value) {
|
|
9563
|
+
const normalized = value.trim().toLowerCase().replace(/-/g, "_");
|
|
9564
|
+
if (normalized === "local")
|
|
9565
|
+
return { mode: "local", deprecatedAlias: null };
|
|
9566
|
+
if (normalized === "cloud")
|
|
9567
|
+
return { mode: "cloud", deprecatedAlias: null };
|
|
9568
|
+
if (DEPRECATED_STORAGE_MODE_ALIASES.includes(normalized)) {
|
|
9569
|
+
return { mode: "cloud", deprecatedAlias: normalized };
|
|
9570
|
+
}
|
|
9571
|
+
throw new Error(`Unknown storage mode: ${value}. Use local or cloud.`);
|
|
9572
|
+
}
|
|
9573
|
+
function envToken(name) {
|
|
9574
|
+
return name.toUpperCase().replace(/-/g, "_");
|
|
9575
|
+
}
|
|
9576
|
+
|
|
9577
|
+
// src/lib/cloud/transport.ts
|
|
9578
|
+
function defaultCloudBaseUrl(name) {
|
|
9579
|
+
return `https://${name}.hasna.xyz`;
|
|
9580
|
+
}
|
|
9581
|
+
function clientTransportEnvKeys(name) {
|
|
9582
|
+
const token = envToken(name);
|
|
9583
|
+
return {
|
|
9584
|
+
modeKeys: [
|
|
9585
|
+
`HASNA_${token}_STORAGE_MODE`,
|
|
9586
|
+
`HASNA_${token}_MODE`,
|
|
9587
|
+
`${token}_STORAGE_MODE`,
|
|
9588
|
+
`${token}_MODE`
|
|
9589
|
+
],
|
|
9590
|
+
apiUrlKeys: [`HASNA_${token}_API_URL`, `${token}_API_URL`],
|
|
9591
|
+
apiKeyKeys: [
|
|
9592
|
+
`HASNA_${token}_API_KEY`,
|
|
9593
|
+
`${token}_API_KEY`,
|
|
9594
|
+
`HASNA_${token}_API_TOKEN`,
|
|
9595
|
+
`${token}_API_TOKEN`
|
|
9596
|
+
]
|
|
9597
|
+
};
|
|
9598
|
+
}
|
|
9599
|
+
function firstEnv(env, keys) {
|
|
9600
|
+
for (const key of keys) {
|
|
9601
|
+
const value = env[key]?.trim();
|
|
9602
|
+
if (value)
|
|
9603
|
+
return { key, value };
|
|
9604
|
+
}
|
|
9605
|
+
return null;
|
|
9606
|
+
}
|
|
9607
|
+
function toV1BaseUrl(apiUrl) {
|
|
9608
|
+
const url = new URL(apiUrl);
|
|
9609
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
9610
|
+
throw new Error("API URL must use http or https.");
|
|
9611
|
+
}
|
|
9612
|
+
let path = url.pathname.replace(/\/+$/, "");
|
|
9613
|
+
if (path.endsWith("/v1"))
|
|
9614
|
+
path = path.slice(0, -"/v1".length);
|
|
9615
|
+
url.pathname = `${path}/v1`;
|
|
9616
|
+
url.search = "";
|
|
9617
|
+
url.hash = "";
|
|
9618
|
+
return url.toString().replace(/\/+$/, "");
|
|
9619
|
+
}
|
|
9620
|
+
function resolveClientTransport(name, env = process.env) {
|
|
9621
|
+
const keys = clientTransportEnvKeys(name);
|
|
9622
|
+
const modeHit = firstEnv(env, keys.modeKeys);
|
|
9623
|
+
const urlHit = firstEnv(env, keys.apiUrlKeys);
|
|
9624
|
+
const keyHit = firstEnv(env, keys.apiKeyKeys);
|
|
9625
|
+
let mode = "local";
|
|
9626
|
+
let deprecatedAlias = null;
|
|
9627
|
+
let modeSource = "default";
|
|
9628
|
+
const warnings = [];
|
|
9629
|
+
if (modeHit) {
|
|
9630
|
+
const normalized = normalizeStorageMode(modeHit.value);
|
|
9631
|
+
mode = normalized.mode;
|
|
9632
|
+
deprecatedAlias = normalized.deprecatedAlias;
|
|
9633
|
+
modeSource = modeHit.key;
|
|
9634
|
+
if (deprecatedAlias) {
|
|
9635
|
+
warnings.push(`Deprecated mode '${deprecatedAlias}' from ${modeHit.key} is treated as 'cloud'. Prefer ${keys.modeKeys[0]}=cloud.`);
|
|
9636
|
+
}
|
|
9637
|
+
}
|
|
9638
|
+
if (mode === "local") {
|
|
9639
|
+
return {
|
|
9640
|
+
transport: "local",
|
|
9641
|
+
mode,
|
|
9642
|
+
deprecatedAlias,
|
|
9643
|
+
modeSource,
|
|
9644
|
+
baseUrl: null,
|
|
9645
|
+
apiUrlSource: null,
|
|
9646
|
+
apiKeyPresent: Boolean(keyHit),
|
|
9647
|
+
apiKeySource: keyHit ? keyHit.key : null,
|
|
9648
|
+
misconfigured: false,
|
|
9649
|
+
warning: warnings.length > 0 ? warnings.join(" ") : null
|
|
9650
|
+
};
|
|
9651
|
+
}
|
|
9652
|
+
if (!keyHit) {
|
|
9653
|
+
warnings.push(`${modeSource}=cloud but no API key is set (${keys.apiKeyKeys[0]}). Refusing to route to cloud; using local store. Set ${keys.apiKeyKeys[0]} to enable the cloud client.`);
|
|
9654
|
+
return {
|
|
9655
|
+
transport: "local",
|
|
9656
|
+
mode,
|
|
9657
|
+
deprecatedAlias,
|
|
9658
|
+
modeSource,
|
|
9659
|
+
baseUrl: null,
|
|
9660
|
+
apiUrlSource: null,
|
|
9661
|
+
apiKeyPresent: false,
|
|
9662
|
+
apiKeySource: null,
|
|
9663
|
+
misconfigured: true,
|
|
9664
|
+
warning: warnings.join(" ")
|
|
9665
|
+
};
|
|
9666
|
+
}
|
|
9667
|
+
const rawUrl = urlHit?.value ?? defaultCloudBaseUrl(name);
|
|
9668
|
+
const apiUrlSource = urlHit ? urlHit.key : "default";
|
|
9669
|
+
let baseUrl;
|
|
9670
|
+
try {
|
|
9671
|
+
baseUrl = toV1BaseUrl(rawUrl);
|
|
9672
|
+
} catch (error) {
|
|
9673
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
9674
|
+
warnings.push(`Invalid API URL from ${apiUrlSource}: ${message}. Using local store.`);
|
|
9675
|
+
return {
|
|
9676
|
+
transport: "local",
|
|
9677
|
+
mode,
|
|
9678
|
+
deprecatedAlias,
|
|
9679
|
+
modeSource,
|
|
9680
|
+
baseUrl: null,
|
|
9681
|
+
apiUrlSource: null,
|
|
9682
|
+
apiKeyPresent: true,
|
|
9683
|
+
apiKeySource: keyHit.key,
|
|
9684
|
+
misconfigured: true,
|
|
9685
|
+
warning: warnings.join(" ")
|
|
9686
|
+
};
|
|
9687
|
+
}
|
|
9688
|
+
return {
|
|
9689
|
+
transport: "cloud-http",
|
|
9690
|
+
mode,
|
|
9691
|
+
deprecatedAlias,
|
|
9692
|
+
modeSource,
|
|
9693
|
+
baseUrl,
|
|
9694
|
+
apiUrlSource,
|
|
9695
|
+
apiKeyPresent: true,
|
|
9696
|
+
apiKeySource: keyHit.key,
|
|
9697
|
+
misconfigured: false,
|
|
9698
|
+
warning: warnings.length > 0 ? warnings.join(" ") : null
|
|
9699
|
+
};
|
|
9700
|
+
}
|
|
9701
|
+
|
|
9702
|
+
class HasnaHttpError extends Error {
|
|
9703
|
+
status;
|
|
9704
|
+
method;
|
|
9705
|
+
path;
|
|
9706
|
+
body;
|
|
9707
|
+
constructor(method, path, status, body) {
|
|
9708
|
+
super(`Hasna cloud request failed: ${method} ${path} -> ${status}`);
|
|
9709
|
+
this.name = "HasnaHttpError";
|
|
9710
|
+
this.status = status;
|
|
9711
|
+
this.method = method;
|
|
9712
|
+
this.path = path;
|
|
9713
|
+
this.body = body;
|
|
9714
|
+
}
|
|
9715
|
+
}
|
|
9716
|
+
var DEFAULT_RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504];
|
|
9717
|
+
var IDEMPOTENT_METHODS = new Set(["GET", "HEAD", "PUT", "DELETE", "OPTIONS"]);
|
|
9718
|
+
function appendQuery(path, query) {
|
|
9719
|
+
if (!query)
|
|
9720
|
+
return path;
|
|
9721
|
+
const params = query instanceof URLSearchParams ? query : new URLSearchParams;
|
|
9722
|
+
if (!(query instanceof URLSearchParams)) {
|
|
9723
|
+
for (const [key, value] of Object.entries(query)) {
|
|
9724
|
+
if (value === null || value === undefined)
|
|
9725
|
+
continue;
|
|
9726
|
+
if (Array.isArray(value)) {
|
|
9727
|
+
for (const v of value)
|
|
9728
|
+
params.append(key, String(v));
|
|
9729
|
+
} else {
|
|
9730
|
+
params.append(key, String(value));
|
|
9731
|
+
}
|
|
9732
|
+
}
|
|
9733
|
+
}
|
|
9734
|
+
const qs = params.toString();
|
|
9735
|
+
if (!qs)
|
|
9736
|
+
return path;
|
|
9737
|
+
return `${path}${path.includes("?") ? "&" : "?"}${qs}`;
|
|
9738
|
+
}
|
|
9739
|
+
var defaultSleep = (ms) => new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
9740
|
+
function createHasnaHttpTransport(options) {
|
|
9741
|
+
const fetchImpl = options.fetchImpl ?? ((input, init) => fetch(input, init));
|
|
9742
|
+
const base = options.baseUrl.replace(/\/+$/, "");
|
|
9743
|
+
const timeoutMs = options.timeoutMs ?? 30000;
|
|
9744
|
+
const sleep = options.sleepImpl ?? defaultSleep;
|
|
9745
|
+
const defaultRetry = options.retry;
|
|
9746
|
+
function resolveRetry(callRetry) {
|
|
9747
|
+
const chosen = callRetry !== undefined ? callRetry : defaultRetry;
|
|
9748
|
+
if (chosen === false)
|
|
9749
|
+
return null;
|
|
9750
|
+
const r = chosen ?? {};
|
|
9751
|
+
return {
|
|
9752
|
+
retries: r.retries ?? 2,
|
|
9753
|
+
baseDelayMs: r.baseDelayMs ?? 200,
|
|
9754
|
+
maxDelayMs: r.maxDelayMs ?? 2000,
|
|
9755
|
+
retryStatuses: r.retryStatuses ?? [...DEFAULT_RETRY_STATUSES]
|
|
9756
|
+
};
|
|
9757
|
+
}
|
|
9758
|
+
async function once2(method, rel, url, body, opts) {
|
|
9759
|
+
const headers = {
|
|
9760
|
+
"x-api-key": options.apiKey,
|
|
9761
|
+
Authorization: `Bearer ${options.apiKey}`,
|
|
9762
|
+
Accept: "application/json",
|
|
9763
|
+
...options.headers ?? {},
|
|
9764
|
+
...opts.headers ?? {}
|
|
9765
|
+
};
|
|
9766
|
+
if (opts.idempotencyKey)
|
|
9767
|
+
headers["Idempotency-Key"] = opts.idempotencyKey;
|
|
9768
|
+
const init = { method, headers };
|
|
9769
|
+
if (body !== undefined) {
|
|
9770
|
+
headers["Content-Type"] = "application/json";
|
|
9771
|
+
init.body = JSON.stringify(body);
|
|
9772
|
+
}
|
|
9773
|
+
const controller = new AbortController;
|
|
9774
|
+
const onAbort = () => controller.abort();
|
|
9775
|
+
if (opts.signal) {
|
|
9776
|
+
if (opts.signal.aborted)
|
|
9777
|
+
controller.abort();
|
|
9778
|
+
else
|
|
9779
|
+
opts.signal.addEventListener("abort", onAbort, { once: true });
|
|
9780
|
+
}
|
|
9781
|
+
const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? timeoutMs);
|
|
9782
|
+
init.signal = controller.signal;
|
|
9783
|
+
let response;
|
|
9784
|
+
try {
|
|
9785
|
+
response = await fetchImpl(url, init);
|
|
9786
|
+
} catch (error) {
|
|
9787
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
9788
|
+
if (opts.signal?.aborted)
|
|
9789
|
+
return { ok: false, retryable: false, error: err };
|
|
9790
|
+
return { ok: false, retryable: true, error: err };
|
|
9791
|
+
} finally {
|
|
9792
|
+
clearTimeout(timer);
|
|
9793
|
+
if (opts.signal)
|
|
9794
|
+
opts.signal.removeEventListener("abort", onAbort);
|
|
9795
|
+
}
|
|
9796
|
+
const text = await response.text();
|
|
9797
|
+
let parsed = undefined;
|
|
9798
|
+
if (text.length > 0) {
|
|
9799
|
+
try {
|
|
9800
|
+
parsed = JSON.parse(text);
|
|
9801
|
+
} catch {
|
|
9802
|
+
parsed = text;
|
|
9803
|
+
}
|
|
9804
|
+
}
|
|
9805
|
+
if (!response.ok) {
|
|
9806
|
+
const retry = resolveRetry(opts.retry);
|
|
9807
|
+
const retryable = retry ? retry.retryStatuses.includes(response.status) : false;
|
|
9808
|
+
return { ok: false, retryable, error: new HasnaHttpError(method, rel, response.status, parsed) };
|
|
9809
|
+
}
|
|
9810
|
+
return { ok: true, value: parsed };
|
|
9811
|
+
}
|
|
9812
|
+
async function request(method, path, body, opts = {}) {
|
|
9813
|
+
const upper = method.toUpperCase();
|
|
9814
|
+
const rel = appendQuery(path.startsWith("/") ? path : `/${path}`, opts.query);
|
|
9815
|
+
const url = `${base}${rel}`;
|
|
9816
|
+
const retry = resolveRetry(opts.retry);
|
|
9817
|
+
const methodRetryable = IDEMPOTENT_METHODS.has(upper) || Boolean(opts.idempotencyKey);
|
|
9818
|
+
const maxAttempts = retry && methodRetryable ? retry.retries + 1 : 1;
|
|
9819
|
+
let last = null;
|
|
9820
|
+
for (let attempt = 1;attempt <= maxAttempts; attempt++) {
|
|
9821
|
+
const result = await once2(upper, rel, url, body, opts);
|
|
9822
|
+
if (result.ok)
|
|
9823
|
+
return result.value;
|
|
9824
|
+
last = result;
|
|
9825
|
+
const canRetry = retry !== null && methodRetryable && result.retryable && attempt < maxAttempts;
|
|
9826
|
+
if (!canRetry)
|
|
9827
|
+
break;
|
|
9828
|
+
const backoff = Math.min(retry.maxDelayMs, retry.baseDelayMs * 2 ** (attempt - 1));
|
|
9829
|
+
const jitter = Math.floor(Math.random() * (backoff / 2 + 1));
|
|
9830
|
+
await sleep(backoff + jitter);
|
|
9831
|
+
}
|
|
9832
|
+
throw last.error;
|
|
9833
|
+
}
|
|
9834
|
+
return {
|
|
9835
|
+
baseUrl: base,
|
|
9836
|
+
request,
|
|
9837
|
+
get: (path, opts) => request("GET", path, undefined, opts),
|
|
9838
|
+
post: (path, body, opts) => request("POST", path, body, opts),
|
|
9839
|
+
put: (path, body, opts) => request("PUT", path, body, opts),
|
|
9840
|
+
patch: (path, body, opts) => request("PATCH", path, body, opts),
|
|
9841
|
+
del: (path, body, opts) => request("DELETE", path, body, opts)
|
|
9842
|
+
};
|
|
9843
|
+
}
|
|
9844
|
+
function createClientTransport(name, env = process.env, overrides) {
|
|
9845
|
+
const resolution = resolveClientTransport(name, env);
|
|
9846
|
+
if (resolution.misconfigured) {
|
|
9847
|
+
throw new Error(resolution.warning ?? `Client for '${name}' is misconfigured for cloud mode.`);
|
|
9848
|
+
}
|
|
9849
|
+
if (resolution.transport === "local" || !resolution.baseUrl) {
|
|
9850
|
+
return { transport: "local", client: null, resolution };
|
|
9851
|
+
}
|
|
9852
|
+
const keys = clientTransportEnvKeys(name);
|
|
9853
|
+
const apiKey = firstEnv(env, keys.apiKeyKeys)?.value;
|
|
9854
|
+
if (!apiKey) {
|
|
9855
|
+
throw new Error(`Client for '${name}' resolved to cloud-http without an API key.`);
|
|
9856
|
+
}
|
|
9857
|
+
return {
|
|
9858
|
+
transport: "cloud-http",
|
|
9859
|
+
client: createHasnaHttpTransport({
|
|
9860
|
+
name,
|
|
9861
|
+
baseUrl: resolution.baseUrl,
|
|
9862
|
+
apiKey,
|
|
9863
|
+
...overrides?.fetchImpl ? { fetchImpl: overrides.fetchImpl } : {},
|
|
9864
|
+
...overrides?.headers ? { headers: overrides.headers } : {},
|
|
9865
|
+
...overrides?.timeoutMs ? { timeoutMs: overrides.timeoutMs } : {},
|
|
9866
|
+
...overrides?.retry !== undefined ? { retry: overrides.retry } : {},
|
|
9867
|
+
...overrides?.sleepImpl ? { sleepImpl: overrides.sleepImpl } : {}
|
|
9868
|
+
}),
|
|
9869
|
+
resolution
|
|
9870
|
+
};
|
|
9871
|
+
}
|
|
9872
|
+
|
|
9873
|
+
// src/lib/cloud/storage.ts
|
|
9874
|
+
function resourcePath(resource) {
|
|
9875
|
+
const trimmed = resource.replace(/^\/+|\/+$/g, "");
|
|
9876
|
+
if (!trimmed)
|
|
9877
|
+
throw new Error("resource must be a non-empty path segment");
|
|
9878
|
+
return `/${trimmed}`;
|
|
9879
|
+
}
|
|
9880
|
+
function entityPath(resource, id) {
|
|
9881
|
+
if (id === undefined || id === null || `${id}`.length === 0) {
|
|
9882
|
+
throw new Error("id must be a non-empty string");
|
|
9883
|
+
}
|
|
9884
|
+
return `${resourcePath(resource)}/${encodeURIComponent(String(id))}`;
|
|
9885
|
+
}
|
|
9886
|
+
function newIdempotencyKey() {
|
|
9887
|
+
const g = globalThis;
|
|
9888
|
+
if (g.crypto?.randomUUID)
|
|
9889
|
+
return g.crypto.randomUUID();
|
|
9890
|
+
return `idmp_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 12)}`;
|
|
9891
|
+
}
|
|
9892
|
+
function extractItems(raw) {
|
|
9893
|
+
if (Array.isArray(raw))
|
|
9894
|
+
return raw;
|
|
9895
|
+
if (raw && typeof raw === "object") {
|
|
9896
|
+
const obj = raw;
|
|
9897
|
+
for (const key of ["items", "data", "results", "rows", "records"]) {
|
|
9898
|
+
if (Array.isArray(obj[key]))
|
|
9899
|
+
return obj[key];
|
|
9900
|
+
}
|
|
9901
|
+
}
|
|
9902
|
+
return [];
|
|
9903
|
+
}
|
|
9904
|
+
function extractTotal(raw) {
|
|
9905
|
+
if (raw && typeof raw === "object") {
|
|
9906
|
+
const obj = raw;
|
|
9907
|
+
for (const key of ["total", "count", "totalCount", "total_count"]) {
|
|
9908
|
+
if (typeof obj[key] === "number")
|
|
9909
|
+
return obj[key];
|
|
9910
|
+
}
|
|
9911
|
+
}
|
|
9912
|
+
return null;
|
|
9913
|
+
}
|
|
9914
|
+
function extractCursor(raw) {
|
|
9915
|
+
if (raw && typeof raw === "object") {
|
|
9916
|
+
const obj = raw;
|
|
9917
|
+
for (const key of ["cursor", "nextCursor", "next_cursor", "next"]) {
|
|
9918
|
+
if (typeof obj[key] === "string")
|
|
9919
|
+
return obj[key];
|
|
9920
|
+
}
|
|
9921
|
+
}
|
|
9922
|
+
return null;
|
|
9923
|
+
}
|
|
9924
|
+
function createHasnaStorageClient(name, transport) {
|
|
9925
|
+
return {
|
|
9926
|
+
name,
|
|
9927
|
+
baseUrl: transport.baseUrl,
|
|
9928
|
+
transport,
|
|
9929
|
+
async list(resource, options = {}) {
|
|
9930
|
+
const raw = await transport.get(resourcePath(resource), options);
|
|
9931
|
+
return {
|
|
9932
|
+
items: extractItems(raw),
|
|
9933
|
+
total: extractTotal(raw),
|
|
9934
|
+
cursor: extractCursor(raw),
|
|
9935
|
+
raw
|
|
9936
|
+
};
|
|
9937
|
+
},
|
|
9938
|
+
async get(resource, id, options = {}) {
|
|
9939
|
+
try {
|
|
9940
|
+
return await transport.get(entityPath(resource, id), options);
|
|
9941
|
+
} catch (error) {
|
|
9942
|
+
if (error instanceof HasnaHttpError && error.status === 404)
|
|
9943
|
+
return null;
|
|
9944
|
+
throw error;
|
|
9945
|
+
}
|
|
9946
|
+
},
|
|
9947
|
+
async create(resource, body, options = {}) {
|
|
9948
|
+
const { idempotencyKey, ...rest } = options;
|
|
9949
|
+
return transport.post(resourcePath(resource), body, {
|
|
9950
|
+
...rest,
|
|
9951
|
+
idempotencyKey: idempotencyKey ?? newIdempotencyKey()
|
|
9952
|
+
});
|
|
9953
|
+
},
|
|
9954
|
+
async update(resource, id, patch, options = {}) {
|
|
9955
|
+
const { method = "PATCH", idempotencyKey, ...rest } = options;
|
|
9956
|
+
const call = method === "PUT" ? transport.put : transport.patch;
|
|
9957
|
+
return call(entityPath(resource, id), patch, { ...rest, ...idempotencyKey ? { idempotencyKey } : {} });
|
|
9958
|
+
},
|
|
9959
|
+
async delete(resource, id, options = {}) {
|
|
9960
|
+
try {
|
|
9961
|
+
await transport.del(entityPath(resource, id), undefined, options);
|
|
9962
|
+
} catch (error) {
|
|
9963
|
+
if (error instanceof HasnaHttpError && error.status === 404)
|
|
9964
|
+
return;
|
|
9965
|
+
throw error;
|
|
9966
|
+
}
|
|
9967
|
+
}
|
|
9968
|
+
};
|
|
9969
|
+
}
|
|
9970
|
+
|
|
9971
|
+
// src/lib/cloud/resolve.ts
|
|
9972
|
+
function firstValue(env, keys) {
|
|
9973
|
+
for (const key of keys) {
|
|
9974
|
+
const value = env[key]?.trim();
|
|
9975
|
+
if (value)
|
|
9976
|
+
return value;
|
|
9977
|
+
}
|
|
9978
|
+
return;
|
|
9979
|
+
}
|
|
9980
|
+
function resolveCloudStorage(name, env = process.env) {
|
|
9981
|
+
const token = envToken(name);
|
|
9982
|
+
const modeKeys = [`HASNA_${token}_STORAGE_MODE`, `HASNA_${token}_MODE`, `${token}_STORAGE_MODE`, `${token}_MODE`];
|
|
9983
|
+
const apiUrlKeys = [`HASNA_${token}_API_URL`, `${token}_API_URL`];
|
|
9984
|
+
const apiKeyKeys = [
|
|
9985
|
+
`HASNA_${token}_API_KEY`,
|
|
9986
|
+
`${token}_API_KEY`,
|
|
9987
|
+
`HASNA_${token}_API_TOKEN`,
|
|
9988
|
+
`${token}_API_TOKEN`
|
|
9989
|
+
];
|
|
9990
|
+
const explicitMode = firstValue(env, modeKeys);
|
|
9991
|
+
if (explicitMode && normalizeStorageMode(explicitMode).mode === "local") {
|
|
9992
|
+
return { transport: "local", client: null };
|
|
9993
|
+
}
|
|
9994
|
+
const apiUrl = firstValue(env, apiUrlKeys);
|
|
9995
|
+
const apiKey = firstValue(env, apiKeyKeys);
|
|
9996
|
+
if (!apiUrl || !apiKey) {
|
|
9997
|
+
return { transport: "local", client: null };
|
|
9998
|
+
}
|
|
9999
|
+
const cloudEnv = { ...env, [`HASNA_${token}_STORAGE_MODE`]: "self_hosted" };
|
|
10000
|
+
const wired = createClientTransport(name, cloudEnv);
|
|
10001
|
+
if (wired.transport !== "cloud-http") {
|
|
10002
|
+
return { transport: "local", client: null };
|
|
10003
|
+
}
|
|
10004
|
+
return {
|
|
10005
|
+
transport: "cloud-http",
|
|
10006
|
+
client: createHasnaStorageClient(name, wired.client),
|
|
10007
|
+
baseUrl: wired.client.baseUrl
|
|
10008
|
+
};
|
|
10009
|
+
}
|
|
10010
|
+
|
|
10011
|
+
// src/lib/store/index.ts
|
|
10012
|
+
class CloudUnsupportedError extends Error {
|
|
10013
|
+
constructor(operation) {
|
|
10014
|
+
super(`operation not supported over the hosted OpenLoops API: ${operation}. ` + `Run it on a machine using local storage, or unset HASNA_LOOPS_API_URL/HASNA_LOOPS_API_KEY.`);
|
|
10015
|
+
this.name = "CloudUnsupportedError";
|
|
10016
|
+
}
|
|
10017
|
+
}
|
|
10018
|
+
|
|
10019
|
+
class LocalStore {
|
|
10020
|
+
transport = "local";
|
|
10021
|
+
store;
|
|
10022
|
+
constructor(store = new Store) {
|
|
10023
|
+
this.store = store;
|
|
10024
|
+
}
|
|
10025
|
+
get raw() {
|
|
10026
|
+
return this.store;
|
|
10027
|
+
}
|
|
10028
|
+
async close() {
|
|
10029
|
+
this.store.close();
|
|
10030
|
+
}
|
|
10031
|
+
async createLoop(input, from) {
|
|
10032
|
+
return from ? this.store.createLoop(input, from) : this.store.createLoop(input);
|
|
10033
|
+
}
|
|
10034
|
+
async getLoop(id) {
|
|
10035
|
+
return this.store.getLoop(id);
|
|
10036
|
+
}
|
|
10037
|
+
async findLoopByName(name) {
|
|
10038
|
+
return this.store.findLoopByName(name);
|
|
10039
|
+
}
|
|
10040
|
+
async requireLoop(idOrName) {
|
|
10041
|
+
return this.store.requireLoop(idOrName);
|
|
10042
|
+
}
|
|
10043
|
+
async requireUniqueLoop(idOrName) {
|
|
10044
|
+
return this.store.requireUniqueLoop(idOrName);
|
|
10045
|
+
}
|
|
10046
|
+
async listLoops(opts = {}) {
|
|
10047
|
+
return this.store.listLoops(opts);
|
|
10048
|
+
}
|
|
10049
|
+
async countLoops(status, opts = {}) {
|
|
10050
|
+
return this.store.countLoops(status, opts);
|
|
10051
|
+
}
|
|
10052
|
+
async updateLoop(id, patch) {
|
|
10053
|
+
return this.store.updateLoop(id, patch);
|
|
10054
|
+
}
|
|
10055
|
+
async renameLoop(id, name) {
|
|
10056
|
+
return this.store.renameLoop(id, name);
|
|
10057
|
+
}
|
|
10058
|
+
async archiveLoop(idOrName) {
|
|
10059
|
+
return this.store.archiveLoop(idOrName);
|
|
10060
|
+
}
|
|
10061
|
+
async unarchiveLoop(idOrName) {
|
|
10062
|
+
return this.store.unarchiveLoop(idOrName);
|
|
10063
|
+
}
|
|
10064
|
+
async deleteLoop(idOrName) {
|
|
10065
|
+
return this.store.deleteLoop(idOrName);
|
|
10066
|
+
}
|
|
10067
|
+
async createWorkflow(input) {
|
|
10068
|
+
return this.store.createWorkflow(input);
|
|
10069
|
+
}
|
|
10070
|
+
async getWorkflow(id) {
|
|
10071
|
+
return this.store.getWorkflow(id);
|
|
10072
|
+
}
|
|
10073
|
+
async findWorkflowByName(name) {
|
|
10074
|
+
return this.store.findWorkflowByName(name);
|
|
10075
|
+
}
|
|
10076
|
+
async requireWorkflow(idOrName) {
|
|
10077
|
+
return this.store.requireWorkflow(idOrName);
|
|
10078
|
+
}
|
|
10079
|
+
async listWorkflows(opts = {}) {
|
|
10080
|
+
return this.store.listWorkflows(opts);
|
|
10081
|
+
}
|
|
10082
|
+
async countWorkflows(opts = {}) {
|
|
10083
|
+
return this.store.countWorkflows(opts);
|
|
10084
|
+
}
|
|
10085
|
+
async archiveWorkflow(idOrName) {
|
|
10086
|
+
return this.store.archiveWorkflow(idOrName);
|
|
10087
|
+
}
|
|
10088
|
+
async getWorkflowRun(id) {
|
|
10089
|
+
return this.store.getWorkflowRun(id);
|
|
10090
|
+
}
|
|
10091
|
+
async requireWorkflowRun(id) {
|
|
10092
|
+
return this.store.requireWorkflowRun(id);
|
|
10093
|
+
}
|
|
10094
|
+
async listWorkflowRuns(opts = {}) {
|
|
10095
|
+
return this.store.listWorkflowRuns(opts);
|
|
10096
|
+
}
|
|
10097
|
+
async listWorkflowStepRuns(workflowRunId) {
|
|
10098
|
+
return this.store.listWorkflowStepRuns(workflowRunId);
|
|
10099
|
+
}
|
|
10100
|
+
async listWorkflowEvents(workflowRunId, limit) {
|
|
10101
|
+
return limit === undefined ? this.store.listWorkflowEvents(workflowRunId) : this.store.listWorkflowEvents(workflowRunId, limit);
|
|
10102
|
+
}
|
|
10103
|
+
async recoverWorkflowRun(workflowRunId, reason) {
|
|
10104
|
+
return reason === undefined ? this.store.recoverWorkflowRun(workflowRunId) : this.store.recoverWorkflowRun(workflowRunId, reason);
|
|
10105
|
+
}
|
|
10106
|
+
async cancelWorkflowRun(workflowRunId, reason) {
|
|
10107
|
+
return reason === undefined ? this.store.cancelWorkflowRun(workflowRunId) : this.store.cancelWorkflowRun(workflowRunId, reason);
|
|
10108
|
+
}
|
|
10109
|
+
async listWorkflowWorkItems(opts = {}) {
|
|
10110
|
+
return this.store.listWorkflowWorkItems(opts);
|
|
10111
|
+
}
|
|
10112
|
+
async getWorkflowWorkItem(id) {
|
|
10113
|
+
return this.store.getWorkflowWorkItem(id);
|
|
10114
|
+
}
|
|
10115
|
+
async requeueWorkflowWorkItem(id, patch = {}) {
|
|
10116
|
+
return this.store.requeueWorkflowWorkItem(id, patch);
|
|
10117
|
+
}
|
|
10118
|
+
async listWorkflowInvocations(opts = {}) {
|
|
10119
|
+
return this.store.listWorkflowInvocations(opts);
|
|
10120
|
+
}
|
|
10121
|
+
async getWorkflowInvocation(id) {
|
|
10122
|
+
return this.store.getWorkflowInvocation(id);
|
|
10123
|
+
}
|
|
10124
|
+
async getGoal(id) {
|
|
10125
|
+
return this.store.getGoal(id);
|
|
10126
|
+
}
|
|
10127
|
+
async findGoalByLoop(idOrName) {
|
|
10128
|
+
return this.store.findGoalByLoop(idOrName);
|
|
10129
|
+
}
|
|
10130
|
+
async findGoalByRunId(id) {
|
|
10131
|
+
return this.store.findGoalByRunId(id);
|
|
10132
|
+
}
|
|
10133
|
+
async listGoals(opts = {}) {
|
|
10134
|
+
return this.store.listGoals(opts);
|
|
10135
|
+
}
|
|
10136
|
+
async listGoalPlanNodes(goalIdOrPlanId) {
|
|
10137
|
+
return this.store.listGoalPlanNodes(goalIdOrPlanId);
|
|
10138
|
+
}
|
|
10139
|
+
async listGoalRuns(opts = {}) {
|
|
10140
|
+
return this.store.listGoalRuns(opts);
|
|
10141
|
+
}
|
|
10142
|
+
async listRuns(opts = {}) {
|
|
10143
|
+
return this.store.listRuns(opts);
|
|
10144
|
+
}
|
|
10145
|
+
async getRun(id) {
|
|
10146
|
+
return this.store.getRun(id);
|
|
10147
|
+
}
|
|
10148
|
+
async writeRunReceipt(input) {
|
|
10149
|
+
return this.store.writeRunReceipt(input);
|
|
10150
|
+
}
|
|
10151
|
+
async getRunReceipt(runId) {
|
|
10152
|
+
return this.store.getRunReceipt(runId);
|
|
10153
|
+
}
|
|
10154
|
+
async listRunReceipts(opts = {}) {
|
|
10155
|
+
return this.store.listRunReceipts(opts);
|
|
10156
|
+
}
|
|
10157
|
+
async pruneHistory(opts) {
|
|
10158
|
+
return this.store.pruneHistory(opts);
|
|
10159
|
+
}
|
|
10160
|
+
}
|
|
10161
|
+
function clean(query) {
|
|
10162
|
+
const out = {};
|
|
10163
|
+
for (const [key, value] of Object.entries(query)) {
|
|
10164
|
+
if (value !== undefined && value !== null && value !== "")
|
|
10165
|
+
out[key] = value;
|
|
10166
|
+
}
|
|
10167
|
+
return out;
|
|
10168
|
+
}
|
|
10169
|
+
function pickArray(raw, key, fallback = []) {
|
|
10170
|
+
if (raw && typeof raw === "object" && Array.isArray(raw[key])) {
|
|
10171
|
+
return raw[key];
|
|
10172
|
+
}
|
|
10173
|
+
return fallback;
|
|
10174
|
+
}
|
|
10175
|
+
function pickObject(raw, key) {
|
|
10176
|
+
if (raw && typeof raw === "object") {
|
|
10177
|
+
const value = raw[key];
|
|
10178
|
+
return value == null ? undefined : value;
|
|
10179
|
+
}
|
|
10180
|
+
return raw == null ? undefined : raw;
|
|
10181
|
+
}
|
|
10182
|
+
|
|
10183
|
+
class ApiStore {
|
|
10184
|
+
client;
|
|
10185
|
+
baseUrl;
|
|
10186
|
+
transport = "cloud-http";
|
|
10187
|
+
constructor(client, baseUrl) {
|
|
10188
|
+
this.client = client;
|
|
10189
|
+
this.baseUrl = baseUrl;
|
|
10190
|
+
}
|
|
10191
|
+
get t() {
|
|
10192
|
+
return this.client.transport;
|
|
10193
|
+
}
|
|
10194
|
+
async close() {}
|
|
10195
|
+
async createLoop(input) {
|
|
10196
|
+
return pickObject(await this.t.post("/loops", input), "loop");
|
|
10197
|
+
}
|
|
10198
|
+
async getLoop(id) {
|
|
8179
10199
|
try {
|
|
8180
|
-
|
|
8181
|
-
|
|
8182
|
-
|
|
8183
|
-
} catch (error) {
|
|
8184
|
-
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
8185
|
-
return;
|
|
8186
|
-
throw error;
|
|
10200
|
+
return pickObject(await this.t.get(`/loops/${encodeURIComponent(id)}`), "loop");
|
|
10201
|
+
} catch {
|
|
10202
|
+
return;
|
|
8187
10203
|
}
|
|
8188
|
-
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
8189
|
-
deps.onRun?.(skipped);
|
|
8190
|
-
return skipped;
|
|
8191
10204
|
}
|
|
8192
|
-
|
|
8193
|
-
|
|
8194
|
-
|
|
8195
|
-
}
|
|
8196
|
-
|
|
10205
|
+
async findLoopByName(name) {
|
|
10206
|
+
const matches = (await this.listLoops({ name, limit: 1000 })).filter((loop) => loop.name === name);
|
|
10207
|
+
return matches[0];
|
|
10208
|
+
}
|
|
10209
|
+
async requireLoop(idOrName) {
|
|
10210
|
+
const loop = await this.resolveLoop(idOrName);
|
|
10211
|
+
if (!loop)
|
|
10212
|
+
throw new LoopNotFoundError(idOrName);
|
|
10213
|
+
return loop;
|
|
10214
|
+
}
|
|
10215
|
+
async requireUniqueLoop(idOrName) {
|
|
10216
|
+
const byId = await this.getLoop(idOrName);
|
|
10217
|
+
if (byId)
|
|
10218
|
+
return byId;
|
|
10219
|
+
const matches = (await this.listLoops({ name: idOrName, limit: 1000 })).filter((loop) => loop.name === idOrName);
|
|
10220
|
+
if (matches.length === 0)
|
|
10221
|
+
throw new LoopNotFoundError(idOrName);
|
|
10222
|
+
if (matches.length === 1)
|
|
10223
|
+
return matches[0];
|
|
10224
|
+
const active = matches.filter((loop) => !loop.archivedAt);
|
|
10225
|
+
if (active.length !== 1)
|
|
10226
|
+
throw new AmbiguousNameError(idOrName);
|
|
10227
|
+
return active[0];
|
|
10228
|
+
}
|
|
10229
|
+
async resolveLoop(idOrName) {
|
|
10230
|
+
const byId = await this.getLoop(idOrName);
|
|
10231
|
+
if (byId)
|
|
10232
|
+
return byId;
|
|
10233
|
+
const matches = (await this.listLoops({ name: idOrName, limit: 1000 })).filter((loop) => loop.name === idOrName);
|
|
10234
|
+
if (matches.length === 0)
|
|
8197
10235
|
return;
|
|
8198
|
-
|
|
10236
|
+
if (matches.length === 1)
|
|
10237
|
+
return matches[0];
|
|
10238
|
+
const active = matches.filter((loop) => !loop.archivedAt);
|
|
10239
|
+
if (active.length === 1)
|
|
10240
|
+
return active[0];
|
|
10241
|
+
throw new AmbiguousNameError(idOrName);
|
|
8199
10242
|
}
|
|
8200
|
-
|
|
8201
|
-
|
|
8202
|
-
return;
|
|
10243
|
+
async listLoops(opts = {}) {
|
|
10244
|
+
const raw = await this.t.get("/loops", { query: clean({ ...opts }) });
|
|
10245
|
+
return pickArray(raw, "loops");
|
|
8203
10246
|
}
|
|
8204
|
-
|
|
8205
|
-
|
|
8206
|
-
|
|
8207
|
-
|
|
8208
|
-
|
|
8209
|
-
|
|
8210
|
-
|
|
8211
|
-
|
|
8212
|
-
|
|
8213
|
-
|
|
8214
|
-
|
|
8215
|
-
|
|
8216
|
-
|
|
8217
|
-
|
|
8218
|
-
|
|
8219
|
-
|
|
8220
|
-
}
|
|
8221
|
-
|
|
8222
|
-
|
|
8223
|
-
|
|
8224
|
-
|
|
8225
|
-
|
|
10247
|
+
async countLoops(status, opts = {}) {
|
|
10248
|
+
const raw = await this.t.get("/loops/count", { query: clean({ status, ...opts }) });
|
|
10249
|
+
return Number(pickObject(raw, "count") ?? 0);
|
|
10250
|
+
}
|
|
10251
|
+
async updateLoop(id, patch) {
|
|
10252
|
+
return pickObject(await this.t.patch(`/loops/${encodeURIComponent(id)}`, patch), "loop");
|
|
10253
|
+
}
|
|
10254
|
+
async renameLoop(id, name) {
|
|
10255
|
+
return pickObject(await this.t.post(`/loops/${encodeURIComponent(id)}/rename`, { name }), "loop");
|
|
10256
|
+
}
|
|
10257
|
+
async archiveLoop(idOrName) {
|
|
10258
|
+
const loop = await this.requireLoop(idOrName);
|
|
10259
|
+
return pickObject(await this.t.post(`/loops/${encodeURIComponent(loop.id)}/archive`), "loop");
|
|
10260
|
+
}
|
|
10261
|
+
async unarchiveLoop(idOrName) {
|
|
10262
|
+
const loop = await this.requireLoop(idOrName);
|
|
10263
|
+
return pickObject(await this.t.post(`/loops/${encodeURIComponent(loop.id)}/unarchive`), "loop");
|
|
10264
|
+
}
|
|
10265
|
+
async deleteLoop(idOrName) {
|
|
10266
|
+
const loop = await this.resolveLoop(idOrName);
|
|
10267
|
+
if (!loop)
|
|
10268
|
+
return false;
|
|
10269
|
+
const raw = await this.t.request("DELETE", `/loops/${encodeURIComponent(loop.id)}`);
|
|
10270
|
+
return Boolean(pickObject(raw, "deleted") ?? true);
|
|
10271
|
+
}
|
|
10272
|
+
async createWorkflow(input) {
|
|
10273
|
+
return pickObject(await this.t.post("/workflows", input), "workflow");
|
|
10274
|
+
}
|
|
10275
|
+
async getWorkflow(id) {
|
|
10276
|
+
try {
|
|
10277
|
+
return pickObject(await this.t.get(`/workflows/${encodeURIComponent(id)}`), "workflow");
|
|
10278
|
+
} catch {
|
|
8226
10279
|
return;
|
|
8227
|
-
|
|
10280
|
+
}
|
|
10281
|
+
}
|
|
10282
|
+
async findWorkflowByName(name) {
|
|
10283
|
+
const matches = (await this.listWorkflows({ limit: 1000 })).filter((wf) => wf.name === name);
|
|
10284
|
+
return matches[0];
|
|
10285
|
+
}
|
|
10286
|
+
async requireWorkflow(idOrName) {
|
|
10287
|
+
const byId = await this.getWorkflow(idOrName);
|
|
10288
|
+
if (byId)
|
|
10289
|
+
return byId;
|
|
10290
|
+
const byName = await this.findWorkflowByName(idOrName);
|
|
10291
|
+
if (byName)
|
|
10292
|
+
return byName;
|
|
10293
|
+
throw new Error(`workflow not found: ${idOrName}`);
|
|
10294
|
+
}
|
|
10295
|
+
async listWorkflows(opts = {}) {
|
|
10296
|
+
const raw = await this.t.get("/workflows", { query: clean({ ...opts }) });
|
|
10297
|
+
return pickArray(raw, "workflows");
|
|
10298
|
+
}
|
|
10299
|
+
async countWorkflows(opts = {}) {
|
|
10300
|
+
const raw = await this.t.get("/workflows/count", { query: clean({ ...opts }) });
|
|
10301
|
+
return Number(pickObject(raw, "count") ?? 0);
|
|
10302
|
+
}
|
|
10303
|
+
async archiveWorkflow(idOrName) {
|
|
10304
|
+
const wf = await this.requireWorkflow(idOrName);
|
|
10305
|
+
return pickObject(await this.t.post(`/workflows/${encodeURIComponent(wf.id)}/archive`), "workflow");
|
|
10306
|
+
}
|
|
10307
|
+
async getWorkflowRun(id) {
|
|
8228
10308
|
try {
|
|
8229
|
-
|
|
8230
|
-
|
|
8231
|
-
|
|
8232
|
-
} catch (error) {
|
|
8233
|
-
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
8234
|
-
return;
|
|
8235
|
-
throw error;
|
|
10309
|
+
return pickObject(await this.t.get(`/workflow-runs/${encodeURIComponent(id)}`), "workflowRun");
|
|
10310
|
+
} catch {
|
|
10311
|
+
return;
|
|
8236
10312
|
}
|
|
8237
|
-
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
8238
|
-
deps.onRun?.(skipped);
|
|
8239
|
-
return skipped;
|
|
8240
10313
|
}
|
|
8241
|
-
|
|
8242
|
-
|
|
8243
|
-
|
|
8244
|
-
|
|
8245
|
-
|
|
10314
|
+
async requireWorkflowRun(id) {
|
|
10315
|
+
const run = await this.getWorkflowRun(id);
|
|
10316
|
+
if (!run)
|
|
10317
|
+
throw new Error(`workflow run not found: ${id}`);
|
|
10318
|
+
return run;
|
|
10319
|
+
}
|
|
10320
|
+
async listWorkflowRuns(opts = {}) {
|
|
10321
|
+
const raw = await this.t.get("/workflow-runs", { query: clean({ ...opts }) });
|
|
10322
|
+
return pickArray(raw, "workflowRuns");
|
|
10323
|
+
}
|
|
10324
|
+
async listWorkflowStepRuns(workflowRunId) {
|
|
10325
|
+
const raw = await this.t.get(`/workflow-runs/${encodeURIComponent(workflowRunId)}/steps`);
|
|
10326
|
+
return pickArray(raw, "steps");
|
|
10327
|
+
}
|
|
10328
|
+
async listWorkflowEvents(workflowRunId, limit) {
|
|
10329
|
+
const raw = await this.t.get(`/workflow-runs/${encodeURIComponent(workflowRunId)}/events`, { query: clean({ limit }) });
|
|
10330
|
+
return pickArray(raw, "events");
|
|
10331
|
+
}
|
|
10332
|
+
async recoverWorkflowRun(workflowRunId, reason) {
|
|
10333
|
+
const raw = await this.t.post(`/workflow-runs/${encodeURIComponent(workflowRunId)}/recover`, { reason });
|
|
10334
|
+
return {
|
|
10335
|
+
run: pickObject(raw, "workflowRun"),
|
|
10336
|
+
recoveredSteps: pickArray(raw, "recoveredSteps")
|
|
10337
|
+
};
|
|
10338
|
+
}
|
|
10339
|
+
async cancelWorkflowRun(_workflowRunId, _reason) {
|
|
10340
|
+
throw new CloudUnsupportedError("workflows cancel");
|
|
10341
|
+
}
|
|
10342
|
+
async listWorkflowWorkItems(opts = {}) {
|
|
10343
|
+
const raw = await this.t.get("/work-items", { query: clean({ ...opts }) });
|
|
10344
|
+
return pickArray(raw, "workItems");
|
|
10345
|
+
}
|
|
10346
|
+
async getWorkflowWorkItem(id) {
|
|
10347
|
+
try {
|
|
10348
|
+
return pickObject(await this.t.get(`/work-items/${encodeURIComponent(id)}`), "workItem");
|
|
10349
|
+
} catch {
|
|
8246
10350
|
return;
|
|
8247
|
-
|
|
10351
|
+
}
|
|
8248
10352
|
}
|
|
8249
|
-
|
|
8250
|
-
|
|
8251
|
-
return;
|
|
10353
|
+
async requeueWorkflowWorkItem(_id, _patch = {}) {
|
|
10354
|
+
throw new CloudUnsupportedError("routes requeue");
|
|
8252
10355
|
}
|
|
8253
|
-
|
|
8254
|
-
|
|
8255
|
-
|
|
8256
|
-
}
|
|
8257
|
-
function recoverAndExpire(deps, now) {
|
|
8258
|
-
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
8259
|
-
const recoveredByLoop = new Map;
|
|
8260
|
-
for (const run of recovered) {
|
|
8261
|
-
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
10356
|
+
async listWorkflowInvocations(opts = {}) {
|
|
10357
|
+
const raw = await this.t.get("/invocations", { query: clean({ ...opts }) });
|
|
10358
|
+
return pickArray(raw, "invocations");
|
|
8262
10359
|
}
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
if (retryable) {
|
|
8269
|
-
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
|
|
8270
|
-
continue;
|
|
10360
|
+
async getWorkflowInvocation(id) {
|
|
10361
|
+
try {
|
|
10362
|
+
return pickObject(await this.t.get(`/invocations/${encodeURIComponent(id)}`), "invocation");
|
|
10363
|
+
} catch {
|
|
10364
|
+
return;
|
|
8271
10365
|
}
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
|
|
8275
|
-
|
|
8276
|
-
|
|
10366
|
+
}
|
|
10367
|
+
async getGoal(id) {
|
|
10368
|
+
try {
|
|
10369
|
+
return pickObject(await this.t.get(`/goals/${encodeURIComponent(id)}`), "goal");
|
|
10370
|
+
} catch {
|
|
10371
|
+
return;
|
|
8277
10372
|
}
|
|
8278
10373
|
}
|
|
8279
|
-
|
|
8280
|
-
|
|
8281
|
-
}
|
|
8282
|
-
|
|
8283
|
-
|
|
8284
|
-
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
8285
|
-
const claims = [];
|
|
8286
|
-
const claimed = [];
|
|
8287
|
-
const skipped = [];
|
|
8288
|
-
const maxClaims = Math.max(0, deps.maxClaims ?? Number.POSITIVE_INFINITY);
|
|
8289
|
-
if (maxClaims === 0)
|
|
8290
|
-
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
8291
|
-
const laneLimits = deps.laneLimits;
|
|
8292
|
-
const laneClaims = { command: 0, agent: 0 };
|
|
8293
|
-
const laneCap = (lane) => laneLimits === undefined ? Number.POSITIVE_INFINITY : Math.max(0, laneLimits[lane] ?? Number.POSITIVE_INFINITY);
|
|
8294
|
-
const laneFull = (lane) => laneClaims[lane] >= laneCap(lane);
|
|
8295
|
-
for (const loop of deps.store.dueLoops(now)) {
|
|
8296
|
-
if (claims.length >= maxClaims)
|
|
8297
|
-
break;
|
|
8298
|
-
const lane = loopLane(loop);
|
|
8299
|
-
if (laneFull(lane))
|
|
8300
|
-
continue;
|
|
8301
|
-
const plan = dueSlots(loop, now);
|
|
8302
|
-
let loopSkips = 0;
|
|
8303
|
-
for (const slot of plan.slots) {
|
|
8304
|
-
if (claims.length >= maxClaims)
|
|
8305
|
-
break;
|
|
8306
|
-
if (laneFull(lane))
|
|
8307
|
-
break;
|
|
8308
|
-
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
8309
|
-
break;
|
|
8310
|
-
const run = claimSlot(deps, loop, slot);
|
|
8311
|
-
if (!run)
|
|
8312
|
-
continue;
|
|
8313
|
-
if ("loop" in run) {
|
|
8314
|
-
claims.push(run);
|
|
8315
|
-
claimed.push(run.run);
|
|
8316
|
-
laneClaims[lane] += 1;
|
|
8317
|
-
} else if (run.status === "skipped") {
|
|
8318
|
-
skipped.push(run);
|
|
8319
|
-
loopSkips += 1;
|
|
8320
|
-
}
|
|
10374
|
+
async findGoalByLoop(idOrName) {
|
|
10375
|
+
try {
|
|
10376
|
+
return pickObject(await this.t.get("/goals", { query: clean({ loop: idOrName }) }), "goal");
|
|
10377
|
+
} catch {
|
|
10378
|
+
return;
|
|
8321
10379
|
}
|
|
8322
10380
|
}
|
|
8323
|
-
|
|
8324
|
-
|
|
8325
|
-
|
|
8326
|
-
|
|
8327
|
-
|
|
8328
|
-
const claimed = [];
|
|
8329
|
-
const completed = [];
|
|
8330
|
-
const skipped = [];
|
|
8331
|
-
for (const loop of deps.store.dueLoops(now)) {
|
|
8332
|
-
const plan = dueSlots(loop, now);
|
|
8333
|
-
let loopSkips = 0;
|
|
8334
|
-
for (const slot of plan.slots) {
|
|
8335
|
-
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
8336
|
-
break;
|
|
8337
|
-
const run = await runSlot(deps, loop, slot);
|
|
8338
|
-
if (!run)
|
|
8339
|
-
continue;
|
|
8340
|
-
if (run.status === "running")
|
|
8341
|
-
claimed.push(run);
|
|
8342
|
-
else if (run.status === "skipped") {
|
|
8343
|
-
skipped.push(run);
|
|
8344
|
-
loopSkips += 1;
|
|
8345
|
-
} else
|
|
8346
|
-
completed.push(run);
|
|
8347
|
-
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
8348
|
-
break;
|
|
10381
|
+
async findGoalByRunId(id) {
|
|
10382
|
+
try {
|
|
10383
|
+
return pickObject(await this.t.get("/goals", { query: clean({ runId: id }) }), "goal");
|
|
10384
|
+
} catch {
|
|
10385
|
+
return;
|
|
8349
10386
|
}
|
|
8350
10387
|
}
|
|
8351
|
-
|
|
10388
|
+
async listGoals(opts = {}) {
|
|
10389
|
+
const raw = await this.t.get("/goals", { query: clean({ ...opts }) });
|
|
10390
|
+
return pickArray(raw, "goals");
|
|
10391
|
+
}
|
|
10392
|
+
async listGoalPlanNodes(goalIdOrPlanId) {
|
|
10393
|
+
const raw = await this.t.get(`/goals/${encodeURIComponent(goalIdOrPlanId)}/plan-nodes`);
|
|
10394
|
+
return pickArray(raw, "nodes");
|
|
10395
|
+
}
|
|
10396
|
+
async listGoalRuns(opts = {}) {
|
|
10397
|
+
const raw = await this.t.get("/goal-runs", { query: clean({ ...opts }) });
|
|
10398
|
+
return pickArray(raw, "goalRuns");
|
|
10399
|
+
}
|
|
10400
|
+
async listRuns(opts = {}) {
|
|
10401
|
+
const raw = await this.t.get("/runs", { query: clean({ ...opts, showOutput: true }) });
|
|
10402
|
+
return pickArray(raw, "runs");
|
|
10403
|
+
}
|
|
10404
|
+
async getRun(id) {
|
|
10405
|
+
try {
|
|
10406
|
+
return pickObject(await this.t.get(`/runs/${encodeURIComponent(id)}`, { query: { showOutput: true } }), "run");
|
|
10407
|
+
} catch {
|
|
10408
|
+
return;
|
|
10409
|
+
}
|
|
10410
|
+
}
|
|
10411
|
+
async writeRunReceipt(input) {
|
|
10412
|
+
return pickObject(await this.t.post("/receipts", input), "receipt");
|
|
10413
|
+
}
|
|
10414
|
+
async getRunReceipt(runId) {
|
|
10415
|
+
try {
|
|
10416
|
+
return pickObject(await this.t.get(`/receipts/${encodeURIComponent(runId)}`), "receipt");
|
|
10417
|
+
} catch {
|
|
10418
|
+
return;
|
|
10419
|
+
}
|
|
10420
|
+
}
|
|
10421
|
+
async listRunReceipts(opts = {}) {
|
|
10422
|
+
const raw = await this.t.get("/receipts", { query: clean({ ...opts }) });
|
|
10423
|
+
return pickArray(raw, "receipts");
|
|
10424
|
+
}
|
|
10425
|
+
async pruneHistory(opts) {
|
|
10426
|
+
const raw = await this.t.post("/history/prune", {
|
|
10427
|
+
maxAgeDays: opts.maxAgeDays,
|
|
10428
|
+
keepPerLoop: opts.keepPerLoop,
|
|
10429
|
+
dryRun: opts.dryRun
|
|
10430
|
+
});
|
|
10431
|
+
return pickObject(raw, "history");
|
|
10432
|
+
}
|
|
10433
|
+
}
|
|
10434
|
+
function getStore(env = process.env) {
|
|
10435
|
+
const resolution = resolveCloudStorage("loops", env);
|
|
10436
|
+
if (resolution.transport === "cloud-http")
|
|
10437
|
+
return new ApiStore(resolution.client, resolution.baseUrl);
|
|
10438
|
+
return new LocalStore;
|
|
10439
|
+
}
|
|
10440
|
+
function isCloudStore(env = process.env) {
|
|
10441
|
+
return resolveCloudStorage("loops", env).transport === "cloud-http";
|
|
8352
10442
|
}
|
|
8353
10443
|
|
|
8354
10444
|
// src/sdk/index.ts
|
|
@@ -8357,10 +10447,16 @@ class LoopsClient {
|
|
|
8357
10447
|
ownStore;
|
|
8358
10448
|
runnerId;
|
|
8359
10449
|
constructor(opts = {}) {
|
|
8360
|
-
this.store = opts.store
|
|
10450
|
+
this.store = opts.store ? new LocalStore(opts.store) : getStore();
|
|
8361
10451
|
this.ownStore = !opts.store;
|
|
8362
10452
|
this.runnerId = opts.runnerId ?? `sdk:${process.pid}`;
|
|
8363
10453
|
}
|
|
10454
|
+
localRuntime(operation) {
|
|
10455
|
+
if (this.store.transport !== "local") {
|
|
10456
|
+
throw new Error(`loops SDK ${operation} operates on this machine's local runtime and is not available while flipped to the hosted OpenLoops API. ` + `Unset HASNA_LOOPS_API_URL/HASNA_LOOPS_API_KEY (or set HASNA_LOOPS_STORAGE_MODE=local) to run it here.`);
|
|
10457
|
+
}
|
|
10458
|
+
return this.store.raw;
|
|
10459
|
+
}
|
|
8364
10460
|
create(input) {
|
|
8365
10461
|
return this.store.createLoop(input);
|
|
8366
10462
|
}
|
|
@@ -8375,11 +10471,12 @@ class LoopsClient {
|
|
|
8375
10471
|
get(idOrName) {
|
|
8376
10472
|
return this.store.requireLoop(idOrName);
|
|
8377
10473
|
}
|
|
8378
|
-
pause(idOrName) {
|
|
8379
|
-
|
|
10474
|
+
async pause(idOrName) {
|
|
10475
|
+
const loop = await this.store.requireUniqueLoop(idOrName);
|
|
10476
|
+
return this.store.updateLoop(loop.id, { status: "paused" });
|
|
8380
10477
|
}
|
|
8381
|
-
resume(idOrName) {
|
|
8382
|
-
const loop = this.store.requireUniqueLoop(idOrName);
|
|
10478
|
+
async resume(idOrName) {
|
|
10479
|
+
const loop = await this.store.requireUniqueLoop(idOrName);
|
|
8383
10480
|
let nextRunAt = loop.nextRunAt;
|
|
8384
10481
|
if (!nextRunAt) {
|
|
8385
10482
|
const now = new Date;
|
|
@@ -8387,8 +10484,9 @@ class LoopsClient {
|
|
|
8387
10484
|
}
|
|
8388
10485
|
return this.store.updateLoop(loop.id, { status: "active", nextRunAt });
|
|
8389
10486
|
}
|
|
8390
|
-
stop(idOrName) {
|
|
8391
|
-
|
|
10487
|
+
async stop(idOrName) {
|
|
10488
|
+
const loop = await this.store.requireUniqueLoop(idOrName);
|
|
10489
|
+
return this.store.updateLoop(loop.id, { status: "stopped", nextRunAt: undefined });
|
|
8392
10490
|
}
|
|
8393
10491
|
archive(idOrName) {
|
|
8394
10492
|
return this.store.archiveLoop(idOrName);
|
|
@@ -8396,14 +10494,15 @@ class LoopsClient {
|
|
|
8396
10494
|
unarchive(idOrName) {
|
|
8397
10495
|
return this.store.unarchiveLoop(idOrName);
|
|
8398
10496
|
}
|
|
8399
|
-
delete(idOrName) {
|
|
8400
|
-
|
|
10497
|
+
async delete(idOrName) {
|
|
10498
|
+
const loop = await this.store.requireUniqueLoop(idOrName);
|
|
10499
|
+
return this.store.deleteLoop(loop.id);
|
|
8401
10500
|
}
|
|
8402
|
-
runs(idOrName, filters = {}) {
|
|
10501
|
+
async runs(idOrName, filters = {}) {
|
|
8403
10502
|
let loopId;
|
|
8404
10503
|
if (idOrName) {
|
|
8405
10504
|
try {
|
|
8406
|
-
loopId = this.get(idOrName).id;
|
|
10505
|
+
loopId = (await this.get(idOrName)).id;
|
|
8407
10506
|
} catch (error) {
|
|
8408
10507
|
if (error instanceof LoopNotFoundError)
|
|
8409
10508
|
return [];
|
|
@@ -8412,41 +10511,59 @@ class LoopsClient {
|
|
|
8412
10511
|
}
|
|
8413
10512
|
return this.store.listRuns({ loopId, status: filters.status, limit: filters.limit });
|
|
8414
10513
|
}
|
|
8415
|
-
|
|
8416
|
-
return
|
|
10514
|
+
writeReceipt(input) {
|
|
10515
|
+
return this.store.writeRunReceipt(input);
|
|
8417
10516
|
}
|
|
8418
|
-
|
|
8419
|
-
return
|
|
10517
|
+
receipt(runId) {
|
|
10518
|
+
return this.store.getRunReceipt(runId);
|
|
10519
|
+
}
|
|
10520
|
+
receipts(filters = {}) {
|
|
10521
|
+
return this.store.listRunReceipts(filters);
|
|
8420
10522
|
}
|
|
8421
|
-
goal(idOrName) {
|
|
8422
|
-
const goal = this.store.getGoal(idOrName) ?? this.store.findGoalByLoop(idOrName) ?? this.store.findGoalByRunId(idOrName);
|
|
10523
|
+
async goal(idOrName) {
|
|
10524
|
+
const goal = await this.store.getGoal(idOrName) ?? await this.store.findGoalByLoop(idOrName) ?? await this.store.findGoalByRunId(idOrName);
|
|
8423
10525
|
return {
|
|
8424
10526
|
goal,
|
|
8425
|
-
runs: goal ? this.store.listGoalRuns({ goalId: goal.goalId }) : []
|
|
10527
|
+
runs: goal ? await this.store.listGoalRuns({ goalId: goal.goalId }) : []
|
|
8426
10528
|
};
|
|
8427
10529
|
}
|
|
10530
|
+
doctor() {
|
|
10531
|
+
return runDoctor(this.localRuntime("doctor()"));
|
|
10532
|
+
}
|
|
10533
|
+
health(opts = {}) {
|
|
10534
|
+
return buildHealthReport(this.localRuntime("health()"), opts);
|
|
10535
|
+
}
|
|
10536
|
+
healthScan(opts = {}) {
|
|
10537
|
+
const store = this.localRuntime("healthScan()");
|
|
10538
|
+
return buildHealthScan(store, {
|
|
10539
|
+
...opts,
|
|
10540
|
+
doctor: opts.doctor ? runDoctor(store) : undefined,
|
|
10541
|
+
daemon: opts.daemon ? daemonStatus(store) : undefined
|
|
10542
|
+
});
|
|
10543
|
+
}
|
|
8428
10544
|
async tick() {
|
|
8429
|
-
return tick({ store: this.
|
|
10545
|
+
return tick({ store: this.localRuntime("tick()"), runnerId: this.runnerId });
|
|
8430
10546
|
}
|
|
8431
10547
|
async runNow(idOrName) {
|
|
8432
|
-
const
|
|
10548
|
+
const store = this.localRuntime("runNow()");
|
|
10549
|
+
const result = await runLoopNow({ store, idOrName: store.requireUniqueLoop(idOrName).id, runnerId: this.runnerId });
|
|
8433
10550
|
return result.run;
|
|
8434
10551
|
}
|
|
8435
10552
|
exportBundle(opts = {}) {
|
|
8436
|
-
return exportLoopsMigrationBundle(this.
|
|
10553
|
+
return exportLoopsMigrationBundle(this.localRuntime("exportBundle()"), opts);
|
|
8437
10554
|
}
|
|
8438
10555
|
planImport(bundle, opts = {}) {
|
|
8439
|
-
return buildImportMigrationPlan(this.
|
|
10556
|
+
return buildImportMigrationPlan(this.localRuntime("planImport()"), bundle, opts);
|
|
8440
10557
|
}
|
|
8441
10558
|
importBundle(bundle, opts = {}) {
|
|
8442
|
-
return applyImportMigrationBundle(this.
|
|
10559
|
+
return applyImportMigrationBundle(this.localRuntime("importBundle()"), bundle, opts);
|
|
8443
10560
|
}
|
|
8444
10561
|
planSelfHostedMigration(opts = {}) {
|
|
8445
|
-
return buildSelfHostedMigrationPlan(this.
|
|
10562
|
+
return buildSelfHostedMigrationPlan(this.localRuntime("planSelfHostedMigration()"), { ...opts, operation: opts.operation ?? "self-hosted-migrate" });
|
|
8446
10563
|
}
|
|
8447
|
-
close() {
|
|
10564
|
+
async close() {
|
|
8448
10565
|
if (this.ownStore)
|
|
8449
|
-
this.store.close();
|
|
10566
|
+
await this.store.close();
|
|
8450
10567
|
}
|
|
8451
10568
|
}
|
|
8452
10569
|
function loops(opts = {}) {
|