@hasna/loops 0.4.5 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +2 -2
- package/dist/api/index.js +19 -10
- package/dist/cli/index.js +368 -192
- package/dist/daemon/index.js +317 -162
- package/dist/index.js +339 -174
- package/dist/lib/executor.d.ts +14 -1
- package/dist/lib/mode.js +1 -1
- package/dist/lib/storage/index.js +56 -4
- package/dist/lib/storage/sqlite.js +56 -4
- package/dist/lib/store.d.ts +7 -0
- package/dist/lib/store.js +56 -4
- package/dist/lib/template-kit.d.ts +6 -0
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +321 -165
- package/dist/runner/index.d.ts +3 -1
- package/dist/runner/index.js +130 -11
- package/dist/sdk/index.js +320 -166
- package/docs/USAGE.md +2 -2
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -1493,11 +1493,14 @@ function ensurePrivateStorePath(file) {
|
|
|
1493
1493
|
class Store {
|
|
1494
1494
|
db;
|
|
1495
1495
|
rootDir;
|
|
1496
|
+
memoryRootDir;
|
|
1496
1497
|
constructor(path) {
|
|
1497
1498
|
const file = path ?? dbPath();
|
|
1498
1499
|
if (file !== ":memory:")
|
|
1499
1500
|
ensurePrivateStorePath(file);
|
|
1500
1501
|
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
|
|
1502
|
+
if (file === ":memory:")
|
|
1503
|
+
this.memoryRootDir = this.rootDir;
|
|
1501
1504
|
this.db = new Database(file);
|
|
1502
1505
|
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
1503
1506
|
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
@@ -1950,9 +1953,12 @@ class Store {
|
|
|
1950
1953
|
const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1951
1954
|
if (rows.length === 0)
|
|
1952
1955
|
throw new LoopNotFoundError(idOrName);
|
|
1953
|
-
if (rows.length
|
|
1956
|
+
if (rows.length === 1)
|
|
1957
|
+
return rowToLoop(rows[0]);
|
|
1958
|
+
const active = this.db.query("SELECT * FROM loops WHERE name = ? AND archived_at IS NULL ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1959
|
+
if (active.length !== 1)
|
|
1954
1960
|
throw new AmbiguousNameError(idOrName);
|
|
1955
|
-
return rowToLoop(
|
|
1961
|
+
return rowToLoop(active[0]);
|
|
1956
1962
|
}
|
|
1957
1963
|
requireLoop(idOrName) {
|
|
1958
1964
|
return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
|
|
@@ -2293,6 +2299,7 @@ class Store {
|
|
|
2293
2299
|
return this.transact(() => {
|
|
2294
2300
|
const loop = this.requireLoop(idOrName);
|
|
2295
2301
|
this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
|
|
2302
|
+
this.db.query("DELETE FROM loop_runs WHERE loop_id = ?").run(loop.id);
|
|
2296
2303
|
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
2297
2304
|
return res.changes > 0;
|
|
2298
2305
|
});
|
|
@@ -2777,7 +2784,10 @@ class Store {
|
|
|
2777
2784
|
return rowToGoal(row);
|
|
2778
2785
|
}
|
|
2779
2786
|
if (context.sourceType && context.sourceId) {
|
|
2780
|
-
const
|
|
2787
|
+
const terminalPlaceholders = GOAL_TERMINAL.map(() => "?").join(", ");
|
|
2788
|
+
const row = this.db.query(`SELECT * FROM goals
|
|
2789
|
+
WHERE source_type = ? AND source_id = ? AND status NOT IN (${terminalPlaceholders})
|
|
2790
|
+
ORDER BY created_at DESC LIMIT 1`).get(context.sourceType, context.sourceId, ...GOAL_TERMINAL);
|
|
2781
2791
|
if (row)
|
|
2782
2792
|
return rowToGoal(row);
|
|
2783
2793
|
}
|
|
@@ -3197,6 +3207,41 @@ class Store {
|
|
|
3197
3207
|
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
3198
3208
|
return run;
|
|
3199
3209
|
}
|
|
3210
|
+
recordWorkflowStepProgress(workflowRunId, stepId, progress, opts = {}) {
|
|
3211
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
3212
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
3213
|
+
try {
|
|
3214
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
3215
|
+
SET stdout=COALESCE($stdout, stdout),
|
|
3216
|
+
stderr=COALESCE($stderr, stderr),
|
|
3217
|
+
updated_at=$updated
|
|
3218
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
3219
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3220
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3221
|
+
))`).run({
|
|
3222
|
+
$workflowRunId: workflowRunId,
|
|
3223
|
+
$stepId: stepId,
|
|
3224
|
+
$stdout: progress.stdout === undefined ? null : scrubbedOrNull(progress.stdout),
|
|
3225
|
+
$stderr: progress.stderr === undefined ? null : scrubbedOrNull(progress.stderr),
|
|
3226
|
+
$updated: now,
|
|
3227
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3228
|
+
$now: now
|
|
3229
|
+
});
|
|
3230
|
+
if (res.changes === 1) {
|
|
3231
|
+
this.appendWorkflowEvent(workflowRunId, "step_progress", stepId, progress.payload);
|
|
3232
|
+
}
|
|
3233
|
+
this.db.exec("COMMIT");
|
|
3234
|
+
} catch (error) {
|
|
3235
|
+
try {
|
|
3236
|
+
this.db.exec("ROLLBACK");
|
|
3237
|
+
} catch {}
|
|
3238
|
+
throw error;
|
|
3239
|
+
}
|
|
3240
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
3241
|
+
if (!run)
|
|
3242
|
+
throw new Error(`workflow step run not found after progress update: ${workflowRunId}/${stepId}`);
|
|
3243
|
+
return run;
|
|
3244
|
+
}
|
|
3200
3245
|
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
3201
3246
|
return this.transact(() => {
|
|
3202
3247
|
const now = nowIso();
|
|
@@ -3634,7 +3679,8 @@ class Store {
|
|
|
3634
3679
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3635
3680
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3636
3681
|
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, claim_token=NULL, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3637
|
-
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3682
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3683
|
+
WHERE id=$id AND status='running'`).run(params);
|
|
3638
3684
|
const run = this.getRun(id);
|
|
3639
3685
|
if (!run)
|
|
3640
3686
|
throw new Error(`run not found after finalize: ${id}`);
|
|
@@ -4162,12 +4208,18 @@ class Store {
|
|
|
4162
4208
|
}
|
|
4163
4209
|
close() {
|
|
4164
4210
|
this.db.close();
|
|
4211
|
+
if (this.memoryRootDir) {
|
|
4212
|
+
try {
|
|
4213
|
+
rmSync2(this.memoryRootDir, { recursive: true, force: true });
|
|
4214
|
+
} catch {}
|
|
4215
|
+
this.memoryRootDir = undefined;
|
|
4216
|
+
}
|
|
4165
4217
|
}
|
|
4166
4218
|
}
|
|
4167
4219
|
// package.json
|
|
4168
4220
|
var package_default = {
|
|
4169
4221
|
name: "@hasna/loops",
|
|
4170
|
-
version: "0.4.
|
|
4222
|
+
version: "0.4.6",
|
|
4171
4223
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4172
4224
|
type: "module",
|
|
4173
4225
|
main: "dist/index.js",
|
|
@@ -4882,7 +4934,12 @@ function notifySpawn(pid, opts) {
|
|
|
4882
4934
|
if (!pid)
|
|
4883
4935
|
return;
|
|
4884
4936
|
opts.onSpawn?.(pid);
|
|
4885
|
-
|
|
4937
|
+
const startedMs = processStartTimeMs(pid);
|
|
4938
|
+
opts.onSpawnProcess?.({
|
|
4939
|
+
pid,
|
|
4940
|
+
pgid: pid,
|
|
4941
|
+
processStartedAt: startedMs !== undefined ? new Date(startedMs).toISOString() : nowIso()
|
|
4942
|
+
});
|
|
4886
4943
|
}
|
|
4887
4944
|
function shellQuote(value) {
|
|
4888
4945
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
@@ -5237,6 +5294,13 @@ function numberField(value, key) {
|
|
|
5237
5294
|
function codewithAgentStatus(readJson) {
|
|
5238
5295
|
return stringField(recordField(readJson, "agent"), "status") ?? stringField(recordField(readJson, "statusSnapshot"), "status");
|
|
5239
5296
|
}
|
|
5297
|
+
function codewithAgentLastEventSeq(readJson) {
|
|
5298
|
+
const agentSeq = numberField(recordField(readJson, "agent"), "lastEventSeq");
|
|
5299
|
+
const snapshotSeq = numberField(recordField(readJson, "statusSnapshot"), "lastEventSeq");
|
|
5300
|
+
if (agentSeq !== undefined && snapshotSeq !== undefined)
|
|
5301
|
+
return Math.max(agentSeq, snapshotSeq);
|
|
5302
|
+
return agentSeq ?? snapshotSeq;
|
|
5303
|
+
}
|
|
5240
5304
|
function codewithAgentEvidence(startJson, readJson, logsJson) {
|
|
5241
5305
|
const agent = recordField(readJson, "agent") ?? recordField(startJson, "agent");
|
|
5242
5306
|
const statusSnapshot = recordField(readJson, "statusSnapshot");
|
|
@@ -5267,6 +5331,21 @@ function codewithAgentEvidence(startJson, readJson, logsJson) {
|
|
|
5267
5331
|
events
|
|
5268
5332
|
}, null, 2);
|
|
5269
5333
|
}
|
|
5334
|
+
function codewithAgentProgress(readJson) {
|
|
5335
|
+
const agent = recordField(readJson, "agent");
|
|
5336
|
+
const statusSnapshot = recordField(readJson, "statusSnapshot");
|
|
5337
|
+
return {
|
|
5338
|
+
provider: "codewith",
|
|
5339
|
+
agentId: stringField(agent, "agentId"),
|
|
5340
|
+
status: stringField(agent, "status") ?? stringField(statusSnapshot, "status"),
|
|
5341
|
+
summary: stringField(statusSnapshot, "summary"),
|
|
5342
|
+
statusReason: stringField(agent, "statusReason"),
|
|
5343
|
+
threadId: stringField(agent, "threadId"),
|
|
5344
|
+
rolloutPath: stringField(agent, "rolloutPath"),
|
|
5345
|
+
pid: numberField(agent, "pid"),
|
|
5346
|
+
lastEventSeq: codewithAgentLastEventSeq(readJson)
|
|
5347
|
+
};
|
|
5348
|
+
}
|
|
5270
5349
|
function sleep(ms) {
|
|
5271
5350
|
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
5272
5351
|
}
|
|
@@ -5301,6 +5380,7 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
|
|
|
5301
5380
|
if (!agentId) {
|
|
5302
5381
|
return failureResult(startedAt, "codewith agent start did not return agent.agentId", { stdout: start.stdout, stderr: stderr.value() });
|
|
5303
5382
|
}
|
|
5383
|
+
opts.onAgentProgress?.(codewithAgentProgress(startJson));
|
|
5304
5384
|
const stopAgent = async () => {
|
|
5305
5385
|
await spawnCapture(spec.command, codewithAgentControlArgs(target, "stop", agentId), {
|
|
5306
5386
|
cwd: spec.cwd,
|
|
@@ -5343,10 +5423,15 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
|
|
|
5343
5423
|
});
|
|
5344
5424
|
}
|
|
5345
5425
|
const status = codewithAgentStatus(lastReadJson);
|
|
5346
|
-
const fingerprint = JSON.stringify({
|
|
5426
|
+
const fingerprint = JSON.stringify({
|
|
5427
|
+
status,
|
|
5428
|
+
agentLastEventSeq: numberField(recordField(lastReadJson, "agent"), "lastEventSeq"),
|
|
5429
|
+
snapshot: recordField(lastReadJson, "statusSnapshot") ?? null
|
|
5430
|
+
});
|
|
5347
5431
|
if (fingerprint !== lastFingerprint) {
|
|
5348
5432
|
lastFingerprint = fingerprint;
|
|
5349
5433
|
lastProgressAt = Date.now();
|
|
5434
|
+
opts.onAgentProgress?.(codewithAgentProgress(lastReadJson));
|
|
5350
5435
|
}
|
|
5351
5436
|
if (status === "completed" || status === "failed" || status === "cancelled") {
|
|
5352
5437
|
const logs = await spawnCapture(spec.command, codewithAgentControlArgs(target, "logs", agentId), {
|
|
@@ -6347,173 +6432,213 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
6347
6432
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
6348
6433
|
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), store.listWorkflowStepRuns(run.id), run.error);
|
|
6349
6434
|
}
|
|
6435
|
+
const resumedRunningSteps = store.listWorkflowStepRuns(run.id).filter((step) => step.status === "running");
|
|
6436
|
+
if (resumedRunningSteps.length > 0 && resumedRunningSteps.every((step) => step.pid !== undefined)) {
|
|
6437
|
+
try {
|
|
6438
|
+
store.recoverWorkflowRun(run.id, "workflow run resumed after lease takeover");
|
|
6439
|
+
} catch {}
|
|
6440
|
+
}
|
|
6350
6441
|
const ordered = workflowExecutionOrder(workflow);
|
|
6351
6442
|
const byId = new Map(workflow.steps.map((step) => [step.id, step]));
|
|
6352
6443
|
let blockingError;
|
|
6353
6444
|
let terminalStatus = "succeeded";
|
|
6354
|
-
|
|
6355
|
-
|
|
6356
|
-
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
const
|
|
6371
|
-
|
|
6372
|
-
|
|
6373
|
-
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
store.
|
|
6380
|
-
|
|
6381
|
-
|
|
6445
|
+
try {
|
|
6446
|
+
for (const step of ordered) {
|
|
6447
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6448
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
6449
|
+
blockingError = "workflow run was cancelled";
|
|
6450
|
+
break;
|
|
6451
|
+
}
|
|
6452
|
+
const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
6453
|
+
if (pendingTimeout) {
|
|
6454
|
+
terminalStatus = "timed_out";
|
|
6455
|
+
blockingError = pendingTimeout;
|
|
6456
|
+
break;
|
|
6457
|
+
}
|
|
6458
|
+
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
6459
|
+
if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
|
|
6460
|
+
continue;
|
|
6461
|
+
const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
|
|
6462
|
+
const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
|
|
6463
|
+
const dependencyStep = byId.get(dependencyId);
|
|
6464
|
+
if (dependencyRun?.status === "succeeded")
|
|
6465
|
+
return false;
|
|
6466
|
+
return !dependencyStep?.continueOnFailure;
|
|
6467
|
+
});
|
|
6468
|
+
if (blockedBy) {
|
|
6469
|
+
opts.beforePersist?.();
|
|
6470
|
+
if (isBlockedStepRun(store.getWorkflowStepRun(run.id, blockedBy))) {
|
|
6471
|
+
store.skipWorkflowStepRun(run.id, step.id, `${BLOCKED_STEP_ERROR_PREFIX} upstream step ${blockedBy} was blocked`, {
|
|
6472
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6473
|
+
});
|
|
6474
|
+
continue;
|
|
6475
|
+
}
|
|
6476
|
+
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
6477
|
+
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
6478
|
+
terminalStatus = "failed";
|
|
6382
6479
|
continue;
|
|
6383
6480
|
}
|
|
6384
|
-
|
|
6385
|
-
|
|
6386
|
-
|
|
6387
|
-
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
|
|
6394
|
-
|
|
6395
|
-
|
|
6396
|
-
|
|
6397
|
-
|
|
6398
|
-
|
|
6399
|
-
|
|
6400
|
-
|
|
6401
|
-
|
|
6402
|
-
|
|
6403
|
-
});
|
|
6404
|
-
let result;
|
|
6405
|
-
const controller = new AbortController;
|
|
6406
|
-
const externalAbort = () => controller.abort();
|
|
6407
|
-
if (opts.signal?.aborted)
|
|
6408
|
-
controller.abort();
|
|
6409
|
-
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
6410
|
-
const cancelTimer = setInterval(() => {
|
|
6411
|
-
if (store.getWorkflowRun(run.id)?.status === "cancelled")
|
|
6481
|
+
opts.beforePersist?.();
|
|
6482
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
|
|
6483
|
+
if (startedStep.status !== "running") {
|
|
6484
|
+
terminalStatus = "failed";
|
|
6485
|
+
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
6486
|
+
break;
|
|
6487
|
+
}
|
|
6488
|
+
const stepContext = goalExecutionContext({
|
|
6489
|
+
loop: opts.loop,
|
|
6490
|
+
loopRun: opts.loopRun,
|
|
6491
|
+
scheduledFor: opts.scheduledFor,
|
|
6492
|
+
workflow,
|
|
6493
|
+
workflowRunId: run.id,
|
|
6494
|
+
workflowStepId: step.id
|
|
6495
|
+
});
|
|
6496
|
+
let result;
|
|
6497
|
+
const controller = new AbortController;
|
|
6498
|
+
const externalAbort = () => controller.abort();
|
|
6499
|
+
if (opts.signal?.aborted)
|
|
6412
6500
|
controller.abort();
|
|
6413
|
-
|
|
6414
|
-
|
|
6415
|
-
|
|
6416
|
-
|
|
6417
|
-
|
|
6418
|
-
|
|
6419
|
-
|
|
6420
|
-
|
|
6421
|
-
|
|
6422
|
-
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
opts
|
|
6431
|
-
|
|
6432
|
-
|
|
6433
|
-
|
|
6501
|
+
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
6502
|
+
const cancelTimer = setInterval(() => {
|
|
6503
|
+
if (store.getWorkflowRun(run.id)?.status === "cancelled")
|
|
6504
|
+
controller.abort();
|
|
6505
|
+
}, opts.cancelPollMs ?? 500);
|
|
6506
|
+
cancelTimer.unref();
|
|
6507
|
+
try {
|
|
6508
|
+
if (step.goal) {
|
|
6509
|
+
result = await runGoal(store, step.goal, {
|
|
6510
|
+
...opts,
|
|
6511
|
+
model: opts.goalModel,
|
|
6512
|
+
target: targetWithStepAccount(step),
|
|
6513
|
+
signal: controller.signal,
|
|
6514
|
+
context: stepContext
|
|
6515
|
+
});
|
|
6516
|
+
} else {
|
|
6517
|
+
result = await executeTarget(targetWithStepAccount(step, opts.goalNodePrompt), executionMetadata(stepContext), {
|
|
6518
|
+
...opts,
|
|
6519
|
+
machine: opts.machine ?? opts.loop?.machine,
|
|
6520
|
+
signal: controller.signal,
|
|
6521
|
+
onAgentProgress: (progress) => {
|
|
6522
|
+
const stdout = JSON.stringify({ agentProgress: progress }, null, 2);
|
|
6523
|
+
opts.beforePersist?.();
|
|
6524
|
+
store.recordWorkflowStepProgress(run.id, step.id, {
|
|
6525
|
+
stdout,
|
|
6526
|
+
payload: progress
|
|
6527
|
+
}, {
|
|
6528
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6529
|
+
});
|
|
6530
|
+
opts.onAgentProgress?.(progress);
|
|
6531
|
+
},
|
|
6532
|
+
onSpawn: (pid) => {
|
|
6533
|
+
opts.beforePersist?.();
|
|
6534
|
+
store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
|
|
6535
|
+
opts.onSpawn?.(pid);
|
|
6536
|
+
}
|
|
6537
|
+
});
|
|
6538
|
+
}
|
|
6539
|
+
} catch (error) {
|
|
6540
|
+
const finishedAt2 = nowIso();
|
|
6541
|
+
result = {
|
|
6542
|
+
status: "failed",
|
|
6543
|
+
stdout: "",
|
|
6544
|
+
stderr: "",
|
|
6545
|
+
error: error instanceof Error ? error.message : String(error),
|
|
6546
|
+
startedAt: startedStep.startedAt ?? finishedAt2,
|
|
6547
|
+
finishedAt: finishedAt2,
|
|
6548
|
+
durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
|
|
6549
|
+
};
|
|
6550
|
+
} finally {
|
|
6551
|
+
clearInterval(cancelTimer);
|
|
6552
|
+
opts.signal?.removeEventListener("abort", externalAbort);
|
|
6553
|
+
}
|
|
6554
|
+
const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
6555
|
+
if (timeoutMessage && result.status === "failed") {
|
|
6556
|
+
result = { ...result, status: "timed_out", error: timeoutMessage };
|
|
6557
|
+
}
|
|
6558
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6559
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
6560
|
+
blockingError = "workflow run was cancelled";
|
|
6561
|
+
break;
|
|
6562
|
+
}
|
|
6563
|
+
opts.beforePersist?.();
|
|
6564
|
+
const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
|
|
6565
|
+
if (blockedExit) {
|
|
6566
|
+
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
6567
|
+
status: "skipped",
|
|
6568
|
+
finishedAt: result.finishedAt,
|
|
6569
|
+
durationMs: result.durationMs,
|
|
6570
|
+
stdout: result.stdout,
|
|
6571
|
+
stderr: result.stderr,
|
|
6572
|
+
exitCode: result.exitCode,
|
|
6573
|
+
error: `${BLOCKED_STEP_ERROR_PREFIX} step ${step.id} exited with blocked exit code ${result.exitCode}`
|
|
6574
|
+
}, {
|
|
6575
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6434
6576
|
});
|
|
6577
|
+
continue;
|
|
6435
6578
|
}
|
|
6436
|
-
} catch (error) {
|
|
6437
|
-
const finishedAt2 = nowIso();
|
|
6438
|
-
result = {
|
|
6439
|
-
status: "failed",
|
|
6440
|
-
stdout: "",
|
|
6441
|
-
stderr: "",
|
|
6442
|
-
error: error instanceof Error ? error.message : String(error),
|
|
6443
|
-
startedAt: startedStep.startedAt ?? finishedAt2,
|
|
6444
|
-
finishedAt: finishedAt2,
|
|
6445
|
-
durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
|
|
6446
|
-
};
|
|
6447
|
-
} finally {
|
|
6448
|
-
clearInterval(cancelTimer);
|
|
6449
|
-
opts.signal?.removeEventListener("abort", externalAbort);
|
|
6450
|
-
}
|
|
6451
|
-
const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
6452
|
-
if (timeoutMessage && result.status === "failed") {
|
|
6453
|
-
result = { ...result, status: "timed_out", error: timeoutMessage };
|
|
6454
|
-
}
|
|
6455
|
-
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6456
|
-
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
6457
|
-
blockingError = "workflow run was cancelled";
|
|
6458
|
-
break;
|
|
6459
|
-
}
|
|
6460
|
-
opts.beforePersist?.();
|
|
6461
|
-
const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
|
|
6462
|
-
if (blockedExit) {
|
|
6463
6579
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
6464
|
-
status:
|
|
6580
|
+
status: result.status,
|
|
6465
6581
|
finishedAt: result.finishedAt,
|
|
6466
6582
|
durationMs: result.durationMs,
|
|
6467
6583
|
stdout: result.stdout,
|
|
6468
6584
|
stderr: result.stderr,
|
|
6469
6585
|
exitCode: result.exitCode,
|
|
6470
|
-
error:
|
|
6586
|
+
error: result.error
|
|
6471
6587
|
}, {
|
|
6472
6588
|
daemonLeaseId: opts.daemonLeaseId
|
|
6473
6589
|
});
|
|
6474
|
-
|
|
6590
|
+
if (result.status !== "succeeded" && !step.continueOnFailure) {
|
|
6591
|
+
terminalStatus = result.status;
|
|
6592
|
+
blockingError = `step ${step.id} ${result.status}${result.error ? `: ${result.error}` : ""}`;
|
|
6593
|
+
break;
|
|
6594
|
+
}
|
|
6475
6595
|
}
|
|
6476
|
-
|
|
6477
|
-
|
|
6478
|
-
|
|
6479
|
-
|
|
6480
|
-
|
|
6481
|
-
|
|
6482
|
-
|
|
6483
|
-
|
|
6596
|
+
if (terminalStatus !== "succeeded") {
|
|
6597
|
+
for (const step of ordered) {
|
|
6598
|
+
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
6599
|
+
if (existing?.status === "pending" || existing?.status === "running") {
|
|
6600
|
+
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
|
|
6601
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6602
|
+
});
|
|
6603
|
+
}
|
|
6604
|
+
}
|
|
6605
|
+
}
|
|
6606
|
+
const finishedAt = nowIso();
|
|
6607
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6608
|
+
const terminalRun = store.requireWorkflowRun(run.id);
|
|
6609
|
+
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
|
|
6610
|
+
}
|
|
6611
|
+
opts.beforePersist?.();
|
|
6612
|
+
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
6613
|
+
finishedAt,
|
|
6614
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
6615
|
+
error: blockingError
|
|
6484
6616
|
}, {
|
|
6485
6617
|
daemonLeaseId: opts.daemonLeaseId
|
|
6486
6618
|
});
|
|
6487
|
-
|
|
6488
|
-
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
if (
|
|
6497
|
-
store.
|
|
6619
|
+
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
|
|
6620
|
+
} catch (error) {
|
|
6621
|
+
if (opts.daemonLeaseId && error instanceof Error && error.message === "daemon lease lost")
|
|
6622
|
+
throw error;
|
|
6623
|
+
if (error instanceof Error && error.message.includes("workflow step is not claimable"))
|
|
6624
|
+
throw error;
|
|
6625
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6626
|
+
const finishedAt = nowIso();
|
|
6627
|
+
try {
|
|
6628
|
+
if (!store.isWorkflowRunTerminal(run.id)) {
|
|
6629
|
+
store.finalizeWorkflowRun(run.id, "failed", {
|
|
6630
|
+
finishedAt,
|
|
6631
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
6632
|
+
error: message
|
|
6633
|
+
}, {
|
|
6498
6634
|
daemonLeaseId: opts.daemonLeaseId
|
|
6499
6635
|
});
|
|
6500
6636
|
}
|
|
6501
|
-
}
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
const terminalRun = store.requireWorkflowRun(run.id);
|
|
6506
|
-
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
|
|
6637
|
+
} catch {}
|
|
6638
|
+
const current = store.getWorkflowRun(run.id) ?? run;
|
|
6639
|
+
const resultStatus = current.status === "running" ? "failed" : current.status;
|
|
6640
|
+
return workflowResult(current, resultStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), current.error ?? message);
|
|
6507
6641
|
}
|
|
6508
|
-
opts.beforePersist?.();
|
|
6509
|
-
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
6510
|
-
finishedAt,
|
|
6511
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
6512
|
-
error: blockingError
|
|
6513
|
-
}, {
|
|
6514
|
-
daemonLeaseId: opts.daemonLeaseId
|
|
6515
|
-
});
|
|
6516
|
-
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
|
|
6517
6642
|
}
|
|
6518
6643
|
function preflightWorkflow(workflow, opts = {}) {
|
|
6519
6644
|
return workflowExecutionOrder(workflow).map((step) => {
|
|
@@ -6935,7 +7060,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6935
7060
|
|
|
6936
7061
|
`);
|
|
6937
7062
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6938
|
-
const tags = ["bug", "openloops", "loop-health", failure.classification];
|
|
7063
|
+
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6939
7064
|
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
6940
7065
|
return {
|
|
6941
7066
|
title,
|
|
@@ -6950,7 +7075,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6950
7075
|
comment: ["todos", "comment", "<task-id>", description]
|
|
6951
7076
|
},
|
|
6952
7077
|
futureNativeUpsert: {
|
|
6953
|
-
command: "todos upsert",
|
|
7078
|
+
command: "todos task upsert",
|
|
6954
7079
|
fields: {
|
|
6955
7080
|
title,
|
|
6956
7081
|
description,
|
|
@@ -7309,6 +7434,25 @@ function advanceOptions(deps) {
|
|
|
7309
7434
|
onRun: deps.onRun
|
|
7310
7435
|
};
|
|
7311
7436
|
}
|
|
7437
|
+
var TERMINAL_RUN_STATUSES2 = new Set([
|
|
7438
|
+
"succeeded",
|
|
7439
|
+
"failed",
|
|
7440
|
+
"timed_out",
|
|
7441
|
+
"abandoned",
|
|
7442
|
+
"skipped"
|
|
7443
|
+
]);
|
|
7444
|
+
function repairWedgedTerminalSlot(deps, loop, scheduledFor, now) {
|
|
7445
|
+
const existing = deps.store.getRunBySlot(loop.id, scheduledFor);
|
|
7446
|
+
if (!existing || !TERMINAL_RUN_STATUSES2.has(existing.status))
|
|
7447
|
+
return;
|
|
7448
|
+
try {
|
|
7449
|
+
advanceLoop(deps.store, loop, existing, new Date(existing.finishedAt ?? now), existing.status === "succeeded", advanceOptions(deps));
|
|
7450
|
+
} catch (error) {
|
|
7451
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
7452
|
+
return;
|
|
7453
|
+
throw error;
|
|
7454
|
+
}
|
|
7455
|
+
}
|
|
7312
7456
|
async function runSlot(deps, loop, scheduledFor) {
|
|
7313
7457
|
const now = deps.now?.() ?? new Date;
|
|
7314
7458
|
deps.beforeRun?.(loop, scheduledFor);
|
|
@@ -7335,8 +7479,10 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
7335
7479
|
return;
|
|
7336
7480
|
throw error;
|
|
7337
7481
|
}
|
|
7338
|
-
if (!claim)
|
|
7482
|
+
if (!claim) {
|
|
7483
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
7339
7484
|
return;
|
|
7485
|
+
}
|
|
7340
7486
|
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
7341
7487
|
deps.onRun?.(claim.run);
|
|
7342
7488
|
const finalRun = await executeClaimedRun({
|
|
@@ -7382,8 +7528,10 @@ function claimSlot(deps, loop, scheduledFor) {
|
|
|
7382
7528
|
return;
|
|
7383
7529
|
throw error;
|
|
7384
7530
|
}
|
|
7385
|
-
if (!claim)
|
|
7531
|
+
if (!claim) {
|
|
7532
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
7386
7533
|
return;
|
|
7534
|
+
}
|
|
7387
7535
|
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
7388
7536
|
deps.onRun?.(claim.run);
|
|
7389
7537
|
return claim;
|
|
@@ -7664,10 +7812,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7664
7812
|
const store = new Store(join5(dirname4(path), "loops.db"));
|
|
7665
7813
|
try {
|
|
7666
7814
|
const lease = store.getDaemonLease();
|
|
7667
|
-
|
|
7668
|
-
removePid(path);
|
|
7669
|
-
return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
|
|
7670
|
-
}
|
|
7815
|
+
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
7671
7816
|
try {
|
|
7672
7817
|
process.kill(state.pid, "SIGTERM");
|
|
7673
7818
|
} catch {
|
|
@@ -7687,11 +7832,14 @@ async function stopDaemon(opts = {}) {
|
|
|
7687
7832
|
} catch {}
|
|
7688
7833
|
await sleep2(150);
|
|
7689
7834
|
removePid(path);
|
|
7690
|
-
|
|
7691
|
-
|
|
7692
|
-
|
|
7693
|
-
|
|
7694
|
-
|
|
7835
|
+
let reapedPgids = [];
|
|
7836
|
+
if (leaseVerified && lease) {
|
|
7837
|
+
const ownedRuns = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.claimedBy !== undefined && run.claimedBy.endsWith(`:${lease.id}`));
|
|
7838
|
+
reapedPgids = await reapProcessGroups(ownedRuns.map(toReapableProcess), {
|
|
7839
|
+
sleep: sleep2,
|
|
7840
|
+
graceMs: opts.reapGraceMs
|
|
7841
|
+
});
|
|
7842
|
+
}
|
|
7695
7843
|
return { wasRunning: true, stopped: !isAlive(state.pid, record?.startedAt), forced: true, pid: state.pid, reapedPgids };
|
|
7696
7844
|
} finally {
|
|
7697
7845
|
store.close();
|
|
@@ -7879,7 +8027,14 @@ async function runDaemon(opts = {}) {
|
|
|
7879
8027
|
try {
|
|
7880
8028
|
const liveInlineOwner = (run) => {
|
|
7881
8029
|
const ownerPid = inlineRunnerOwnerPid(run.claimedBy);
|
|
7882
|
-
|
|
8030
|
+
if (ownerPid === undefined || !isAlive(ownerPid))
|
|
8031
|
+
return false;
|
|
8032
|
+
const ownerStartMs = processStartTimeMs(ownerPid);
|
|
8033
|
+
const runStartMs = run.startedAt ? Date.parse(run.startedAt) : Number.NaN;
|
|
8034
|
+
if (ownerStartMs !== undefined && Number.isFinite(runStartMs) && ownerStartMs > runStartMs + START_TIME_TOLERANCE_MS) {
|
|
8035
|
+
return false;
|
|
8036
|
+
}
|
|
8037
|
+
return true;
|
|
7883
8038
|
};
|
|
7884
8039
|
const staleCandidates = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.leaseExpiresAt !== undefined && new Date(run.leaseExpiresAt).getTime() <= Date.now());
|
|
7885
8040
|
const startup = claimDueRuns({ store, runnerId, daemonLeaseId: leaseId, maxClaims: 0 });
|
|
@@ -9164,6 +9319,9 @@ function roleFragment(role, flow) {
|
|
|
9164
9319
|
function goalHeaderFragment(goal, role, flow) {
|
|
9165
9320
|
return [`/goal ${goal}`, "", roleFragment(role, flow)];
|
|
9166
9321
|
}
|
|
9322
|
+
function boundedStepHeaderFragment(objective, role, flow) {
|
|
9323
|
+
return [`Objective: ${objective}`, "", roleFragment(role, flow)];
|
|
9324
|
+
}
|
|
9167
9325
|
function adversarialReviewFragment(inspect, focus) {
|
|
9168
9326
|
return `Use fresh context. Inspect ${inspect}. Act as an adversarial reviewer focused on ${focus}.`;
|
|
9169
9327
|
}
|
|
@@ -10296,7 +10454,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10296
10454
|
const blockTaskCommand = `todos --project ${todosProjectPath} update ${input.taskId} --status blocked`;
|
|
10297
10455
|
const gateStopFragment = (stage, stops) => `The deterministic ${stage} gate will stop ${stops} unless the latest ${stage} marker is the exact go marker and the task has no blocked/completed/done/cancelled/failed/archived/no-auto/manual/approval-required state.`;
|
|
10298
10456
|
const triagePrompt = [
|
|
10299
|
-
...
|
|
10457
|
+
...boundedStepHeaderFragment(`Triage todos task ${input.taskId} for safe automated execution.`, "triage", "lifecycle"),
|
|
10300
10458
|
shared,
|
|
10301
10459
|
"Decide whether the task is eligible for loop execution. Check status, dependencies, duplicate tasks, no-auto/manual/approval metadata, project path, acceptance criteria, and whether the requested work should be split before implementation.",
|
|
10302
10460
|
"Do not implement repo changes in this step.",
|
|
@@ -10308,7 +10466,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10308
10466
|
].join(`
|
|
10309
10467
|
`);
|
|
10310
10468
|
const plannerPrompt = [
|
|
10311
|
-
...
|
|
10469
|
+
...boundedStepHeaderFragment(`Plan todos task ${input.taskId} before implementation.`, "planner", "lifecycle"),
|
|
10312
10470
|
shared,
|
|
10313
10471
|
"Read the triage comment and current task details.",
|
|
10314
10472
|
`If the task is ready for implementation, add a task comment whose first line is exactly: ${gateMarker("planner", "go")}`,
|
|
@@ -10319,7 +10477,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10319
10477
|
].join(`
|
|
10320
10478
|
`);
|
|
10321
10479
|
const workerPrompt = [
|
|
10322
|
-
...
|
|
10480
|
+
...boundedStepHeaderFragment(`Complete todos task ${input.taskId} according to the planner evidence.`, "worker", "lifecycle"),
|
|
10323
10481
|
shared,
|
|
10324
10482
|
todosStartLine(todosProjectPath, input.taskId),
|
|
10325
10483
|
"Read the triage and planner comments first. Implement only the scoped task, run focused validation, and record changed files, commits, evidence, blockers, and residual risks.",
|
|
@@ -10328,7 +10486,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10328
10486
|
].filter(Boolean).join(`
|
|
10329
10487
|
`);
|
|
10330
10488
|
const verifierPrompt = [
|
|
10331
|
-
...
|
|
10489
|
+
...boundedStepHeaderFragment(`Verify todos task ${input.taskId} after the full lifecycle worker step.`, "verifier", "lifecycle"),
|
|
10332
10490
|
shared,
|
|
10333
10491
|
todosVerificationLine(todosProjectPath, input.taskId),
|
|
10334
10492
|
todosDoneLine(todosProjectPath, input.taskId),
|
|
@@ -11034,7 +11192,7 @@ function taskRouteEligibility(data, metadata) {
|
|
|
11034
11192
|
const records = taskEventRecords(data, metadata);
|
|
11035
11193
|
const automation = automationRecords(data, metadata);
|
|
11036
11194
|
const tags = taskEventTags(records);
|
|
11037
|
-
const hasRouteOptIn = tags.includes("auto:route") || hasTruthyField(records, ["route_enabled", "routeEnabled", "automation_allowed", "automationAllowed"]) || hasTruthyField(automation, ["allowed"]);
|
|
11195
|
+
const hasRouteOptIn = tags.includes("auto:route") || tags.includes("route:enabled") || hasTruthyField(records, ["route_enabled", "routeEnabled", "automation_allowed", "automationAllowed"]) || hasTruthyField(automation, ["allowed"]);
|
|
11038
11196
|
if (!hasRouteOptIn)
|
|
11039
11197
|
return { eligible: false, reason: "missing explicit route opt-in", tags };
|
|
11040
11198
|
const status = taskEventField(data, ["status", "task_status", "taskStatus"])?.toLowerCase();
|
|
@@ -12335,7 +12493,8 @@ function compactDrainResult(result) {
|
|
|
12335
12493
|
workflowName: stringField2(workflow?.name),
|
|
12336
12494
|
providerRouting,
|
|
12337
12495
|
requeue,
|
|
12338
|
-
queuedAtSource: value.queuedAtSource
|
|
12496
|
+
queuedAtSource: value.queuedAtSource,
|
|
12497
|
+
fatal: value.fatal === true ? true : undefined
|
|
12339
12498
|
};
|
|
12340
12499
|
}
|
|
12341
12500
|
function loadReadyTodosTasks(opts, scanLimit) {
|
|
@@ -12455,10 +12614,12 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12455
12614
|
result = routeTodosTaskEvent(event, opts);
|
|
12456
12615
|
} catch (error) {
|
|
12457
12616
|
const message = error instanceof Error ? error.message : String(error);
|
|
12458
|
-
if (
|
|
12459
|
-
|
|
12460
|
-
|
|
12461
|
-
|
|
12617
|
+
if (isSkippableDrainRouteError(message)) {
|
|
12618
|
+
const sourceTaskUpdate = opts.dryRun ? { attempted: false, reason: "dry-run" } : markInvalidDrainTaskNonRouteable(todosProject, task, message);
|
|
12619
|
+
result = skippedDrainTask(task, event, redact(message, 640) ?? "route task failed", { sourceTaskUpdate });
|
|
12620
|
+
} else {
|
|
12621
|
+
result = skippedDrainTask(task, event, redact(message, 640) ?? "route task failed", { fatal: true });
|
|
12622
|
+
}
|
|
12462
12623
|
}
|
|
12463
12624
|
results.push(result);
|
|
12464
12625
|
if (result.kind === "created")
|
|
@@ -12485,6 +12646,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12485
12646
|
deduped: results.filter((result) => result.kind === "deduped").length,
|
|
12486
12647
|
throttled: results.filter((result) => result.kind === "throttled").length,
|
|
12487
12648
|
skipped: results.filter((result) => result.kind === "skipped").length,
|
|
12649
|
+
fatal: results.filter((result) => result.value.fatal === true).length,
|
|
12488
12650
|
maxDispatch,
|
|
12489
12651
|
source: "todos ready",
|
|
12490
12652
|
dryRun: Boolean(opts.dryRun),
|
|
@@ -12512,6 +12674,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12512
12674
|
deduped: report.deduped,
|
|
12513
12675
|
throttled: report.throttled,
|
|
12514
12676
|
skipped: report.skipped,
|
|
12677
|
+
fatal: report.fatal,
|
|
12515
12678
|
maxDispatch: report.maxDispatch,
|
|
12516
12679
|
source: report.source,
|
|
12517
12680
|
dryRun: report.dryRun,
|
|
@@ -12520,7 +12683,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12520
12683
|
} : { ...report, evidencePath };
|
|
12521
12684
|
return {
|
|
12522
12685
|
value,
|
|
12523
|
-
human: `drained todos ready queue: considered=${report.considered} created=${report.created} deduped=${report.deduped} throttled=${report.throttled} skipped=${report.skipped}`
|
|
12686
|
+
human: `drained todos ready queue: considered=${report.considered} created=${report.created} deduped=${report.deduped} throttled=${report.throttled} skipped=${report.skipped} fatal=${report.fatal}`
|
|
12524
12687
|
};
|
|
12525
12688
|
}
|
|
12526
12689
|
// src/lib/route/route-tasks.ts
|
|
@@ -13721,6 +13884,11 @@ function handleRouteDrain(kind, opts) {
|
|
|
13721
13884
|
throw new ValidationError("route drain currently supports kind todos-task");
|
|
13722
13885
|
const result = drainTodosTaskRoutes(opts);
|
|
13723
13886
|
print(result.value, result.human);
|
|
13887
|
+
const fatal = Number(result.value.fatal ?? 0);
|
|
13888
|
+
if (fatal > 0) {
|
|
13889
|
+
console.error(`route drain hit ${fatal} non-skippable task error(s); see evidence file`);
|
|
13890
|
+
process.exitCode = 1;
|
|
13891
|
+
}
|
|
13724
13892
|
}
|
|
13725
13893
|
addRouteEventOptions(routes.command("preview <kind>").description("(deprecated: use 'routes create <kind> --dry-run') preview a route-created workflow invocation without storing it")).action(runAction(async (kind, opts) => handleRouteEvent(kind, { ...opts, dryRun: true })));
|
|
13726
13894
|
addRouteEventOptions(routes.command("create <kind>").description("create a route workflow invocation and admit it when capacity allows"), { dryRunDescription: "preview the generated workflow invocation and loop without storing anything" }).action(runAction(async (kind, opts) => handleRouteEvent(kind, opts)));
|
|
@@ -13922,7 +14090,7 @@ workflows.command("runs [idOrName]").description("list workflow runs, optionally
|
|
|
13922
14090
|
const store = new Store;
|
|
13923
14091
|
try {
|
|
13924
14092
|
const workflow = idOrName ? store.requireWorkflow(idOrName) : undefined;
|
|
13925
|
-
const runs = store.listWorkflowRuns({ workflowId: workflow?.id, limit:
|
|
14093
|
+
const runs = store.listWorkflowRuns({ workflowId: workflow?.id, limit: positiveInteger(opts.limit, "--limit") ?? 50 });
|
|
13926
14094
|
if (isJson())
|
|
13927
14095
|
print(runs.map(publicWorkflowRun));
|
|
13928
14096
|
else {
|
|
@@ -13937,7 +14105,7 @@ workflows.command("runs [idOrName]").description("list workflow runs, optionally
|
|
|
13937
14105
|
workflows.command("events <runId>").description("list step/lifecycle events for a workflow run").option("--limit <n>", "limit", "200").action(runAction((runId, opts) => {
|
|
13938
14106
|
const store = new Store;
|
|
13939
14107
|
try {
|
|
13940
|
-
const runEvents = store.listWorkflowEvents(runId,
|
|
14108
|
+
const runEvents = store.listWorkflowEvents(runId, positiveInteger(opts.limit, "--limit") ?? 200);
|
|
13941
14109
|
if (isJson())
|
|
13942
14110
|
print(runEvents.map(publicWorkflowEvent));
|
|
13943
14111
|
else {
|
|
@@ -14178,7 +14346,7 @@ program.command("runs [idOrName]").description("list recent runs, optionally for
|
|
|
14178
14346
|
const store = new Store;
|
|
14179
14347
|
try {
|
|
14180
14348
|
const loop = idOrName ? store.requireLoop(idOrName) : undefined;
|
|
14181
|
-
const runs = store.listRuns({ loopId: loop?.id, limit:
|
|
14349
|
+
const runs = store.listRuns({ loopId: loop?.id, limit: positiveInteger(opts.limit, "--limit") ?? 50 });
|
|
14182
14350
|
if (isJson())
|
|
14183
14351
|
print(runs.map((run) => publicRun(run, opts.showOutput)));
|
|
14184
14352
|
else {
|
|
@@ -14195,7 +14363,7 @@ program.command("runs [idOrName]").description("list recent runs, optionally for
|
|
|
14195
14363
|
program.command("expectations [idOrName]").description("evaluate deterministic loop expectations without mutating external task systems").option("--limit <n>", "maximum loops to inspect when no loop is specified", "200").action(runAction((idOrName, opts) => {
|
|
14196
14364
|
const store = new Store;
|
|
14197
14365
|
try {
|
|
14198
|
-
const loops = idOrName ? [store.requireLoop(idOrName)] : store.listLoops({ limit:
|
|
14366
|
+
const loops = idOrName ? [store.requireLoop(idOrName)] : store.listLoops({ limit: positiveInteger(opts.limit, "--limit") ?? 200 });
|
|
14199
14367
|
const values = loops.map((loop) => expectationForLoop(store, loop));
|
|
14200
14368
|
if (isJson())
|
|
14201
14369
|
console.log(JSON.stringify(idOrName ? values[0] : values, null, 2));
|
|
@@ -14233,7 +14401,7 @@ var health = program.command("health").description("summarize loop health and la
|
|
|
14233
14401
|
health.command("route-tasks").description("upsert deduped todos tasks for failed loop health expectations").option("--project <path>", "todos project path", defaultLoopsProject()).option("--task-list <slug>", "todos task-list slug", "loop-error-self-heal").option("--limit <n>", "maximum loops to inspect", "200").option("--max-actions <n>", "maximum todos tasks to upsert", "5").option("--include-inactive", "also route stopped or expired loops").option("--auto-route", "opt routed tasks into task-created headless worker/verifier automation").option("--route-project-path <path>", "fallback project path for --auto-route when the failed loop has no cwd").option("--evidence-dir <path>", "write the route result JSON to this directory").option("--dry-run", "print intended task upserts without mutating todos").action(runAction((opts) => {
|
|
14234
14402
|
const store = new Store;
|
|
14235
14403
|
try {
|
|
14236
|
-
const report = buildHealthReport(store, { limit:
|
|
14404
|
+
const report = buildHealthReport(store, { limit: positiveInteger(opts.limit, "--limit") ?? 200, includeInactive: Boolean(opts.includeInactive) });
|
|
14237
14405
|
const failures = report.expectations.filter((entry) => !entry.ok && entry.recommendedTask);
|
|
14238
14406
|
const result = upsertRouteTasks({
|
|
14239
14407
|
project: opts.project,
|
|
@@ -14246,7 +14414,7 @@ health.command("route-tasks").description("upsert deduped todos tasks for failed
|
|
|
14246
14414
|
autoRoute: Boolean(opts.autoRoute),
|
|
14247
14415
|
routeProjectPath: opts.routeProjectPath
|
|
14248
14416
|
}),
|
|
14249
|
-
maxActions:
|
|
14417
|
+
maxActions: positiveInteger(opts.maxActions, "--max-actions") ?? 5,
|
|
14250
14418
|
dryRun: Boolean(opts.dryRun),
|
|
14251
14419
|
autoRoute: Boolean(opts.autoRoute),
|
|
14252
14420
|
routeProjectPath: opts.routeProjectPath,
|
|
@@ -14300,7 +14468,7 @@ hygiene.command("names").description("check or apply canonical machine-/repo-pre
|
|
|
14300
14468
|
apply: false,
|
|
14301
14469
|
includeStopped: Boolean(opts.includeStopped),
|
|
14302
14470
|
includeInactive: Boolean(opts.includeInactive),
|
|
14303
|
-
limit:
|
|
14471
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14304
14472
|
});
|
|
14305
14473
|
let outputReport = report;
|
|
14306
14474
|
const backupPath = opts.apply && report.changed > 0 ? backupLoopsDatabase("name-hygiene") : undefined;
|
|
@@ -14309,7 +14477,7 @@ hygiene.command("names").description("check or apply canonical machine-/repo-pre
|
|
|
14309
14477
|
apply: true,
|
|
14310
14478
|
includeStopped: Boolean(opts.includeStopped),
|
|
14311
14479
|
includeInactive: Boolean(opts.includeInactive),
|
|
14312
|
-
limit:
|
|
14480
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14313
14481
|
});
|
|
14314
14482
|
} else if (opts.apply) {
|
|
14315
14483
|
outputReport = { ...report, applied: true };
|
|
@@ -14336,7 +14504,7 @@ hygiene.command("duplicates").description("detect duplicate/overlapping loops wi
|
|
|
14336
14504
|
try {
|
|
14337
14505
|
const report = buildDuplicateOverlapReport(store, {
|
|
14338
14506
|
includeInactive: Boolean(opts.includeInactive),
|
|
14339
|
-
limit:
|
|
14507
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14340
14508
|
});
|
|
14341
14509
|
if (isJson())
|
|
14342
14510
|
console.log(JSON.stringify(report, null, 2));
|
|
@@ -14358,7 +14526,7 @@ hygiene.command("scripts").description("inventory loops still backed by local ~/
|
|
|
14358
14526
|
const report = buildScriptInventoryReport(store, {
|
|
14359
14527
|
scriptsDir: opts.scriptsDir,
|
|
14360
14528
|
includeInactive: Boolean(opts.includeInactive),
|
|
14361
|
-
limit:
|
|
14529
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14362
14530
|
});
|
|
14363
14531
|
if (isJson())
|
|
14364
14532
|
console.log(JSON.stringify(report, null, 2));
|
|
@@ -14380,7 +14548,7 @@ hygiene.command("route-tasks").description("upsert deduped todos tasks for hygie
|
|
|
14380
14548
|
const route = buildHygieneRouteTasks(store, {
|
|
14381
14549
|
checks,
|
|
14382
14550
|
includeInactive: Boolean(opts.includeInactive),
|
|
14383
|
-
limit:
|
|
14551
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000,
|
|
14384
14552
|
scriptsDir: opts.scriptsDir
|
|
14385
14553
|
});
|
|
14386
14554
|
const result = upsertRouteTasks({
|
|
@@ -14394,7 +14562,7 @@ hygiene.command("route-tasks").description("upsert deduped todos tasks for hygie
|
|
|
14394
14562
|
autoRoute: Boolean(opts.autoRoute),
|
|
14395
14563
|
routeProjectPath: opts.routeProjectPath
|
|
14396
14564
|
}),
|
|
14397
|
-
maxActions:
|
|
14565
|
+
maxActions: positiveInteger(opts.maxActions, "--max-actions") ?? 10,
|
|
14398
14566
|
dryRun: Boolean(opts.dryRun),
|
|
14399
14567
|
autoRoute: Boolean(opts.autoRoute),
|
|
14400
14568
|
routeProjectPath: opts.routeProjectPath,
|
|
@@ -14434,7 +14602,7 @@ program.command("stop <idOrName>").description("stop a loop and clear its next s
|
|
|
14434
14602
|
program.command("rename <idOrName> <newName>").description("rename a loop without changing its id, schedule, runs, or history").action(runAction((idOrName, newName) => {
|
|
14435
14603
|
const store = new Store;
|
|
14436
14604
|
try {
|
|
14437
|
-
const loop = store.
|
|
14605
|
+
const loop = store.requireUniqueLoop(idOrName);
|
|
14438
14606
|
const oldName = loop.name;
|
|
14439
14607
|
const trimmed = String(newName).trim();
|
|
14440
14608
|
if (!trimmed)
|
|
@@ -14471,10 +14639,17 @@ backup=${backupPath ?? "skipped (recent rename backup exists)"}`);
|
|
|
14471
14639
|
function updateStatus(idOrName, status) {
|
|
14472
14640
|
const store = new Store;
|
|
14473
14641
|
try {
|
|
14474
|
-
const loop = store.
|
|
14642
|
+
const loop = store.requireUniqueLoop(idOrName);
|
|
14475
14643
|
if (loop.archivedAt)
|
|
14476
14644
|
throw new Error(`loop is archived; run 'loops unarchive ${idOrName}' first`);
|
|
14477
|
-
|
|
14645
|
+
let nextRunAt = loop.nextRunAt;
|
|
14646
|
+
if (status === "stopped") {
|
|
14647
|
+
nextRunAt = undefined;
|
|
14648
|
+
} else if (status === "active" && !loop.nextRunAt) {
|
|
14649
|
+
const now = new Date;
|
|
14650
|
+
nextRunAt = computeNextAfter(loop.schedule, now, now);
|
|
14651
|
+
}
|
|
14652
|
+
const updated = store.updateLoop(loop.id, { status, nextRunAt });
|
|
14478
14653
|
print(publicLoop(updated), `${updated.id} ${updated.status}`);
|
|
14479
14654
|
} finally {
|
|
14480
14655
|
store.close();
|
|
@@ -14483,7 +14658,7 @@ function updateStatus(idOrName, status) {
|
|
|
14483
14658
|
program.command("remove <idOrName>").alias("rm").description("delete a loop and its run history").action(runAction((idOrName) => {
|
|
14484
14659
|
const store = new Store;
|
|
14485
14660
|
try {
|
|
14486
|
-
const removed = store.deleteLoop(idOrName);
|
|
14661
|
+
const removed = store.deleteLoop(store.requireUniqueLoop(idOrName).id);
|
|
14487
14662
|
print({ removed }, removed ? "removed" : "not removed");
|
|
14488
14663
|
} finally {
|
|
14489
14664
|
store.close();
|
|
@@ -14510,7 +14685,7 @@ program.command("unarchive <idOrName>").alias("restore").description("restore an
|
|
|
14510
14685
|
program.command("run-now <idOrName>").description("claim and execute one loop run immediately").option("--show-output", "show stdout/stderr").action(runAction(async (idOrName, opts) => {
|
|
14511
14686
|
const store = new Store;
|
|
14512
14687
|
try {
|
|
14513
|
-
const loop = store.
|
|
14688
|
+
const loop = store.requireUniqueLoop(idOrName);
|
|
14514
14689
|
if (loop.archivedAt)
|
|
14515
14690
|
throw new Error(`loop is archived; run 'loops unarchive ${idOrName}' before running it`);
|
|
14516
14691
|
const runnerId = `manual:${process.pid}`;
|
|
@@ -14690,15 +14865,16 @@ ${result.enableResults.map((item) => `${item.command} -> ${item.status === 0 ? "
|
|
|
14690
14865
|
${result.instructions.join(`
|
|
14691
14866
|
`)}${enableText}`);
|
|
14692
14867
|
}));
|
|
14693
|
-
daemon.command("logs").description("print the tail of the daemon log").option("-n, --lines <n>", "lines", "80").action(runAction((opts) => {
|
|
14868
|
+
daemon.command("logs").description("print the tail of the daemon log").option("-n, --lines <n>", "lines", "80").option("--tail <n>", "alias for --lines").action(runAction((opts) => {
|
|
14694
14869
|
const path = daemonLogPath();
|
|
14695
14870
|
if (!existsSync10(path)) {
|
|
14696
14871
|
console.log("");
|
|
14697
14872
|
return;
|
|
14698
14873
|
}
|
|
14874
|
+
const count = positiveInteger(opts.tail ?? opts.lines, opts.tail !== undefined ? "--tail" : "--lines") ?? 80;
|
|
14699
14875
|
const lines = readFileSync10(path, "utf8").trimEnd().split(`
|
|
14700
14876
|
`);
|
|
14701
|
-
console.log(lines.slice(-
|
|
14877
|
+
console.log(lines.slice(-count).join(`
|
|
14702
14878
|
`));
|
|
14703
14879
|
}));
|
|
14704
14880
|
await program.parseAsync(process.argv);
|