@hasna/loops 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/README.md +2 -2
- package/dist/api/index.js +19 -10
- package/dist/cli/index.js +368 -193
- package/dist/daemon/index.js +317 -163
- package/dist/index.js +339 -175
- package/dist/lib/executor.d.ts +14 -1
- package/dist/lib/mode.js +1 -1
- package/dist/lib/storage/index.js +56 -5
- package/dist/lib/storage/sqlite.js +56 -5
- package/dist/lib/store.d.ts +7 -0
- package/dist/lib/store.js +56 -5
- package/dist/lib/template-kit.d.ts +6 -0
- package/dist/mcp/index.d.ts +1 -1
- package/dist/mcp/index.js +321 -166
- package/dist/runner/index.d.ts +3 -1
- package/dist/runner/index.js +130 -11
- package/dist/sdk/index.js +320 -167
- package/docs/USAGE.md +2 -2
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -1493,11 +1493,14 @@ function ensurePrivateStorePath(file) {
|
|
|
1493
1493
|
class Store {
|
|
1494
1494
|
db;
|
|
1495
1495
|
rootDir;
|
|
1496
|
+
memoryRootDir;
|
|
1496
1497
|
constructor(path) {
|
|
1497
1498
|
const file = path ?? dbPath();
|
|
1498
1499
|
if (file !== ":memory:")
|
|
1499
1500
|
ensurePrivateStorePath(file);
|
|
1500
1501
|
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
|
|
1502
|
+
if (file === ":memory:")
|
|
1503
|
+
this.memoryRootDir = this.rootDir;
|
|
1501
1504
|
this.db = new Database(file);
|
|
1502
1505
|
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
1503
1506
|
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
@@ -1642,7 +1645,6 @@ class Store {
|
|
|
1642
1645
|
CREATE INDEX IF NOT EXISTS idx_runs_status ON loop_runs(status);
|
|
1643
1646
|
CREATE INDEX IF NOT EXISTS idx_runs_status_lease ON loop_runs(status, lease_expires_at);
|
|
1644
1647
|
CREATE INDEX IF NOT EXISTS idx_runs_scheduled ON loop_runs(scheduled_for);
|
|
1645
|
-
CREATE INDEX IF NOT EXISTS idx_runs_claim_token ON loop_runs(claim_token) WHERE claim_token IS NOT NULL;
|
|
1646
1648
|
|
|
1647
1649
|
CREATE TABLE IF NOT EXISTS daemon_lease (
|
|
1648
1650
|
id TEXT PRIMARY KEY,
|
|
@@ -1951,9 +1953,12 @@ class Store {
|
|
|
1951
1953
|
const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1952
1954
|
if (rows.length === 0)
|
|
1953
1955
|
throw new LoopNotFoundError(idOrName);
|
|
1954
|
-
if (rows.length
|
|
1956
|
+
if (rows.length === 1)
|
|
1957
|
+
return rowToLoop(rows[0]);
|
|
1958
|
+
const active = this.db.query("SELECT * FROM loops WHERE name = ? AND archived_at IS NULL ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1959
|
+
if (active.length !== 1)
|
|
1955
1960
|
throw new AmbiguousNameError(idOrName);
|
|
1956
|
-
return rowToLoop(
|
|
1961
|
+
return rowToLoop(active[0]);
|
|
1957
1962
|
}
|
|
1958
1963
|
requireLoop(idOrName) {
|
|
1959
1964
|
return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
|
|
@@ -2294,6 +2299,7 @@ class Store {
|
|
|
2294
2299
|
return this.transact(() => {
|
|
2295
2300
|
const loop = this.requireLoop(idOrName);
|
|
2296
2301
|
this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
|
|
2302
|
+
this.db.query("DELETE FROM loop_runs WHERE loop_id = ?").run(loop.id);
|
|
2297
2303
|
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
2298
2304
|
return res.changes > 0;
|
|
2299
2305
|
});
|
|
@@ -2778,7 +2784,10 @@ class Store {
|
|
|
2778
2784
|
return rowToGoal(row);
|
|
2779
2785
|
}
|
|
2780
2786
|
if (context.sourceType && context.sourceId) {
|
|
2781
|
-
const
|
|
2787
|
+
const terminalPlaceholders = GOAL_TERMINAL.map(() => "?").join(", ");
|
|
2788
|
+
const row = this.db.query(`SELECT * FROM goals
|
|
2789
|
+
WHERE source_type = ? AND source_id = ? AND status NOT IN (${terminalPlaceholders})
|
|
2790
|
+
ORDER BY created_at DESC LIMIT 1`).get(context.sourceType, context.sourceId, ...GOAL_TERMINAL);
|
|
2782
2791
|
if (row)
|
|
2783
2792
|
return rowToGoal(row);
|
|
2784
2793
|
}
|
|
@@ -3198,6 +3207,41 @@ class Store {
|
|
|
3198
3207
|
throw new Error(`workflow step run not found after pid update: ${workflowRunId}/${stepId}`);
|
|
3199
3208
|
return run;
|
|
3200
3209
|
}
|
|
3210
|
+
recordWorkflowStepProgress(workflowRunId, stepId, progress, opts = {}) {
|
|
3211
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
3212
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
3213
|
+
try {
|
|
3214
|
+
const res = this.db.query(`UPDATE workflow_step_runs
|
|
3215
|
+
SET stdout=COALESCE($stdout, stdout),
|
|
3216
|
+
stderr=COALESCE($stderr, stderr),
|
|
3217
|
+
updated_at=$updated
|
|
3218
|
+
WHERE workflow_run_id=$workflowRunId AND step_id=$stepId AND status='running'
|
|
3219
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3220
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3221
|
+
))`).run({
|
|
3222
|
+
$workflowRunId: workflowRunId,
|
|
3223
|
+
$stepId: stepId,
|
|
3224
|
+
$stdout: progress.stdout === undefined ? null : scrubbedOrNull(progress.stdout),
|
|
3225
|
+
$stderr: progress.stderr === undefined ? null : scrubbedOrNull(progress.stderr),
|
|
3226
|
+
$updated: now,
|
|
3227
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3228
|
+
$now: now
|
|
3229
|
+
});
|
|
3230
|
+
if (res.changes === 1) {
|
|
3231
|
+
this.appendWorkflowEvent(workflowRunId, "step_progress", stepId, progress.payload);
|
|
3232
|
+
}
|
|
3233
|
+
this.db.exec("COMMIT");
|
|
3234
|
+
} catch (error) {
|
|
3235
|
+
try {
|
|
3236
|
+
this.db.exec("ROLLBACK");
|
|
3237
|
+
} catch {}
|
|
3238
|
+
throw error;
|
|
3239
|
+
}
|
|
3240
|
+
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
3241
|
+
if (!run)
|
|
3242
|
+
throw new Error(`workflow step run not found after progress update: ${workflowRunId}/${stepId}`);
|
|
3243
|
+
return run;
|
|
3244
|
+
}
|
|
3201
3245
|
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
3202
3246
|
return this.transact(() => {
|
|
3203
3247
|
const now = nowIso();
|
|
@@ -3635,7 +3679,8 @@ class Store {
|
|
|
3635
3679
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3636
3680
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3637
3681
|
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, claim_token=NULL, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3638
|
-
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3682
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3683
|
+
WHERE id=$id AND status='running'`).run(params);
|
|
3639
3684
|
const run = this.getRun(id);
|
|
3640
3685
|
if (!run)
|
|
3641
3686
|
throw new Error(`run not found after finalize: ${id}`);
|
|
@@ -4163,12 +4208,18 @@ class Store {
|
|
|
4163
4208
|
}
|
|
4164
4209
|
close() {
|
|
4165
4210
|
this.db.close();
|
|
4211
|
+
if (this.memoryRootDir) {
|
|
4212
|
+
try {
|
|
4213
|
+
rmSync2(this.memoryRootDir, { recursive: true, force: true });
|
|
4214
|
+
} catch {}
|
|
4215
|
+
this.memoryRootDir = undefined;
|
|
4216
|
+
}
|
|
4166
4217
|
}
|
|
4167
4218
|
}
|
|
4168
4219
|
// package.json
|
|
4169
4220
|
var package_default = {
|
|
4170
4221
|
name: "@hasna/loops",
|
|
4171
|
-
version: "0.4.
|
|
4222
|
+
version: "0.4.6",
|
|
4172
4223
|
description: "Persistent local loop and workflow runner for deterministic commands and headless AI coding agents",
|
|
4173
4224
|
type: "module",
|
|
4174
4225
|
main: "dist/index.js",
|
|
@@ -4883,7 +4934,12 @@ function notifySpawn(pid, opts) {
|
|
|
4883
4934
|
if (!pid)
|
|
4884
4935
|
return;
|
|
4885
4936
|
opts.onSpawn?.(pid);
|
|
4886
|
-
|
|
4937
|
+
const startedMs = processStartTimeMs(pid);
|
|
4938
|
+
opts.onSpawnProcess?.({
|
|
4939
|
+
pid,
|
|
4940
|
+
pgid: pid,
|
|
4941
|
+
processStartedAt: startedMs !== undefined ? new Date(startedMs).toISOString() : nowIso()
|
|
4942
|
+
});
|
|
4887
4943
|
}
|
|
4888
4944
|
function shellQuote(value) {
|
|
4889
4945
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
@@ -5238,6 +5294,13 @@ function numberField(value, key) {
|
|
|
5238
5294
|
function codewithAgentStatus(readJson) {
|
|
5239
5295
|
return stringField(recordField(readJson, "agent"), "status") ?? stringField(recordField(readJson, "statusSnapshot"), "status");
|
|
5240
5296
|
}
|
|
5297
|
+
function codewithAgentLastEventSeq(readJson) {
|
|
5298
|
+
const agentSeq = numberField(recordField(readJson, "agent"), "lastEventSeq");
|
|
5299
|
+
const snapshotSeq = numberField(recordField(readJson, "statusSnapshot"), "lastEventSeq");
|
|
5300
|
+
if (agentSeq !== undefined && snapshotSeq !== undefined)
|
|
5301
|
+
return Math.max(agentSeq, snapshotSeq);
|
|
5302
|
+
return agentSeq ?? snapshotSeq;
|
|
5303
|
+
}
|
|
5241
5304
|
function codewithAgentEvidence(startJson, readJson, logsJson) {
|
|
5242
5305
|
const agent = recordField(readJson, "agent") ?? recordField(startJson, "agent");
|
|
5243
5306
|
const statusSnapshot = recordField(readJson, "statusSnapshot");
|
|
@@ -5268,6 +5331,21 @@ function codewithAgentEvidence(startJson, readJson, logsJson) {
|
|
|
5268
5331
|
events
|
|
5269
5332
|
}, null, 2);
|
|
5270
5333
|
}
|
|
5334
|
+
function codewithAgentProgress(readJson) {
|
|
5335
|
+
const agent = recordField(readJson, "agent");
|
|
5336
|
+
const statusSnapshot = recordField(readJson, "statusSnapshot");
|
|
5337
|
+
return {
|
|
5338
|
+
provider: "codewith",
|
|
5339
|
+
agentId: stringField(agent, "agentId"),
|
|
5340
|
+
status: stringField(agent, "status") ?? stringField(statusSnapshot, "status"),
|
|
5341
|
+
summary: stringField(statusSnapshot, "summary"),
|
|
5342
|
+
statusReason: stringField(agent, "statusReason"),
|
|
5343
|
+
threadId: stringField(agent, "threadId"),
|
|
5344
|
+
rolloutPath: stringField(agent, "rolloutPath"),
|
|
5345
|
+
pid: numberField(agent, "pid"),
|
|
5346
|
+
lastEventSeq: codewithAgentLastEventSeq(readJson)
|
|
5347
|
+
};
|
|
5348
|
+
}
|
|
5271
5349
|
function sleep(ms) {
|
|
5272
5350
|
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
5273
5351
|
}
|
|
@@ -5302,6 +5380,7 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
|
|
|
5302
5380
|
if (!agentId) {
|
|
5303
5381
|
return failureResult(startedAt, "codewith agent start did not return agent.agentId", { stdout: start.stdout, stderr: stderr.value() });
|
|
5304
5382
|
}
|
|
5383
|
+
opts.onAgentProgress?.(codewithAgentProgress(startJson));
|
|
5305
5384
|
const stopAgent = async () => {
|
|
5306
5385
|
await spawnCapture(spec.command, codewithAgentControlArgs(target, "stop", agentId), {
|
|
5307
5386
|
cwd: spec.cwd,
|
|
@@ -5344,10 +5423,15 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
|
|
|
5344
5423
|
});
|
|
5345
5424
|
}
|
|
5346
5425
|
const status = codewithAgentStatus(lastReadJson);
|
|
5347
|
-
const fingerprint = JSON.stringify({
|
|
5426
|
+
const fingerprint = JSON.stringify({
|
|
5427
|
+
status,
|
|
5428
|
+
agentLastEventSeq: numberField(recordField(lastReadJson, "agent"), "lastEventSeq"),
|
|
5429
|
+
snapshot: recordField(lastReadJson, "statusSnapshot") ?? null
|
|
5430
|
+
});
|
|
5348
5431
|
if (fingerprint !== lastFingerprint) {
|
|
5349
5432
|
lastFingerprint = fingerprint;
|
|
5350
5433
|
lastProgressAt = Date.now();
|
|
5434
|
+
opts.onAgentProgress?.(codewithAgentProgress(lastReadJson));
|
|
5351
5435
|
}
|
|
5352
5436
|
if (status === "completed" || status === "failed" || status === "cancelled") {
|
|
5353
5437
|
const logs = await spawnCapture(spec.command, codewithAgentControlArgs(target, "logs", agentId), {
|
|
@@ -6348,173 +6432,213 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
6348
6432
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
6349
6433
|
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), store.listWorkflowStepRuns(run.id), run.error);
|
|
6350
6434
|
}
|
|
6435
|
+
const resumedRunningSteps = store.listWorkflowStepRuns(run.id).filter((step) => step.status === "running");
|
|
6436
|
+
if (resumedRunningSteps.length > 0 && resumedRunningSteps.every((step) => step.pid !== undefined)) {
|
|
6437
|
+
try {
|
|
6438
|
+
store.recoverWorkflowRun(run.id, "workflow run resumed after lease takeover");
|
|
6439
|
+
} catch {}
|
|
6440
|
+
}
|
|
6351
6441
|
const ordered = workflowExecutionOrder(workflow);
|
|
6352
6442
|
const byId = new Map(workflow.steps.map((step) => [step.id, step]));
|
|
6353
6443
|
let blockingError;
|
|
6354
6444
|
let terminalStatus = "succeeded";
|
|
6355
|
-
|
|
6356
|
-
|
|
6357
|
-
|
|
6358
|
-
|
|
6359
|
-
|
|
6360
|
-
|
|
6361
|
-
|
|
6362
|
-
|
|
6363
|
-
|
|
6364
|
-
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
6368
|
-
|
|
6369
|
-
|
|
6370
|
-
|
|
6371
|
-
const
|
|
6372
|
-
|
|
6373
|
-
|
|
6374
|
-
|
|
6375
|
-
|
|
6376
|
-
|
|
6377
|
-
|
|
6378
|
-
|
|
6379
|
-
|
|
6380
|
-
store.
|
|
6381
|
-
|
|
6382
|
-
|
|
6445
|
+
try {
|
|
6446
|
+
for (const step of ordered) {
|
|
6447
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6448
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
6449
|
+
blockingError = "workflow run was cancelled";
|
|
6450
|
+
break;
|
|
6451
|
+
}
|
|
6452
|
+
const pendingTimeout = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
6453
|
+
if (pendingTimeout) {
|
|
6454
|
+
terminalStatus = "timed_out";
|
|
6455
|
+
blockingError = pendingTimeout;
|
|
6456
|
+
break;
|
|
6457
|
+
}
|
|
6458
|
+
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
6459
|
+
if (existing?.status === "succeeded" || existing?.status === "skipped" || existing?.status === "cancelled")
|
|
6460
|
+
continue;
|
|
6461
|
+
const blockedBy = (step.dependsOn ?? []).find((dependencyId) => {
|
|
6462
|
+
const dependencyRun = store.getWorkflowStepRun(run.id, dependencyId);
|
|
6463
|
+
const dependencyStep = byId.get(dependencyId);
|
|
6464
|
+
if (dependencyRun?.status === "succeeded")
|
|
6465
|
+
return false;
|
|
6466
|
+
return !dependencyStep?.continueOnFailure;
|
|
6467
|
+
});
|
|
6468
|
+
if (blockedBy) {
|
|
6469
|
+
opts.beforePersist?.();
|
|
6470
|
+
if (isBlockedStepRun(store.getWorkflowStepRun(run.id, blockedBy))) {
|
|
6471
|
+
store.skipWorkflowStepRun(run.id, step.id, `${BLOCKED_STEP_ERROR_PREFIX} upstream step ${blockedBy} was blocked`, {
|
|
6472
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6473
|
+
});
|
|
6474
|
+
continue;
|
|
6475
|
+
}
|
|
6476
|
+
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
6477
|
+
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
6478
|
+
terminalStatus = "failed";
|
|
6383
6479
|
continue;
|
|
6384
6480
|
}
|
|
6385
|
-
|
|
6386
|
-
|
|
6387
|
-
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
|
|
6394
|
-
|
|
6395
|
-
|
|
6396
|
-
|
|
6397
|
-
|
|
6398
|
-
|
|
6399
|
-
|
|
6400
|
-
|
|
6401
|
-
|
|
6402
|
-
|
|
6403
|
-
|
|
6404
|
-
});
|
|
6405
|
-
let result;
|
|
6406
|
-
const controller = new AbortController;
|
|
6407
|
-
const externalAbort = () => controller.abort();
|
|
6408
|
-
if (opts.signal?.aborted)
|
|
6409
|
-
controller.abort();
|
|
6410
|
-
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
6411
|
-
const cancelTimer = setInterval(() => {
|
|
6412
|
-
if (store.getWorkflowRun(run.id)?.status === "cancelled")
|
|
6481
|
+
opts.beforePersist?.();
|
|
6482
|
+
const startedStep = store.startWorkflowStepRun(run.id, step.id, { daemonLeaseId: opts.daemonLeaseId });
|
|
6483
|
+
if (startedStep.status !== "running") {
|
|
6484
|
+
terminalStatus = "failed";
|
|
6485
|
+
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
6486
|
+
break;
|
|
6487
|
+
}
|
|
6488
|
+
const stepContext = goalExecutionContext({
|
|
6489
|
+
loop: opts.loop,
|
|
6490
|
+
loopRun: opts.loopRun,
|
|
6491
|
+
scheduledFor: opts.scheduledFor,
|
|
6492
|
+
workflow,
|
|
6493
|
+
workflowRunId: run.id,
|
|
6494
|
+
workflowStepId: step.id
|
|
6495
|
+
});
|
|
6496
|
+
let result;
|
|
6497
|
+
const controller = new AbortController;
|
|
6498
|
+
const externalAbort = () => controller.abort();
|
|
6499
|
+
if (opts.signal?.aborted)
|
|
6413
6500
|
controller.abort();
|
|
6414
|
-
|
|
6415
|
-
|
|
6416
|
-
|
|
6417
|
-
|
|
6418
|
-
|
|
6419
|
-
|
|
6420
|
-
|
|
6421
|
-
|
|
6422
|
-
|
|
6423
|
-
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
|
|
6427
|
-
|
|
6428
|
-
|
|
6429
|
-
|
|
6430
|
-
|
|
6431
|
-
opts
|
|
6432
|
-
|
|
6433
|
-
|
|
6434
|
-
|
|
6501
|
+
opts.signal?.addEventListener("abort", externalAbort, { once: true });
|
|
6502
|
+
const cancelTimer = setInterval(() => {
|
|
6503
|
+
if (store.getWorkflowRun(run.id)?.status === "cancelled")
|
|
6504
|
+
controller.abort();
|
|
6505
|
+
}, opts.cancelPollMs ?? 500);
|
|
6506
|
+
cancelTimer.unref();
|
|
6507
|
+
try {
|
|
6508
|
+
if (step.goal) {
|
|
6509
|
+
result = await runGoal(store, step.goal, {
|
|
6510
|
+
...opts,
|
|
6511
|
+
model: opts.goalModel,
|
|
6512
|
+
target: targetWithStepAccount(step),
|
|
6513
|
+
signal: controller.signal,
|
|
6514
|
+
context: stepContext
|
|
6515
|
+
});
|
|
6516
|
+
} else {
|
|
6517
|
+
result = await executeTarget(targetWithStepAccount(step, opts.goalNodePrompt), executionMetadata(stepContext), {
|
|
6518
|
+
...opts,
|
|
6519
|
+
machine: opts.machine ?? opts.loop?.machine,
|
|
6520
|
+
signal: controller.signal,
|
|
6521
|
+
onAgentProgress: (progress) => {
|
|
6522
|
+
const stdout = JSON.stringify({ agentProgress: progress }, null, 2);
|
|
6523
|
+
opts.beforePersist?.();
|
|
6524
|
+
store.recordWorkflowStepProgress(run.id, step.id, {
|
|
6525
|
+
stdout,
|
|
6526
|
+
payload: progress
|
|
6527
|
+
}, {
|
|
6528
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6529
|
+
});
|
|
6530
|
+
opts.onAgentProgress?.(progress);
|
|
6531
|
+
},
|
|
6532
|
+
onSpawn: (pid) => {
|
|
6533
|
+
opts.beforePersist?.();
|
|
6534
|
+
store.markWorkflowStepPid(run.id, step.id, pid, { daemonLeaseId: opts.daemonLeaseId });
|
|
6535
|
+
opts.onSpawn?.(pid);
|
|
6536
|
+
}
|
|
6537
|
+
});
|
|
6538
|
+
}
|
|
6539
|
+
} catch (error) {
|
|
6540
|
+
const finishedAt2 = nowIso();
|
|
6541
|
+
result = {
|
|
6542
|
+
status: "failed",
|
|
6543
|
+
stdout: "",
|
|
6544
|
+
stderr: "",
|
|
6545
|
+
error: error instanceof Error ? error.message : String(error),
|
|
6546
|
+
startedAt: startedStep.startedAt ?? finishedAt2,
|
|
6547
|
+
finishedAt: finishedAt2,
|
|
6548
|
+
durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
|
|
6549
|
+
};
|
|
6550
|
+
} finally {
|
|
6551
|
+
clearInterval(cancelTimer);
|
|
6552
|
+
opts.signal?.removeEventListener("abort", externalAbort);
|
|
6553
|
+
}
|
|
6554
|
+
const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
6555
|
+
if (timeoutMessage && result.status === "failed") {
|
|
6556
|
+
result = { ...result, status: "timed_out", error: timeoutMessage };
|
|
6557
|
+
}
|
|
6558
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6559
|
+
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
6560
|
+
blockingError = "workflow run was cancelled";
|
|
6561
|
+
break;
|
|
6562
|
+
}
|
|
6563
|
+
opts.beforePersist?.();
|
|
6564
|
+
const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
|
|
6565
|
+
if (blockedExit) {
|
|
6566
|
+
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
6567
|
+
status: "skipped",
|
|
6568
|
+
finishedAt: result.finishedAt,
|
|
6569
|
+
durationMs: result.durationMs,
|
|
6570
|
+
stdout: result.stdout,
|
|
6571
|
+
stderr: result.stderr,
|
|
6572
|
+
exitCode: result.exitCode,
|
|
6573
|
+
error: `${BLOCKED_STEP_ERROR_PREFIX} step ${step.id} exited with blocked exit code ${result.exitCode}`
|
|
6574
|
+
}, {
|
|
6575
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6435
6576
|
});
|
|
6577
|
+
continue;
|
|
6436
6578
|
}
|
|
6437
|
-
} catch (error) {
|
|
6438
|
-
const finishedAt2 = nowIso();
|
|
6439
|
-
result = {
|
|
6440
|
-
status: "failed",
|
|
6441
|
-
stdout: "",
|
|
6442
|
-
stderr: "",
|
|
6443
|
-
error: error instanceof Error ? error.message : String(error),
|
|
6444
|
-
startedAt: startedStep.startedAt ?? finishedAt2,
|
|
6445
|
-
finishedAt: finishedAt2,
|
|
6446
|
-
durationMs: new Date(finishedAt2).getTime() - new Date(startedStep.startedAt ?? finishedAt2).getTime()
|
|
6447
|
-
};
|
|
6448
|
-
} finally {
|
|
6449
|
-
clearInterval(cancelTimer);
|
|
6450
|
-
opts.signal?.removeEventListener("abort", externalAbort);
|
|
6451
|
-
}
|
|
6452
|
-
const timeoutMessage = opts.signal?.aborted ? opts.signalTimeoutMessage?.() : undefined;
|
|
6453
|
-
if (timeoutMessage && result.status === "failed") {
|
|
6454
|
-
result = { ...result, status: "timed_out", error: timeoutMessage };
|
|
6455
|
-
}
|
|
6456
|
-
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6457
|
-
terminalStatus = store.requireWorkflowRun(run.id).status;
|
|
6458
|
-
blockingError = "workflow run was cancelled";
|
|
6459
|
-
break;
|
|
6460
|
-
}
|
|
6461
|
-
opts.beforePersist?.();
|
|
6462
|
-
const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
|
|
6463
|
-
if (blockedExit) {
|
|
6464
6579
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
6465
|
-
status:
|
|
6580
|
+
status: result.status,
|
|
6466
6581
|
finishedAt: result.finishedAt,
|
|
6467
6582
|
durationMs: result.durationMs,
|
|
6468
6583
|
stdout: result.stdout,
|
|
6469
6584
|
stderr: result.stderr,
|
|
6470
6585
|
exitCode: result.exitCode,
|
|
6471
|
-
error:
|
|
6586
|
+
error: result.error
|
|
6472
6587
|
}, {
|
|
6473
6588
|
daemonLeaseId: opts.daemonLeaseId
|
|
6474
6589
|
});
|
|
6475
|
-
|
|
6590
|
+
if (result.status !== "succeeded" && !step.continueOnFailure) {
|
|
6591
|
+
terminalStatus = result.status;
|
|
6592
|
+
blockingError = `step ${step.id} ${result.status}${result.error ? `: ${result.error}` : ""}`;
|
|
6593
|
+
break;
|
|
6594
|
+
}
|
|
6476
6595
|
}
|
|
6477
|
-
|
|
6478
|
-
|
|
6479
|
-
|
|
6480
|
-
|
|
6481
|
-
|
|
6482
|
-
|
|
6483
|
-
|
|
6484
|
-
|
|
6596
|
+
if (terminalStatus !== "succeeded") {
|
|
6597
|
+
for (const step of ordered) {
|
|
6598
|
+
const existing = store.getWorkflowStepRun(run.id, step.id);
|
|
6599
|
+
if (existing?.status === "pending" || existing?.status === "running") {
|
|
6600
|
+
store.skipWorkflowStepRun(run.id, step.id, blockingError ?? "workflow stopped before step could run", {
|
|
6601
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6602
|
+
});
|
|
6603
|
+
}
|
|
6604
|
+
}
|
|
6605
|
+
}
|
|
6606
|
+
const finishedAt = nowIso();
|
|
6607
|
+
if (store.isWorkflowRunTerminal(run.id)) {
|
|
6608
|
+
const terminalRun = store.requireWorkflowRun(run.id);
|
|
6609
|
+
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
|
|
6610
|
+
}
|
|
6611
|
+
opts.beforePersist?.();
|
|
6612
|
+
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
6613
|
+
finishedAt,
|
|
6614
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
6615
|
+
error: blockingError
|
|
6485
6616
|
}, {
|
|
6486
6617
|
daemonLeaseId: opts.daemonLeaseId
|
|
6487
6618
|
});
|
|
6488
|
-
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
if (
|
|
6498
|
-
store.
|
|
6619
|
+
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
|
|
6620
|
+
} catch (error) {
|
|
6621
|
+
if (opts.daemonLeaseId && error instanceof Error && error.message === "daemon lease lost")
|
|
6622
|
+
throw error;
|
|
6623
|
+
if (error instanceof Error && error.message.includes("workflow step is not claimable"))
|
|
6624
|
+
throw error;
|
|
6625
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
6626
|
+
const finishedAt = nowIso();
|
|
6627
|
+
try {
|
|
6628
|
+
if (!store.isWorkflowRunTerminal(run.id)) {
|
|
6629
|
+
store.finalizeWorkflowRun(run.id, "failed", {
|
|
6630
|
+
finishedAt,
|
|
6631
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
6632
|
+
error: message
|
|
6633
|
+
}, {
|
|
6499
6634
|
daemonLeaseId: opts.daemonLeaseId
|
|
6500
6635
|
});
|
|
6501
6636
|
}
|
|
6502
|
-
}
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
|
|
6506
|
-
const terminalRun = store.requireWorkflowRun(run.id);
|
|
6507
|
-
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
|
|
6637
|
+
} catch {}
|
|
6638
|
+
const current = store.getWorkflowRun(run.id) ?? run;
|
|
6639
|
+
const resultStatus = current.status === "running" ? "failed" : current.status;
|
|
6640
|
+
return workflowResult(current, resultStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), current.error ?? message);
|
|
6508
6641
|
}
|
|
6509
|
-
opts.beforePersist?.();
|
|
6510
|
-
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
6511
|
-
finishedAt,
|
|
6512
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime(),
|
|
6513
|
-
error: blockingError
|
|
6514
|
-
}, {
|
|
6515
|
-
daemonLeaseId: opts.daemonLeaseId
|
|
6516
|
-
});
|
|
6517
|
-
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
|
|
6518
6642
|
}
|
|
6519
6643
|
function preflightWorkflow(workflow, opts = {}) {
|
|
6520
6644
|
return workflowExecutionOrder(workflow).map((step) => {
|
|
@@ -6936,7 +7060,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6936
7060
|
|
|
6937
7061
|
`);
|
|
6938
7062
|
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
6939
|
-
const tags = ["bug", "openloops", "loop-health", failure.classification];
|
|
7063
|
+
const tags = ["bug", "openloops", "loops", "loop-health", failure.classification];
|
|
6940
7064
|
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
6941
7065
|
return {
|
|
6942
7066
|
title,
|
|
@@ -6951,7 +7075,7 @@ ${failure.evidence.stderr}` : undefined
|
|
|
6951
7075
|
comment: ["todos", "comment", "<task-id>", description]
|
|
6952
7076
|
},
|
|
6953
7077
|
futureNativeUpsert: {
|
|
6954
|
-
command: "todos upsert",
|
|
7078
|
+
command: "todos task upsert",
|
|
6955
7079
|
fields: {
|
|
6956
7080
|
title,
|
|
6957
7081
|
description,
|
|
@@ -7310,6 +7434,25 @@ function advanceOptions(deps) {
|
|
|
7310
7434
|
onRun: deps.onRun
|
|
7311
7435
|
};
|
|
7312
7436
|
}
|
|
7437
|
+
var TERMINAL_RUN_STATUSES2 = new Set([
|
|
7438
|
+
"succeeded",
|
|
7439
|
+
"failed",
|
|
7440
|
+
"timed_out",
|
|
7441
|
+
"abandoned",
|
|
7442
|
+
"skipped"
|
|
7443
|
+
]);
|
|
7444
|
+
function repairWedgedTerminalSlot(deps, loop, scheduledFor, now) {
|
|
7445
|
+
const existing = deps.store.getRunBySlot(loop.id, scheduledFor);
|
|
7446
|
+
if (!existing || !TERMINAL_RUN_STATUSES2.has(existing.status))
|
|
7447
|
+
return;
|
|
7448
|
+
try {
|
|
7449
|
+
advanceLoop(deps.store, loop, existing, new Date(existing.finishedAt ?? now), existing.status === "succeeded", advanceOptions(deps));
|
|
7450
|
+
} catch (error) {
|
|
7451
|
+
if (deps.daemonLeaseId && isDaemonLeaseLost(error))
|
|
7452
|
+
return;
|
|
7453
|
+
throw error;
|
|
7454
|
+
}
|
|
7455
|
+
}
|
|
7313
7456
|
async function runSlot(deps, loop, scheduledFor) {
|
|
7314
7457
|
const now = deps.now?.() ?? new Date;
|
|
7315
7458
|
deps.beforeRun?.(loop, scheduledFor);
|
|
@@ -7336,8 +7479,10 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
7336
7479
|
return;
|
|
7337
7480
|
throw error;
|
|
7338
7481
|
}
|
|
7339
|
-
if (!claim)
|
|
7482
|
+
if (!claim) {
|
|
7483
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
7340
7484
|
return;
|
|
7485
|
+
}
|
|
7341
7486
|
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
7342
7487
|
deps.onRun?.(claim.run);
|
|
7343
7488
|
const finalRun = await executeClaimedRun({
|
|
@@ -7383,8 +7528,10 @@ function claimSlot(deps, loop, scheduledFor) {
|
|
|
7383
7528
|
return;
|
|
7384
7529
|
throw error;
|
|
7385
7530
|
}
|
|
7386
|
-
if (!claim)
|
|
7531
|
+
if (!claim) {
|
|
7532
|
+
repairWedgedTerminalSlot(deps, loop, scheduledFor, now);
|
|
7387
7533
|
return;
|
|
7534
|
+
}
|
|
7388
7535
|
deps.beforeRun?.(claim.loop, claim.run.scheduledFor);
|
|
7389
7536
|
deps.onRun?.(claim.run);
|
|
7390
7537
|
return claim;
|
|
@@ -7665,10 +7812,7 @@ async function stopDaemon(opts = {}) {
|
|
|
7665
7812
|
const store = new Store(join5(dirname4(path), "loops.db"));
|
|
7666
7813
|
try {
|
|
7667
7814
|
const lease = store.getDaemonLease();
|
|
7668
|
-
|
|
7669
|
-
removePid(path);
|
|
7670
|
-
return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
|
|
7671
|
-
}
|
|
7815
|
+
const leaseVerified = Boolean(lease && lease.pid === state.pid && new Date(lease.expiresAt).getTime() > Date.now());
|
|
7672
7816
|
try {
|
|
7673
7817
|
process.kill(state.pid, "SIGTERM");
|
|
7674
7818
|
} catch {
|
|
@@ -7688,11 +7832,14 @@ async function stopDaemon(opts = {}) {
|
|
|
7688
7832
|
} catch {}
|
|
7689
7833
|
await sleep2(150);
|
|
7690
7834
|
removePid(path);
|
|
7691
|
-
|
|
7692
|
-
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7835
|
+
let reapedPgids = [];
|
|
7836
|
+
if (leaseVerified && lease) {
|
|
7837
|
+
const ownedRuns = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.claimedBy !== undefined && run.claimedBy.endsWith(`:${lease.id}`));
|
|
7838
|
+
reapedPgids = await reapProcessGroups(ownedRuns.map(toReapableProcess), {
|
|
7839
|
+
sleep: sleep2,
|
|
7840
|
+
graceMs: opts.reapGraceMs
|
|
7841
|
+
});
|
|
7842
|
+
}
|
|
7696
7843
|
return { wasRunning: true, stopped: !isAlive(state.pid, record?.startedAt), forced: true, pid: state.pid, reapedPgids };
|
|
7697
7844
|
} finally {
|
|
7698
7845
|
store.close();
|
|
@@ -7880,7 +8027,14 @@ async function runDaemon(opts = {}) {
|
|
|
7880
8027
|
try {
|
|
7881
8028
|
const liveInlineOwner = (run) => {
|
|
7882
8029
|
const ownerPid = inlineRunnerOwnerPid(run.claimedBy);
|
|
7883
|
-
|
|
8030
|
+
if (ownerPid === undefined || !isAlive(ownerPid))
|
|
8031
|
+
return false;
|
|
8032
|
+
const ownerStartMs = processStartTimeMs(ownerPid);
|
|
8033
|
+
const runStartMs = run.startedAt ? Date.parse(run.startedAt) : Number.NaN;
|
|
8034
|
+
if (ownerStartMs !== undefined && Number.isFinite(runStartMs) && ownerStartMs > runStartMs + START_TIME_TOLERANCE_MS) {
|
|
8035
|
+
return false;
|
|
8036
|
+
}
|
|
8037
|
+
return true;
|
|
7884
8038
|
};
|
|
7885
8039
|
const staleCandidates = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.leaseExpiresAt !== undefined && new Date(run.leaseExpiresAt).getTime() <= Date.now());
|
|
7886
8040
|
const startup = claimDueRuns({ store, runnerId, daemonLeaseId: leaseId, maxClaims: 0 });
|
|
@@ -9165,6 +9319,9 @@ function roleFragment(role, flow) {
|
|
|
9165
9319
|
function goalHeaderFragment(goal, role, flow) {
|
|
9166
9320
|
return [`/goal ${goal}`, "", roleFragment(role, flow)];
|
|
9167
9321
|
}
|
|
9322
|
+
function boundedStepHeaderFragment(objective, role, flow) {
|
|
9323
|
+
return [`Objective: ${objective}`, "", roleFragment(role, flow)];
|
|
9324
|
+
}
|
|
9168
9325
|
function adversarialReviewFragment(inspect, focus) {
|
|
9169
9326
|
return `Use fresh context. Inspect ${inspect}. Act as an adversarial reviewer focused on ${focus}.`;
|
|
9170
9327
|
}
|
|
@@ -10297,7 +10454,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10297
10454
|
const blockTaskCommand = `todos --project ${todosProjectPath} update ${input.taskId} --status blocked`;
|
|
10298
10455
|
const gateStopFragment = (stage, stops) => `The deterministic ${stage} gate will stop ${stops} unless the latest ${stage} marker is the exact go marker and the task has no blocked/completed/done/cancelled/failed/archived/no-auto/manual/approval-required state.`;
|
|
10299
10456
|
const triagePrompt = [
|
|
10300
|
-
...
|
|
10457
|
+
...boundedStepHeaderFragment(`Triage todos task ${input.taskId} for safe automated execution.`, "triage", "lifecycle"),
|
|
10301
10458
|
shared,
|
|
10302
10459
|
"Decide whether the task is eligible for loop execution. Check status, dependencies, duplicate tasks, no-auto/manual/approval metadata, project path, acceptance criteria, and whether the requested work should be split before implementation.",
|
|
10303
10460
|
"Do not implement repo changes in this step.",
|
|
@@ -10309,7 +10466,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10309
10466
|
].join(`
|
|
10310
10467
|
`);
|
|
10311
10468
|
const plannerPrompt = [
|
|
10312
|
-
...
|
|
10469
|
+
...boundedStepHeaderFragment(`Plan todos task ${input.taskId} before implementation.`, "planner", "lifecycle"),
|
|
10313
10470
|
shared,
|
|
10314
10471
|
"Read the triage comment and current task details.",
|
|
10315
10472
|
`If the task is ready for implementation, add a task comment whose first line is exactly: ${gateMarker("planner", "go")}`,
|
|
@@ -10320,7 +10477,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10320
10477
|
].join(`
|
|
10321
10478
|
`);
|
|
10322
10479
|
const workerPrompt = [
|
|
10323
|
-
...
|
|
10480
|
+
...boundedStepHeaderFragment(`Complete todos task ${input.taskId} according to the planner evidence.`, "worker", "lifecycle"),
|
|
10324
10481
|
shared,
|
|
10325
10482
|
todosStartLine(todosProjectPath, input.taskId),
|
|
10326
10483
|
"Read the triage and planner comments first. Implement only the scoped task, run focused validation, and record changed files, commits, evidence, blockers, and residual risks.",
|
|
@@ -10329,7 +10486,7 @@ function renderTaskLifecycleWorkflow(input) {
|
|
|
10329
10486
|
].filter(Boolean).join(`
|
|
10330
10487
|
`);
|
|
10331
10488
|
const verifierPrompt = [
|
|
10332
|
-
...
|
|
10489
|
+
...boundedStepHeaderFragment(`Verify todos task ${input.taskId} after the full lifecycle worker step.`, "verifier", "lifecycle"),
|
|
10333
10490
|
shared,
|
|
10334
10491
|
todosVerificationLine(todosProjectPath, input.taskId),
|
|
10335
10492
|
todosDoneLine(todosProjectPath, input.taskId),
|
|
@@ -11035,7 +11192,7 @@ function taskRouteEligibility(data, metadata) {
|
|
|
11035
11192
|
const records = taskEventRecords(data, metadata);
|
|
11036
11193
|
const automation = automationRecords(data, metadata);
|
|
11037
11194
|
const tags = taskEventTags(records);
|
|
11038
|
-
const hasRouteOptIn = tags.includes("auto:route") || hasTruthyField(records, ["route_enabled", "routeEnabled", "automation_allowed", "automationAllowed"]) || hasTruthyField(automation, ["allowed"]);
|
|
11195
|
+
const hasRouteOptIn = tags.includes("auto:route") || tags.includes("route:enabled") || hasTruthyField(records, ["route_enabled", "routeEnabled", "automation_allowed", "automationAllowed"]) || hasTruthyField(automation, ["allowed"]);
|
|
11039
11196
|
if (!hasRouteOptIn)
|
|
11040
11197
|
return { eligible: false, reason: "missing explicit route opt-in", tags };
|
|
11041
11198
|
const status = taskEventField(data, ["status", "task_status", "taskStatus"])?.toLowerCase();
|
|
@@ -12336,7 +12493,8 @@ function compactDrainResult(result) {
|
|
|
12336
12493
|
workflowName: stringField2(workflow?.name),
|
|
12337
12494
|
providerRouting,
|
|
12338
12495
|
requeue,
|
|
12339
|
-
queuedAtSource: value.queuedAtSource
|
|
12496
|
+
queuedAtSource: value.queuedAtSource,
|
|
12497
|
+
fatal: value.fatal === true ? true : undefined
|
|
12340
12498
|
};
|
|
12341
12499
|
}
|
|
12342
12500
|
function loadReadyTodosTasks(opts, scanLimit) {
|
|
@@ -12456,10 +12614,12 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12456
12614
|
result = routeTodosTaskEvent(event, opts);
|
|
12457
12615
|
} catch (error) {
|
|
12458
12616
|
const message = error instanceof Error ? error.message : String(error);
|
|
12459
|
-
if (
|
|
12460
|
-
|
|
12461
|
-
|
|
12462
|
-
|
|
12617
|
+
if (isSkippableDrainRouteError(message)) {
|
|
12618
|
+
const sourceTaskUpdate = opts.dryRun ? { attempted: false, reason: "dry-run" } : markInvalidDrainTaskNonRouteable(todosProject, task, message);
|
|
12619
|
+
result = skippedDrainTask(task, event, redact(message, 640) ?? "route task failed", { sourceTaskUpdate });
|
|
12620
|
+
} else {
|
|
12621
|
+
result = skippedDrainTask(task, event, redact(message, 640) ?? "route task failed", { fatal: true });
|
|
12622
|
+
}
|
|
12463
12623
|
}
|
|
12464
12624
|
results.push(result);
|
|
12465
12625
|
if (result.kind === "created")
|
|
@@ -12486,6 +12646,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12486
12646
|
deduped: results.filter((result) => result.kind === "deduped").length,
|
|
12487
12647
|
throttled: results.filter((result) => result.kind === "throttled").length,
|
|
12488
12648
|
skipped: results.filter((result) => result.kind === "skipped").length,
|
|
12649
|
+
fatal: results.filter((result) => result.value.fatal === true).length,
|
|
12489
12650
|
maxDispatch,
|
|
12490
12651
|
source: "todos ready",
|
|
12491
12652
|
dryRun: Boolean(opts.dryRun),
|
|
@@ -12513,6 +12674,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12513
12674
|
deduped: report.deduped,
|
|
12514
12675
|
throttled: report.throttled,
|
|
12515
12676
|
skipped: report.skipped,
|
|
12677
|
+
fatal: report.fatal,
|
|
12516
12678
|
maxDispatch: report.maxDispatch,
|
|
12517
12679
|
source: report.source,
|
|
12518
12680
|
dryRun: report.dryRun,
|
|
@@ -12521,7 +12683,7 @@ function drainTodosTaskRoutes(opts) {
|
|
|
12521
12683
|
} : { ...report, evidencePath };
|
|
12522
12684
|
return {
|
|
12523
12685
|
value,
|
|
12524
|
-
human: `drained todos ready queue: considered=${report.considered} created=${report.created} deduped=${report.deduped} throttled=${report.throttled} skipped=${report.skipped}`
|
|
12686
|
+
human: `drained todos ready queue: considered=${report.considered} created=${report.created} deduped=${report.deduped} throttled=${report.throttled} skipped=${report.skipped} fatal=${report.fatal}`
|
|
12525
12687
|
};
|
|
12526
12688
|
}
|
|
12527
12689
|
// src/lib/route/route-tasks.ts
|
|
@@ -13722,6 +13884,11 @@ function handleRouteDrain(kind, opts) {
|
|
|
13722
13884
|
throw new ValidationError("route drain currently supports kind todos-task");
|
|
13723
13885
|
const result = drainTodosTaskRoutes(opts);
|
|
13724
13886
|
print(result.value, result.human);
|
|
13887
|
+
const fatal = Number(result.value.fatal ?? 0);
|
|
13888
|
+
if (fatal > 0) {
|
|
13889
|
+
console.error(`route drain hit ${fatal} non-skippable task error(s); see evidence file`);
|
|
13890
|
+
process.exitCode = 1;
|
|
13891
|
+
}
|
|
13725
13892
|
}
|
|
13726
13893
|
addRouteEventOptions(routes.command("preview <kind>").description("(deprecated: use 'routes create <kind> --dry-run') preview a route-created workflow invocation without storing it")).action(runAction(async (kind, opts) => handleRouteEvent(kind, { ...opts, dryRun: true })));
|
|
13727
13894
|
addRouteEventOptions(routes.command("create <kind>").description("create a route workflow invocation and admit it when capacity allows"), { dryRunDescription: "preview the generated workflow invocation and loop without storing anything" }).action(runAction(async (kind, opts) => handleRouteEvent(kind, opts)));
|
|
@@ -13923,7 +14090,7 @@ workflows.command("runs [idOrName]").description("list workflow runs, optionally
|
|
|
13923
14090
|
const store = new Store;
|
|
13924
14091
|
try {
|
|
13925
14092
|
const workflow = idOrName ? store.requireWorkflow(idOrName) : undefined;
|
|
13926
|
-
const runs = store.listWorkflowRuns({ workflowId: workflow?.id, limit:
|
|
14093
|
+
const runs = store.listWorkflowRuns({ workflowId: workflow?.id, limit: positiveInteger(opts.limit, "--limit") ?? 50 });
|
|
13927
14094
|
if (isJson())
|
|
13928
14095
|
print(runs.map(publicWorkflowRun));
|
|
13929
14096
|
else {
|
|
@@ -13938,7 +14105,7 @@ workflows.command("runs [idOrName]").description("list workflow runs, optionally
|
|
|
13938
14105
|
workflows.command("events <runId>").description("list step/lifecycle events for a workflow run").option("--limit <n>", "limit", "200").action(runAction((runId, opts) => {
|
|
13939
14106
|
const store = new Store;
|
|
13940
14107
|
try {
|
|
13941
|
-
const runEvents = store.listWorkflowEvents(runId,
|
|
14108
|
+
const runEvents = store.listWorkflowEvents(runId, positiveInteger(opts.limit, "--limit") ?? 200);
|
|
13942
14109
|
if (isJson())
|
|
13943
14110
|
print(runEvents.map(publicWorkflowEvent));
|
|
13944
14111
|
else {
|
|
@@ -14179,7 +14346,7 @@ program.command("runs [idOrName]").description("list recent runs, optionally for
|
|
|
14179
14346
|
const store = new Store;
|
|
14180
14347
|
try {
|
|
14181
14348
|
const loop = idOrName ? store.requireLoop(idOrName) : undefined;
|
|
14182
|
-
const runs = store.listRuns({ loopId: loop?.id, limit:
|
|
14349
|
+
const runs = store.listRuns({ loopId: loop?.id, limit: positiveInteger(opts.limit, "--limit") ?? 50 });
|
|
14183
14350
|
if (isJson())
|
|
14184
14351
|
print(runs.map((run) => publicRun(run, opts.showOutput)));
|
|
14185
14352
|
else {
|
|
@@ -14196,7 +14363,7 @@ program.command("runs [idOrName]").description("list recent runs, optionally for
|
|
|
14196
14363
|
program.command("expectations [idOrName]").description("evaluate deterministic loop expectations without mutating external task systems").option("--limit <n>", "maximum loops to inspect when no loop is specified", "200").action(runAction((idOrName, opts) => {
|
|
14197
14364
|
const store = new Store;
|
|
14198
14365
|
try {
|
|
14199
|
-
const loops = idOrName ? [store.requireLoop(idOrName)] : store.listLoops({ limit:
|
|
14366
|
+
const loops = idOrName ? [store.requireLoop(idOrName)] : store.listLoops({ limit: positiveInteger(opts.limit, "--limit") ?? 200 });
|
|
14200
14367
|
const values = loops.map((loop) => expectationForLoop(store, loop));
|
|
14201
14368
|
if (isJson())
|
|
14202
14369
|
console.log(JSON.stringify(idOrName ? values[0] : values, null, 2));
|
|
@@ -14234,7 +14401,7 @@ var health = program.command("health").description("summarize loop health and la
|
|
|
14234
14401
|
health.command("route-tasks").description("upsert deduped todos tasks for failed loop health expectations").option("--project <path>", "todos project path", defaultLoopsProject()).option("--task-list <slug>", "todos task-list slug", "loop-error-self-heal").option("--limit <n>", "maximum loops to inspect", "200").option("--max-actions <n>", "maximum todos tasks to upsert", "5").option("--include-inactive", "also route stopped or expired loops").option("--auto-route", "opt routed tasks into task-created headless worker/verifier automation").option("--route-project-path <path>", "fallback project path for --auto-route when the failed loop has no cwd").option("--evidence-dir <path>", "write the route result JSON to this directory").option("--dry-run", "print intended task upserts without mutating todos").action(runAction((opts) => {
|
|
14235
14402
|
const store = new Store;
|
|
14236
14403
|
try {
|
|
14237
|
-
const report = buildHealthReport(store, { limit:
|
|
14404
|
+
const report = buildHealthReport(store, { limit: positiveInteger(opts.limit, "--limit") ?? 200, includeInactive: Boolean(opts.includeInactive) });
|
|
14238
14405
|
const failures = report.expectations.filter((entry) => !entry.ok && entry.recommendedTask);
|
|
14239
14406
|
const result = upsertRouteTasks({
|
|
14240
14407
|
project: opts.project,
|
|
@@ -14247,7 +14414,7 @@ health.command("route-tasks").description("upsert deduped todos tasks for failed
|
|
|
14247
14414
|
autoRoute: Boolean(opts.autoRoute),
|
|
14248
14415
|
routeProjectPath: opts.routeProjectPath
|
|
14249
14416
|
}),
|
|
14250
|
-
maxActions:
|
|
14417
|
+
maxActions: positiveInteger(opts.maxActions, "--max-actions") ?? 5,
|
|
14251
14418
|
dryRun: Boolean(opts.dryRun),
|
|
14252
14419
|
autoRoute: Boolean(opts.autoRoute),
|
|
14253
14420
|
routeProjectPath: opts.routeProjectPath,
|
|
@@ -14301,7 +14468,7 @@ hygiene.command("names").description("check or apply canonical machine-/repo-pre
|
|
|
14301
14468
|
apply: false,
|
|
14302
14469
|
includeStopped: Boolean(opts.includeStopped),
|
|
14303
14470
|
includeInactive: Boolean(opts.includeInactive),
|
|
14304
|
-
limit:
|
|
14471
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14305
14472
|
});
|
|
14306
14473
|
let outputReport = report;
|
|
14307
14474
|
const backupPath = opts.apply && report.changed > 0 ? backupLoopsDatabase("name-hygiene") : undefined;
|
|
@@ -14310,7 +14477,7 @@ hygiene.command("names").description("check or apply canonical machine-/repo-pre
|
|
|
14310
14477
|
apply: true,
|
|
14311
14478
|
includeStopped: Boolean(opts.includeStopped),
|
|
14312
14479
|
includeInactive: Boolean(opts.includeInactive),
|
|
14313
|
-
limit:
|
|
14480
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14314
14481
|
});
|
|
14315
14482
|
} else if (opts.apply) {
|
|
14316
14483
|
outputReport = { ...report, applied: true };
|
|
@@ -14337,7 +14504,7 @@ hygiene.command("duplicates").description("detect duplicate/overlapping loops wi
|
|
|
14337
14504
|
try {
|
|
14338
14505
|
const report = buildDuplicateOverlapReport(store, {
|
|
14339
14506
|
includeInactive: Boolean(opts.includeInactive),
|
|
14340
|
-
limit:
|
|
14507
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14341
14508
|
});
|
|
14342
14509
|
if (isJson())
|
|
14343
14510
|
console.log(JSON.stringify(report, null, 2));
|
|
@@ -14359,7 +14526,7 @@ hygiene.command("scripts").description("inventory loops still backed by local ~/
|
|
|
14359
14526
|
const report = buildScriptInventoryReport(store, {
|
|
14360
14527
|
scriptsDir: opts.scriptsDir,
|
|
14361
14528
|
includeInactive: Boolean(opts.includeInactive),
|
|
14362
|
-
limit:
|
|
14529
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000
|
|
14363
14530
|
});
|
|
14364
14531
|
if (isJson())
|
|
14365
14532
|
console.log(JSON.stringify(report, null, 2));
|
|
@@ -14381,7 +14548,7 @@ hygiene.command("route-tasks").description("upsert deduped todos tasks for hygie
|
|
|
14381
14548
|
const route = buildHygieneRouteTasks(store, {
|
|
14382
14549
|
checks,
|
|
14383
14550
|
includeInactive: Boolean(opts.includeInactive),
|
|
14384
|
-
limit:
|
|
14551
|
+
limit: positiveInteger(opts.limit, "--limit") ?? 1000,
|
|
14385
14552
|
scriptsDir: opts.scriptsDir
|
|
14386
14553
|
});
|
|
14387
14554
|
const result = upsertRouteTasks({
|
|
@@ -14395,7 +14562,7 @@ hygiene.command("route-tasks").description("upsert deduped todos tasks for hygie
|
|
|
14395
14562
|
autoRoute: Boolean(opts.autoRoute),
|
|
14396
14563
|
routeProjectPath: opts.routeProjectPath
|
|
14397
14564
|
}),
|
|
14398
|
-
maxActions:
|
|
14565
|
+
maxActions: positiveInteger(opts.maxActions, "--max-actions") ?? 10,
|
|
14399
14566
|
dryRun: Boolean(opts.dryRun),
|
|
14400
14567
|
autoRoute: Boolean(opts.autoRoute),
|
|
14401
14568
|
routeProjectPath: opts.routeProjectPath,
|
|
@@ -14435,7 +14602,7 @@ program.command("stop <idOrName>").description("stop a loop and clear its next s
|
|
|
14435
14602
|
program.command("rename <idOrName> <newName>").description("rename a loop without changing its id, schedule, runs, or history").action(runAction((idOrName, newName) => {
|
|
14436
14603
|
const store = new Store;
|
|
14437
14604
|
try {
|
|
14438
|
-
const loop = store.
|
|
14605
|
+
const loop = store.requireUniqueLoop(idOrName);
|
|
14439
14606
|
const oldName = loop.name;
|
|
14440
14607
|
const trimmed = String(newName).trim();
|
|
14441
14608
|
if (!trimmed)
|
|
@@ -14472,10 +14639,17 @@ backup=${backupPath ?? "skipped (recent rename backup exists)"}`);
|
|
|
14472
14639
|
function updateStatus(idOrName, status) {
|
|
14473
14640
|
const store = new Store;
|
|
14474
14641
|
try {
|
|
14475
|
-
const loop = store.
|
|
14642
|
+
const loop = store.requireUniqueLoop(idOrName);
|
|
14476
14643
|
if (loop.archivedAt)
|
|
14477
14644
|
throw new Error(`loop is archived; run 'loops unarchive ${idOrName}' first`);
|
|
14478
|
-
|
|
14645
|
+
let nextRunAt = loop.nextRunAt;
|
|
14646
|
+
if (status === "stopped") {
|
|
14647
|
+
nextRunAt = undefined;
|
|
14648
|
+
} else if (status === "active" && !loop.nextRunAt) {
|
|
14649
|
+
const now = new Date;
|
|
14650
|
+
nextRunAt = computeNextAfter(loop.schedule, now, now);
|
|
14651
|
+
}
|
|
14652
|
+
const updated = store.updateLoop(loop.id, { status, nextRunAt });
|
|
14479
14653
|
print(publicLoop(updated), `${updated.id} ${updated.status}`);
|
|
14480
14654
|
} finally {
|
|
14481
14655
|
store.close();
|
|
@@ -14484,7 +14658,7 @@ function updateStatus(idOrName, status) {
|
|
|
14484
14658
|
program.command("remove <idOrName>").alias("rm").description("delete a loop and its run history").action(runAction((idOrName) => {
|
|
14485
14659
|
const store = new Store;
|
|
14486
14660
|
try {
|
|
14487
|
-
const removed = store.deleteLoop(idOrName);
|
|
14661
|
+
const removed = store.deleteLoop(store.requireUniqueLoop(idOrName).id);
|
|
14488
14662
|
print({ removed }, removed ? "removed" : "not removed");
|
|
14489
14663
|
} finally {
|
|
14490
14664
|
store.close();
|
|
@@ -14511,7 +14685,7 @@ program.command("unarchive <idOrName>").alias("restore").description("restore an
|
|
|
14511
14685
|
program.command("run-now <idOrName>").description("claim and execute one loop run immediately").option("--show-output", "show stdout/stderr").action(runAction(async (idOrName, opts) => {
|
|
14512
14686
|
const store = new Store;
|
|
14513
14687
|
try {
|
|
14514
|
-
const loop = store.
|
|
14688
|
+
const loop = store.requireUniqueLoop(idOrName);
|
|
14515
14689
|
if (loop.archivedAt)
|
|
14516
14690
|
throw new Error(`loop is archived; run 'loops unarchive ${idOrName}' before running it`);
|
|
14517
14691
|
const runnerId = `manual:${process.pid}`;
|
|
@@ -14691,15 +14865,16 @@ ${result.enableResults.map((item) => `${item.command} -> ${item.status === 0 ? "
|
|
|
14691
14865
|
${result.instructions.join(`
|
|
14692
14866
|
`)}${enableText}`);
|
|
14693
14867
|
}));
|
|
14694
|
-
daemon.command("logs").description("print the tail of the daemon log").option("-n, --lines <n>", "lines", "80").action(runAction((opts) => {
|
|
14868
|
+
daemon.command("logs").description("print the tail of the daemon log").option("-n, --lines <n>", "lines", "80").option("--tail <n>", "alias for --lines").action(runAction((opts) => {
|
|
14695
14869
|
const path = daemonLogPath();
|
|
14696
14870
|
if (!existsSync10(path)) {
|
|
14697
14871
|
console.log("");
|
|
14698
14872
|
return;
|
|
14699
14873
|
}
|
|
14874
|
+
const count = positiveInteger(opts.tail ?? opts.lines, opts.tail !== undefined ? "--tail" : "--lines") ?? 80;
|
|
14700
14875
|
const lines = readFileSync10(path, "utf8").trimEnd().split(`
|
|
14701
14876
|
`);
|
|
14702
|
-
console.log(lines.slice(-
|
|
14877
|
+
console.log(lines.slice(-count).join(`
|
|
14703
14878
|
`));
|
|
14704
14879
|
}));
|
|
14705
14880
|
await program.parseAsync(process.argv);
|