@hasna/loops 0.3.60 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +265 -0
- package/LICENSE +197 -13
- package/README.md +95 -85
- package/dist/api/index.d.ts +11 -0
- package/dist/api/index.js +333 -0
- package/dist/cli/index.js +8013 -6659
- package/dist/daemon/control.d.ts +21 -1
- package/dist/daemon/daemon.d.ts +5 -0
- package/dist/daemon/index.js +2836 -1222
- package/dist/daemon/install.d.ts +1 -1
- package/dist/index.d.ts +23 -8
- package/dist/index.js +5715 -4541
- package/dist/lib/accounts.d.ts +6 -1
- package/dist/lib/agent-adapter.d.ts +74 -0
- package/dist/lib/backup.d.ts +25 -0
- package/dist/lib/errors.d.ts +21 -0
- package/dist/lib/executor.d.ts +10 -1
- package/dist/lib/goal/metadata.d.ts +16 -0
- package/dist/lib/goal/model-factory.d.ts +1 -0
- package/dist/lib/goal/prompts.d.ts +3 -1
- package/dist/lib/goal/types.d.ts +3 -0
- package/dist/lib/health.d.ts +1 -1
- package/dist/lib/ids.d.ts +8 -1
- package/dist/lib/machines.d.ts +6 -0
- package/dist/lib/mode.d.ts +48 -0
- package/dist/lib/mode.js +260 -0
- package/dist/lib/process-identity.d.ts +16 -0
- package/dist/lib/{schedule.d.ts → recurrence.d.ts} +1 -0
- package/dist/lib/redact.d.ts +21 -0
- package/dist/lib/route/cursors.d.ts +18 -0
- package/dist/lib/route/drain.d.ts +6 -0
- package/dist/lib/route/fields.d.ts +27 -0
- package/dist/lib/route/gates.d.ts +26 -0
- package/dist/lib/route/index.d.ts +18 -0
- package/dist/lib/route/options.d.ts +31 -0
- package/dist/lib/route/parse.d.ts +8 -0
- package/dist/lib/route/pr-review.d.ts +11 -0
- package/dist/lib/route/provider.d.ts +41 -0
- package/dist/lib/route/route-event.d.ts +23 -0
- package/dist/lib/route/route-tasks.d.ts +75 -0
- package/dist/lib/route/throttle.d.ts +47 -0
- package/dist/lib/route/todos-cli.d.ts +21 -0
- package/dist/lib/route/types.d.ts +88 -0
- package/dist/lib/run-artifacts.d.ts +17 -2
- package/dist/lib/run-envelope.d.ts +41 -0
- package/dist/lib/scheduler.d.ts +78 -2
- package/dist/lib/store.d.ts +63 -0
- package/dist/lib/store.js +1007 -287
- package/dist/lib/template-kit.d.ts +70 -0
- package/dist/lib/templates-custom.d.ts +34 -0
- package/dist/lib/templates.d.ts +13 -81
- package/dist/lib/workflow-runner.d.ts +2 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.js +3231 -1382
- package/dist/runner/index.d.ts +9 -0
- package/dist/runner/index.js +295 -0
- package/dist/sdk/index.d.ts +29 -3
- package/dist/sdk/index.js +2743 -1044
- package/dist/test-helpers.d.ts +37 -0
- package/dist/types.d.ts +3 -1
- package/docs/DEPLOYMENT_MODES.md +140 -0
- package/docs/USAGE.md +102 -46
- package/package.json +23 -5
package/dist/sdk/index.js
CHANGED
|
@@ -1,14 +1,50 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/lib/store.ts
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync } from "fs";
|
|
4
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
|
|
5
5
|
import { tmpdir } from "os";
|
|
6
|
-
import { dirname, join as join3 } from "path";
|
|
6
|
+
import { basename as basename2, dirname as dirname2, join as join3 } from "path";
|
|
7
|
+
|
|
8
|
+
// src/lib/errors.ts
|
|
9
|
+
class CodedError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
constructor(code, message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = new.target.name;
|
|
14
|
+
this.code = code;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
class LoopNotFoundError extends CodedError {
|
|
19
|
+
constructor(idOrName) {
|
|
20
|
+
super("LOOP_NOT_FOUND", `loop not found: ${idOrName}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class LoopArchivedError extends CodedError {
|
|
25
|
+
constructor(idOrName) {
|
|
26
|
+
super("LOOP_ARCHIVED", `loop is archived: ${idOrName}; unarchive it before modifying`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
class AmbiguousNameError extends CodedError {
|
|
31
|
+
constructor(name) {
|
|
32
|
+
super("AMBIGUOUS_NAME", `ambiguous loop name: ${name}; use a loop id`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class ValidationError extends CodedError {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super("VALIDATION_ERROR", message);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
7
41
|
|
|
8
42
|
// src/lib/ids.ts
|
|
9
43
|
import { randomBytes } from "crypto";
|
|
10
|
-
|
|
11
|
-
|
|
44
|
+
var TIME_HEX_LENGTH = 12;
|
|
45
|
+
function genId() {
|
|
46
|
+
const time = Date.now().toString(16).padStart(TIME_HEX_LENGTH, "0");
|
|
47
|
+
return `${time}${randomBytes(10).toString("hex")}`;
|
|
12
48
|
}
|
|
13
49
|
function nowIso() {
|
|
14
50
|
return new Date().toISOString();
|
|
@@ -42,7 +78,166 @@ function launchdPlistPath() {
|
|
|
42
78
|
return join(homedir(), "Library", "LaunchAgents", "com.hasna.loops.daemon.plist");
|
|
43
79
|
}
|
|
44
80
|
|
|
45
|
-
// src/lib/
|
|
81
|
+
// src/lib/process-identity.ts
|
|
82
|
+
import { readFileSync } from "fs";
|
|
83
|
+
import { spawnSync } from "child_process";
|
|
84
|
+
var START_TIME_TOLERANCE_MS = 5000;
|
|
85
|
+
var clockTicksCache;
|
|
86
|
+
function clockTicksPerSecond() {
|
|
87
|
+
if (clockTicksCache !== undefined)
|
|
88
|
+
return clockTicksCache;
|
|
89
|
+
try {
|
|
90
|
+
const run = spawnSync("getconf", ["CLK_TCK"], { encoding: "utf8" });
|
|
91
|
+
const value = Number(run.stdout.trim());
|
|
92
|
+
clockTicksCache = run.status === 0 && Number.isFinite(value) && value > 0 ? value : 100;
|
|
93
|
+
} catch {
|
|
94
|
+
clockTicksCache = 100;
|
|
95
|
+
}
|
|
96
|
+
return clockTicksCache;
|
|
97
|
+
}
|
|
98
|
+
var bootTimeCache;
|
|
99
|
+
var bootTimeResolved = false;
|
|
100
|
+
function bootTimeMs() {
|
|
101
|
+
if (bootTimeResolved)
|
|
102
|
+
return bootTimeCache;
|
|
103
|
+
bootTimeResolved = true;
|
|
104
|
+
try {
|
|
105
|
+
const match = readFileSync("/proc/stat", "utf8").match(/^btime (\d+)$/m);
|
|
106
|
+
bootTimeCache = match ? Number(match[1]) * 1000 : undefined;
|
|
107
|
+
} catch {
|
|
108
|
+
bootTimeCache = undefined;
|
|
109
|
+
}
|
|
110
|
+
return bootTimeCache;
|
|
111
|
+
}
|
|
112
|
+
function procStatFields(path) {
|
|
113
|
+
try {
|
|
114
|
+
const stat = readFileSync(path, "utf8");
|
|
115
|
+
const closeParen = stat.lastIndexOf(")");
|
|
116
|
+
if (closeParen < 0)
|
|
117
|
+
return;
|
|
118
|
+
return stat.slice(closeParen + 2).split(" ");
|
|
119
|
+
} catch {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function processStartTimeMs(pid) {
|
|
124
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
125
|
+
return;
|
|
126
|
+
if (process.platform === "linux") {
|
|
127
|
+
const fields = procStatFields(`/proc/${pid}/stat`);
|
|
128
|
+
const startTicks = fields ? Number(fields[19]) : Number.NaN;
|
|
129
|
+
const boot = bootTimeMs();
|
|
130
|
+
if (Number.isFinite(startTicks) && boot !== undefined) {
|
|
131
|
+
return boot + Math.round(startTicks / clockTicksPerSecond() * 1000);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
const run = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" });
|
|
136
|
+
if (run.status === 0) {
|
|
137
|
+
const parsed = Date.parse(run.stdout.trim());
|
|
138
|
+
if (Number.isFinite(parsed))
|
|
139
|
+
return parsed;
|
|
140
|
+
}
|
|
141
|
+
} catch {}
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
function sameProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
145
|
+
if (recorded === undefined || actualMs === undefined)
|
|
146
|
+
return true;
|
|
147
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
148
|
+
if (!Number.isFinite(recordedMs))
|
|
149
|
+
return true;
|
|
150
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
151
|
+
}
|
|
152
|
+
function verifiedProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
153
|
+
if (recorded === undefined || actualMs === undefined)
|
|
154
|
+
return false;
|
|
155
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
156
|
+
if (!Number.isFinite(recordedMs))
|
|
157
|
+
return false;
|
|
158
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/lib/redact.ts
|
|
162
|
+
var SCRUBBED = "[SCRUBBED]";
|
|
163
|
+
var TOKEN_PATTERNS = [
|
|
164
|
+
/\bsk-ant-[A-Za-z0-9_-]{8,}/g,
|
|
165
|
+
/\bsk-proj-[A-Za-z0-9_-]{8,}/g,
|
|
166
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
167
|
+
/\bghp_[A-Za-z0-9]{16,}\b/g,
|
|
168
|
+
/\bgithub_pat_[A-Za-z0-9_]{16,}\b/g,
|
|
169
|
+
/\bxox[a-z]-[A-Za-z0-9-]{8,}/g,
|
|
170
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{10,}\b/g
|
|
171
|
+
];
|
|
172
|
+
var AUTHORIZATION_PATTERN = /(\bAuthorization(?:\\?["'])?\s*[:=]\s*(?:\\?["'])?(?:Bearer|Basic)\s+)[A-Za-z0-9._~+/=-]{16,}/gi;
|
|
173
|
+
var PEM_PATTERN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|$)/g;
|
|
174
|
+
var ASSIGNMENT_PATTERN = /([A-Za-z0-9_.-]{0,64}(?:key|token|secret|passwd|password|passphrase|credentials?)(?:\\?["'])?\s*[:=]\s*)(\\"([^"\\\r\n]{12,})\\"|"((?:[^"\\\r\n]|\\[^\r\n]){12,})"|'([^'\\\r\n]{12,})'|([A-Za-z0-9+/_.=-]{16,}))/gi;
|
|
175
|
+
var BENIGN_KEY_NAME_PATTERN = /(?:^|[._-])(?:idempotency|dedupe|route)[_-]?key$/i;
|
|
176
|
+
var ASSIGNMENT_HINTS = ["key", "token", "secret", "pass", "credential"];
|
|
177
|
+
function mayContainAssignment(text) {
|
|
178
|
+
const lowered = text.toLowerCase();
|
|
179
|
+
return ASSIGNMENT_HINTS.some((hint) => lowered.includes(hint));
|
|
180
|
+
}
|
|
181
|
+
function shannonEntropy(value) {
|
|
182
|
+
const counts = new Map;
|
|
183
|
+
for (const char of value)
|
|
184
|
+
counts.set(char, (counts.get(char) ?? 0) + 1);
|
|
185
|
+
let entropy = 0;
|
|
186
|
+
for (const count of counts.values()) {
|
|
187
|
+
const probability = count / value.length;
|
|
188
|
+
entropy -= probability * Math.log2(probability);
|
|
189
|
+
}
|
|
190
|
+
return entropy;
|
|
191
|
+
}
|
|
192
|
+
function isLikelySecretValue(value) {
|
|
193
|
+
if (value.length < 12)
|
|
194
|
+
return false;
|
|
195
|
+
if (value.includes(SCRUBBED))
|
|
196
|
+
return false;
|
|
197
|
+
if (value.startsWith("/") || value.startsWith("~") || value.startsWith("./"))
|
|
198
|
+
return false;
|
|
199
|
+
return shannonEntropy(value) >= 3.2;
|
|
200
|
+
}
|
|
201
|
+
function scrubSecrets(text) {
|
|
202
|
+
if (!text)
|
|
203
|
+
return text;
|
|
204
|
+
let scrubbed = text;
|
|
205
|
+
for (const pattern of TOKEN_PATTERNS)
|
|
206
|
+
scrubbed = scrubbed.replace(pattern, SCRUBBED);
|
|
207
|
+
scrubbed = scrubbed.replace(AUTHORIZATION_PATTERN, `$1${SCRUBBED}`);
|
|
208
|
+
scrubbed = scrubbed.replace(PEM_PATTERN, SCRUBBED);
|
|
209
|
+
if (!mayContainAssignment(scrubbed))
|
|
210
|
+
return scrubbed;
|
|
211
|
+
scrubbed = scrubbed.replace(ASSIGNMENT_PATTERN, (match, prefix, quotedValue, escapedQuoted, doubleQuoted, singleQuoted, bare) => {
|
|
212
|
+
const value = escapedQuoted ?? doubleQuoted ?? singleQuoted ?? bare ?? "";
|
|
213
|
+
const keyName = /^[A-Za-z0-9_.-]*/.exec(prefix)?.[0] ?? "";
|
|
214
|
+
if (BENIGN_KEY_NAME_PATTERN.test(keyName))
|
|
215
|
+
return match;
|
|
216
|
+
if (!isLikelySecretValue(value))
|
|
217
|
+
return match;
|
|
218
|
+
const quote = quotedValue.startsWith("\\\"") ? "\\\"" : quotedValue.startsWith('"') ? '"' : quotedValue.startsWith("'") ? "'" : "";
|
|
219
|
+
return `${prefix}${quote}${SCRUBBED}${quote}`;
|
|
220
|
+
});
|
|
221
|
+
return scrubbed;
|
|
222
|
+
}
|
|
223
|
+
function scrubSecretsDeep(value) {
|
|
224
|
+
if (typeof value === "string")
|
|
225
|
+
return scrubSecrets(value);
|
|
226
|
+
if (Array.isArray(value))
|
|
227
|
+
return value.map((entry) => scrubSecretsDeep(entry));
|
|
228
|
+
if (value !== null && typeof value === "object") {
|
|
229
|
+
const toJSON = value.toJSON;
|
|
230
|
+
if (typeof toJSON === "function")
|
|
231
|
+
return scrubSecretsDeep(toJSON.call(value));
|
|
232
|
+
const scrubbed = {};
|
|
233
|
+
for (const [key, entry] of Object.entries(value))
|
|
234
|
+
scrubbed[key] = scrubSecretsDeep(entry);
|
|
235
|
+
return scrubbed;
|
|
236
|
+
}
|
|
237
|
+
return value;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/lib/recurrence.ts
|
|
46
241
|
function assertDate(value, label) {
|
|
47
242
|
const date = new Date(value);
|
|
48
243
|
if (Number.isNaN(date.getTime()))
|
|
@@ -79,7 +274,20 @@ function parseField(expr, min, max) {
|
|
|
79
274
|
}
|
|
80
275
|
return out;
|
|
81
276
|
}
|
|
277
|
+
var MINUTE_MS = 60000;
|
|
278
|
+
var PARSED_CRON_CACHE_LIMIT = 256;
|
|
279
|
+
var parsedCronCache = new Map;
|
|
280
|
+
var emittedScheduleWarnings = new Set;
|
|
281
|
+
function warnOnce(key, message) {
|
|
282
|
+
if (emittedScheduleWarnings.has(key))
|
|
283
|
+
return;
|
|
284
|
+
emittedScheduleWarnings.add(key);
|
|
285
|
+
console.warn(`[open-loops] WARN ${message}`);
|
|
286
|
+
}
|
|
82
287
|
function parseCron(expr) {
|
|
288
|
+
const cached = parsedCronCache.get(expr);
|
|
289
|
+
if (cached)
|
|
290
|
+
return cached;
|
|
83
291
|
const fields = expr.trim().split(/\s+/);
|
|
84
292
|
if (fields.length !== 5)
|
|
85
293
|
throw new Error(`cron must have 5 fields, got ${fields.length}: "${expr}"`);
|
|
@@ -89,7 +297,7 @@ function parseCron(expr) {
|
|
|
89
297
|
dowSet.delete(7);
|
|
90
298
|
dowSet.add(0);
|
|
91
299
|
}
|
|
92
|
-
|
|
300
|
+
const parsed = {
|
|
93
301
|
minute: parseField(minute, 0, 59),
|
|
94
302
|
hour: parseField(hour, 0, 23),
|
|
95
303
|
dom: parseField(dom, 1, 31),
|
|
@@ -98,6 +306,13 @@ function parseCron(expr) {
|
|
|
98
306
|
domRestricted: dom !== "*",
|
|
99
307
|
dowRestricted: dow !== "*"
|
|
100
308
|
};
|
|
309
|
+
if (parsedCronCache.size >= PARSED_CRON_CACHE_LIMIT) {
|
|
310
|
+
const oldest = parsedCronCache.keys().next().value;
|
|
311
|
+
if (oldest !== undefined)
|
|
312
|
+
parsedCronCache.delete(oldest);
|
|
313
|
+
}
|
|
314
|
+
parsedCronCache.set(expr, parsed);
|
|
315
|
+
return parsed;
|
|
101
316
|
}
|
|
102
317
|
function cronMatches(cron, date) {
|
|
103
318
|
if (!cron.minute.has(date.getMinutes()))
|
|
@@ -171,14 +386,59 @@ function latestIntervalSlot(first, now, everyMs) {
|
|
|
171
386
|
const steps = Math.floor((now.getTime() - first.getTime()) / everyMs);
|
|
172
387
|
return new Date(first.getTime() + steps * everyMs);
|
|
173
388
|
}
|
|
389
|
+
function floorToMinute(date) {
|
|
390
|
+
const floored = new Date(date.getTime());
|
|
391
|
+
floored.setSeconds(0, 0);
|
|
392
|
+
return floored;
|
|
393
|
+
}
|
|
394
|
+
var LATEST_CRON_SCAN_LIMIT_MINUTES = 366 * 24 * 60;
|
|
395
|
+
function latestDailyCronSlot(cron, now) {
|
|
396
|
+
const hoursDesc = [...cron.hour].sort((a, b) => b - a);
|
|
397
|
+
const minutesDesc = [...cron.minute].sort((a, b) => b - a);
|
|
398
|
+
const candidate = floorToMinute(now);
|
|
399
|
+
for (let dayOffset = 0;dayOffset < 2; dayOffset += 1) {
|
|
400
|
+
const isToday = dayOffset === 0;
|
|
401
|
+
for (const hour of hoursDesc) {
|
|
402
|
+
if (isToday && hour > now.getHours())
|
|
403
|
+
continue;
|
|
404
|
+
const minuteCap = isToday && hour === now.getHours() ? now.getMinutes() : 59;
|
|
405
|
+
const minute = minutesDesc.find((value) => value <= minuteCap);
|
|
406
|
+
if (minute === undefined)
|
|
407
|
+
continue;
|
|
408
|
+
candidate.setHours(hour, minute, 0, 0);
|
|
409
|
+
return candidate;
|
|
410
|
+
}
|
|
411
|
+
candidate.setDate(candidate.getDate() - 1);
|
|
412
|
+
}
|
|
413
|
+
throw new Error("unreachable: daily cron pattern must match within two days");
|
|
414
|
+
}
|
|
174
415
|
function latestCronSlot(first, now, expression) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
416
|
+
if (first.getTime() > now.getTime())
|
|
417
|
+
return first;
|
|
418
|
+
const cron = parseCron(expression);
|
|
419
|
+
if (!cron.domRestricted && !cron.dowRestricted && cron.month.size === 12) {
|
|
420
|
+
const latest = latestDailyCronSlot(cron, now);
|
|
421
|
+
return latest.getTime() >= first.getTime() ? latest : first;
|
|
422
|
+
}
|
|
423
|
+
const scanFloorMs = Math.max(first.getTime(), now.getTime() - LATEST_CRON_SCAN_LIMIT_MINUTES * MINUTE_MS);
|
|
424
|
+
const cursor = floorToMinute(now);
|
|
425
|
+
while (cursor.getTime() >= scanFloorMs) {
|
|
426
|
+
if (cronMatches(cron, cursor))
|
|
427
|
+
return cursor;
|
|
428
|
+
cursor.setMinutes(cursor.getMinutes() - 1);
|
|
181
429
|
}
|
|
430
|
+
if (first.getTime() < scanFloorMs) {
|
|
431
|
+
warnOnce(`latest-cron:${expression}`, `latestCronSlot: no match for "${expression}" within the ${LATEST_CRON_SCAN_LIMIT_MINUTES}-minute scan window; using the stored next run as the latest slot`);
|
|
432
|
+
}
|
|
433
|
+
return first;
|
|
434
|
+
}
|
|
435
|
+
var MAX_CATCH_UP_SLOTS = 1000;
|
|
436
|
+
function catchUpSlotLimit(loop) {
|
|
437
|
+
if (loop.catchUpLimit > MAX_CATCH_UP_SLOTS) {
|
|
438
|
+
warnOnce(`catch-up-limit:${loop.id}`, `dueSlots: loop ${loop.id} catchUpLimit ${loop.catchUpLimit} exceeds the per-plan cap; using ${MAX_CATCH_UP_SLOTS}`);
|
|
439
|
+
return MAX_CATCH_UP_SLOTS;
|
|
440
|
+
}
|
|
441
|
+
return loop.catchUpLimit;
|
|
182
442
|
}
|
|
183
443
|
function dueSlots(loop, now) {
|
|
184
444
|
if (!loop.nextRunAt || loop.status !== "active")
|
|
@@ -197,9 +457,10 @@ function dueSlots(loop, now) {
|
|
|
197
457
|
return { slots: [next.toISOString()] };
|
|
198
458
|
case "interval": {
|
|
199
459
|
if (catchUp === "all") {
|
|
460
|
+
const limit = catchUpSlotLimit(loop);
|
|
200
461
|
const slots = [];
|
|
201
462
|
let cursor = next;
|
|
202
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
463
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
203
464
|
slots.push(cursor.toISOString());
|
|
204
465
|
cursor = new Date(cursor.getTime() + loop.schedule.everyMs);
|
|
205
466
|
}
|
|
@@ -211,9 +472,10 @@ function dueSlots(loop, now) {
|
|
|
211
472
|
}
|
|
212
473
|
case "cron": {
|
|
213
474
|
if (catchUp === "all") {
|
|
475
|
+
const limit = catchUpSlotLimit(loop);
|
|
214
476
|
const slots = [];
|
|
215
477
|
let cursor = next;
|
|
216
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
478
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
217
479
|
slots.push(cursor.toISOString());
|
|
218
480
|
cursor = nextCronRun(loop.schedule.expression, cursor);
|
|
219
481
|
}
|
|
@@ -312,8 +574,311 @@ function updateReadyFlags(nodes, planStatus) {
|
|
|
312
574
|
}
|
|
313
575
|
|
|
314
576
|
// src/lib/workflow-spec.ts
|
|
315
|
-
import { readFileSync } from "fs";
|
|
577
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
316
578
|
import { isAbsolute, resolve } from "path";
|
|
579
|
+
|
|
580
|
+
// src/lib/agent-adapter.ts
|
|
581
|
+
import { spawn } from "child_process";
|
|
582
|
+
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS = new Set([
|
|
583
|
+
"e",
|
|
584
|
+
"exec",
|
|
585
|
+
"agent",
|
|
586
|
+
"start",
|
|
587
|
+
"--ephemeral",
|
|
588
|
+
"--ignore-rules",
|
|
589
|
+
"--skip-git-repo-check",
|
|
590
|
+
"--json",
|
|
591
|
+
"--output-last-message",
|
|
592
|
+
"-o",
|
|
593
|
+
"--output-schema",
|
|
594
|
+
"--dangerously-bypass-approvals-and-sandbox"
|
|
595
|
+
]);
|
|
596
|
+
var CODEX_LIKE_SANDBOXES = ["read-only", "workspace-write", "danger-full-access"];
|
|
597
|
+
var CURSOR_SANDBOXES = ["enabled", "disabled"];
|
|
598
|
+
var PERMISSION_MODES = ["default", "plan", "auto", "bypass"];
|
|
599
|
+
function assertOptionalNonEmptyString(value, label) {
|
|
600
|
+
if (value === undefined)
|
|
601
|
+
return;
|
|
602
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
603
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
604
|
+
}
|
|
605
|
+
function validateAgentOptions(target, label, capabilities) {
|
|
606
|
+
const provider = target.provider;
|
|
607
|
+
if (typeof target.prompt !== "string" || target.prompt.trim() === "") {
|
|
608
|
+
throw new Error(`${label}.prompt must be a non-empty string`);
|
|
609
|
+
}
|
|
610
|
+
assertOptionalNonEmptyString(target.model, `${label}.model`);
|
|
611
|
+
assertOptionalNonEmptyString(target.variant, `${label}.variant`);
|
|
612
|
+
assertOptionalNonEmptyString(target.agent, `${label}.agent`);
|
|
613
|
+
assertOptionalNonEmptyString(target.authProfile, `${label}.authProfile`);
|
|
614
|
+
assertOptionalNonEmptyString(target.configIsolation, `${label}.configIsolation`);
|
|
615
|
+
if (target.configIsolation !== undefined && target.configIsolation !== "safe" && target.configIsolation !== "none") {
|
|
616
|
+
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
617
|
+
}
|
|
618
|
+
if (target.authProfile !== undefined && provider !== "codewith") {
|
|
619
|
+
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
620
|
+
}
|
|
621
|
+
if (provider === "opencode" && (typeof target.model !== "string" || target.model.trim() === "")) {
|
|
622
|
+
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
623
|
+
}
|
|
624
|
+
if (provider === "cursor" && target.variant !== undefined) {
|
|
625
|
+
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
626
|
+
}
|
|
627
|
+
if (provider === "codex" && target.agent !== undefined) {
|
|
628
|
+
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
629
|
+
}
|
|
630
|
+
if (provider === "codewith" && target.agent !== undefined) {
|
|
631
|
+
throw new Error(`${label}.agent is not supported for provider codewith durable background-agent execution`);
|
|
632
|
+
}
|
|
633
|
+
if (provider === "codewith") {
|
|
634
|
+
const unsafe = target.extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS.has(arg));
|
|
635
|
+
if (unsafe) {
|
|
636
|
+
throw new Error(`${label}.extraArgs cannot include ${unsafe}; codewith agent steps use durable agent start, not exec/ephemeral flags`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (target.addDirs?.length && !["codewith", "codex"].includes(provider)) {
|
|
640
|
+
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
641
|
+
}
|
|
642
|
+
if (target.permissionMode !== undefined) {
|
|
643
|
+
if (!PERMISSION_MODES.includes(target.permissionMode)) {
|
|
644
|
+
throw new Error(`${label}.permissionMode must be one of ${PERMISSION_MODES.join(", ")}`);
|
|
645
|
+
}
|
|
646
|
+
if (target.permissionMode === "plan" && !["claude", "cursor"].includes(provider)) {
|
|
647
|
+
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
648
|
+
}
|
|
649
|
+
if (target.permissionMode === "auto" && provider !== "claude") {
|
|
650
|
+
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (target.sandbox !== undefined) {
|
|
654
|
+
if (!capabilities.sandbox.length) {
|
|
655
|
+
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
656
|
+
}
|
|
657
|
+
if (!capabilities.sandbox.includes(target.sandbox)) {
|
|
658
|
+
throw new Error(`${label}.sandbox must be one of ${capabilities.sandbox.join(", ")}`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function codewithLikeSandbox(target) {
|
|
663
|
+
return target.sandbox ?? (target.permissionMode === "bypass" ? "danger-full-access" : "workspace-write");
|
|
664
|
+
}
|
|
665
|
+
function configStringValue(value) {
|
|
666
|
+
return JSON.stringify(value);
|
|
667
|
+
}
|
|
668
|
+
function buildAgentInvocation(target) {
|
|
669
|
+
const isolation = target.configIsolation ?? "safe";
|
|
670
|
+
const permissionMode = target.permissionMode ?? "default";
|
|
671
|
+
const args = [];
|
|
672
|
+
switch (target.provider) {
|
|
673
|
+
case "claude": {
|
|
674
|
+
if (isolation === "safe")
|
|
675
|
+
args.push("--safe-mode", "--setting-sources", "local", "--no-session-persistence");
|
|
676
|
+
if (permissionMode !== "default") {
|
|
677
|
+
const mode = permissionMode === "bypass" ? "bypassPermissions" : permissionMode;
|
|
678
|
+
args.push("--permission-mode", mode);
|
|
679
|
+
}
|
|
680
|
+
args.push("-p", "--output-format", "json");
|
|
681
|
+
if (target.model)
|
|
682
|
+
args.push("--model", target.model);
|
|
683
|
+
if (target.variant)
|
|
684
|
+
args.push("--effort", target.variant);
|
|
685
|
+
if (target.agent)
|
|
686
|
+
args.push("--agent", target.agent);
|
|
687
|
+
args.push(...target.extraArgs ?? []);
|
|
688
|
+
return { command: "claude", args, stdin: target.prompt };
|
|
689
|
+
}
|
|
690
|
+
case "cursor": {
|
|
691
|
+
args.push("-c", [
|
|
692
|
+
"set -eu",
|
|
693
|
+
"if command -v agent >/dev/null 2>&1; then",
|
|
694
|
+
' exec agent "$@"',
|
|
695
|
+
"else",
|
|
696
|
+
" echo 'Executable not found in PATH: agent' >&2",
|
|
697
|
+
" exit 127",
|
|
698
|
+
"fi"
|
|
699
|
+
].join(`
|
|
700
|
+
`), "openloops-cursor", "-p");
|
|
701
|
+
if (permissionMode === "plan")
|
|
702
|
+
args.push("--mode", "plan");
|
|
703
|
+
if (permissionMode === "bypass")
|
|
704
|
+
args.push("--force");
|
|
705
|
+
const cursorSandbox = target.sandbox ?? (isolation === "safe" ? "enabled" : undefined);
|
|
706
|
+
if (cursorSandbox)
|
|
707
|
+
args.push("--sandbox", cursorSandbox);
|
|
708
|
+
if (target.model)
|
|
709
|
+
args.push("--model", target.model);
|
|
710
|
+
if (target.agent)
|
|
711
|
+
args.push("--agent", target.agent);
|
|
712
|
+
args.push(...target.extraArgs ?? []);
|
|
713
|
+
return { command: "sh", args, stdin: target.prompt, preflightAnyOf: ["agent"] };
|
|
714
|
+
}
|
|
715
|
+
case "codewith": {
|
|
716
|
+
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : []);
|
|
717
|
+
if (target.variant)
|
|
718
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
719
|
+
args.push("--ask-for-approval", "never", "--sandbox", codewithLikeSandbox(target));
|
|
720
|
+
if (target.cwd)
|
|
721
|
+
args.push("--cd", target.cwd);
|
|
722
|
+
for (const dir of target.addDirs ?? [])
|
|
723
|
+
args.push("--add-dir", dir);
|
|
724
|
+
if (target.model)
|
|
725
|
+
args.push("--model", target.model);
|
|
726
|
+
args.push(...target.extraArgs ?? []);
|
|
727
|
+
args.push("agent", "start");
|
|
728
|
+
if (target.cwd)
|
|
729
|
+
args.push("--cwd", target.cwd);
|
|
730
|
+
args.push(target.prompt);
|
|
731
|
+
return { command: "codewith", args };
|
|
732
|
+
}
|
|
733
|
+
case "codex": {
|
|
734
|
+
if (target.variant)
|
|
735
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
736
|
+
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
737
|
+
if (isolation === "safe")
|
|
738
|
+
args.push("--ignore-rules");
|
|
739
|
+
if (target.cwd)
|
|
740
|
+
args.push("--cd", target.cwd);
|
|
741
|
+
for (const dir of target.addDirs ?? [])
|
|
742
|
+
args.push("--add-dir", dir);
|
|
743
|
+
if (target.model)
|
|
744
|
+
args.push("--model", target.model);
|
|
745
|
+
args.push(...target.extraArgs ?? []);
|
|
746
|
+
return { command: "codex", args, stdin: target.prompt };
|
|
747
|
+
}
|
|
748
|
+
case "aicopilot":
|
|
749
|
+
case "opencode": {
|
|
750
|
+
args.push("run", "--format", "json");
|
|
751
|
+
if (isolation === "safe")
|
|
752
|
+
args.push("--pure");
|
|
753
|
+
if (permissionMode === "bypass")
|
|
754
|
+
args.push("--dangerously-skip-permissions");
|
|
755
|
+
if (target.cwd)
|
|
756
|
+
args.push("--dir", target.cwd);
|
|
757
|
+
if (target.model)
|
|
758
|
+
args.push("--model", target.model);
|
|
759
|
+
if (target.variant)
|
|
760
|
+
args.push("--variant", target.variant);
|
|
761
|
+
if (target.agent)
|
|
762
|
+
args.push("--agent", target.agent);
|
|
763
|
+
args.push(...target.extraArgs ?? []);
|
|
764
|
+
return { command: target.provider, args, stdin: target.prompt };
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
function adapterFor(provider, capabilities) {
|
|
769
|
+
return {
|
|
770
|
+
provider,
|
|
771
|
+
capabilities,
|
|
772
|
+
validate(target, label = provider) {
|
|
773
|
+
validateAgentOptions(target, label, capabilities);
|
|
774
|
+
},
|
|
775
|
+
buildInvocation(target) {
|
|
776
|
+
validateAgentOptions(target, provider, capabilities);
|
|
777
|
+
return buildAgentInvocation(target);
|
|
778
|
+
}
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
var PROVIDER_ADAPTERS = {
|
|
782
|
+
claude: adapterFor("claude", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
783
|
+
cursor: adapterFor("cursor", { sandbox: CURSOR_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
784
|
+
codewith: adapterFor("codewith", { sandbox: CODEX_LIKE_SANDBOXES, durable: true, remote: false, promptChannel: "argv" }),
|
|
785
|
+
codex: adapterFor("codex", { sandbox: CODEX_LIKE_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
786
|
+
aicopilot: adapterFor("aicopilot", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
787
|
+
opencode: adapterFor("opencode", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" })
|
|
788
|
+
};
|
|
789
|
+
var AGENT_PROVIDERS = Object.keys(PROVIDER_ADAPTERS);
|
|
790
|
+
function providerAdapter(provider) {
|
|
791
|
+
const adapter = PROVIDER_ADAPTERS[provider];
|
|
792
|
+
if (!adapter)
|
|
793
|
+
throw new Error(`unsupported agent provider: ${String(provider)}`);
|
|
794
|
+
return adapter;
|
|
795
|
+
}
|
|
796
|
+
var DEFAULT_CAPTURE_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
797
|
+
function killProcessGroup(pgid) {
|
|
798
|
+
try {
|
|
799
|
+
process.kill(-pgid, "SIGTERM");
|
|
800
|
+
} catch {
|
|
801
|
+
try {
|
|
802
|
+
process.kill(pgid, "SIGTERM");
|
|
803
|
+
} catch {}
|
|
804
|
+
}
|
|
805
|
+
setTimeout(() => {
|
|
806
|
+
try {
|
|
807
|
+
process.kill(-pgid, "SIGKILL");
|
|
808
|
+
} catch {
|
|
809
|
+
try {
|
|
810
|
+
process.kill(pgid, "SIGKILL");
|
|
811
|
+
} catch {}
|
|
812
|
+
}
|
|
813
|
+
}, 2000).unref();
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
class BoundedOutputBuffer {
|
|
817
|
+
maxBytes;
|
|
818
|
+
text = "";
|
|
819
|
+
truncatedBytes = 0;
|
|
820
|
+
constructor(maxBytes) {
|
|
821
|
+
this.maxBytes = maxBytes;
|
|
822
|
+
}
|
|
823
|
+
append(chunk) {
|
|
824
|
+
if (!chunk)
|
|
825
|
+
return;
|
|
826
|
+
this.text += chunk;
|
|
827
|
+
if (Buffer.byteLength(this.text, "utf8") <= this.maxBytes)
|
|
828
|
+
return;
|
|
829
|
+
this.text = scrubSecrets(this.text);
|
|
830
|
+
const encoded = Buffer.from(this.text, "utf8");
|
|
831
|
+
if (encoded.length <= this.maxBytes)
|
|
832
|
+
return;
|
|
833
|
+
let start = encoded.length - this.maxBytes;
|
|
834
|
+
while (start < encoded.length && (encoded[start] & 192) === 128)
|
|
835
|
+
start++;
|
|
836
|
+
this.truncatedBytes += start;
|
|
837
|
+
this.text = encoded.subarray(start).toString("utf8");
|
|
838
|
+
}
|
|
839
|
+
value() {
|
|
840
|
+
if (this.truncatedBytes === 0)
|
|
841
|
+
return this.text;
|
|
842
|
+
return `[truncated ${this.truncatedBytes} bytes]
|
|
843
|
+
${this.text}`;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
async function spawnCapture(command, args, opts) {
|
|
847
|
+
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_CAPTURE_MAX_OUTPUT_BYTES;
|
|
848
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
849
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
850
|
+
let timedOut = false;
|
|
851
|
+
let error;
|
|
852
|
+
const child = spawn(command, args, {
|
|
853
|
+
cwd: opts.cwd,
|
|
854
|
+
env: opts.env,
|
|
855
|
+
detached: true,
|
|
856
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
857
|
+
});
|
|
858
|
+
const timer = setTimeout(() => {
|
|
859
|
+
timedOut = true;
|
|
860
|
+
if (child.pid)
|
|
861
|
+
killProcessGroup(child.pid);
|
|
862
|
+
}, opts.timeoutMs);
|
|
863
|
+
timer.unref();
|
|
864
|
+
child.stdout?.setEncoding("utf8");
|
|
865
|
+
child.stderr?.setEncoding("utf8");
|
|
866
|
+
child.stdout?.on("data", (chunk) => stdout.append(chunk));
|
|
867
|
+
child.stderr?.on("data", (chunk) => stderr.append(chunk));
|
|
868
|
+
const [status, signal] = await new Promise((resolve) => {
|
|
869
|
+
child.once("error", (err) => {
|
|
870
|
+
error = err.message;
|
|
871
|
+
resolve([null, null]);
|
|
872
|
+
});
|
|
873
|
+
child.once("close", (code, sig) => resolve([code, sig]));
|
|
874
|
+
});
|
|
875
|
+
clearTimeout(timer);
|
|
876
|
+
if (timedOut && !error)
|
|
877
|
+
error = `timed out after ${opts.timeoutMs}ms`;
|
|
878
|
+
return { status, signal, stdout: stdout.value(), stderr: stderr.value(), error, timedOut };
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// src/lib/workflow-spec.ts
|
|
317
882
|
function assertObject(value, label) {
|
|
318
883
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
319
884
|
throw new Error(`${label} must be an object`);
|
|
@@ -356,20 +921,6 @@ function optionalStringArray(value, label) {
|
|
|
356
921
|
}).filter(Boolean);
|
|
357
922
|
return values.length ? values : undefined;
|
|
358
923
|
}
|
|
359
|
-
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS = new Set([
|
|
360
|
-
"e",
|
|
361
|
-
"exec",
|
|
362
|
-
"agent",
|
|
363
|
-
"start",
|
|
364
|
-
"--ephemeral",
|
|
365
|
-
"--ignore-rules",
|
|
366
|
-
"--skip-git-repo-check",
|
|
367
|
-
"--json",
|
|
368
|
-
"--output-last-message",
|
|
369
|
-
"-o",
|
|
370
|
-
"--output-schema",
|
|
371
|
-
"--dangerously-bypass-approvals-and-sandbox"
|
|
372
|
-
]);
|
|
373
924
|
function optionalAccountRef(value, label) {
|
|
374
925
|
if (value === undefined)
|
|
375
926
|
return;
|
|
@@ -390,7 +941,7 @@ function readPromptFile(rawPath, label, opts) {
|
|
|
390
941
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
391
942
|
let prompt;
|
|
392
943
|
try {
|
|
393
|
-
prompt =
|
|
944
|
+
prompt = readFileSync2(path, "utf8");
|
|
394
945
|
} catch (error) {
|
|
395
946
|
const code = error && typeof error === "object" && "code" in error ? String(error.code) : undefined;
|
|
396
947
|
throw new Error(`${label}.promptFile could not be read${code ? `: ${code}` : ""}`);
|
|
@@ -409,7 +960,7 @@ function matchingPromptSource(value, prompt, opts) {
|
|
|
409
960
|
return;
|
|
410
961
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
411
962
|
try {
|
|
412
|
-
return
|
|
963
|
+
return readFileSync2(path, "utf8") === prompt ? { type: "file", path } : undefined;
|
|
413
964
|
} catch {
|
|
414
965
|
return;
|
|
415
966
|
}
|
|
@@ -456,74 +1007,14 @@ function validateTarget(value, label, opts) {
|
|
|
456
1007
|
if (!hasPrompt && !hasPromptFile)
|
|
457
1008
|
throw new Error(`${label}.prompt must be a non-empty string or ${label}.promptFile must be set`);
|
|
458
1009
|
const promptFields = hasPrompt ? { prompt: value.prompt, promptSource: matchingPromptSource(value.promptSource, value.prompt, opts) } : readPromptFile(value.promptFile, label, opts);
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
1010
|
+
if (!AGENT_PROVIDERS.includes(value.provider)) {
|
|
1011
|
+
throw new Error(`${label}.provider must be one of ${AGENT_PROVIDERS.join(", ")}`);
|
|
1012
|
+
}
|
|
462
1013
|
optionalTimeoutMs(value.timeoutMs, `${label}.timeoutMs`);
|
|
463
1014
|
optionalPositiveInteger(value.idleTimeoutMs, `${label}.idleTimeoutMs`);
|
|
464
|
-
if (value.authProfile !== undefined) {
|
|
465
|
-
assertString(value.authProfile, `${label}.authProfile`);
|
|
466
|
-
if (value.provider !== "codewith")
|
|
467
|
-
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
468
|
-
}
|
|
469
|
-
if (value.configIsolation !== undefined) {
|
|
470
|
-
assertString(value.configIsolation, `${label}.configIsolation`);
|
|
471
|
-
if (!["safe", "none"].includes(value.configIsolation))
|
|
472
|
-
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
473
|
-
}
|
|
474
|
-
if (value.model !== undefined)
|
|
475
|
-
assertString(value.model, `${label}.model`);
|
|
476
|
-
if (value.provider === "opencode" && (typeof value.model !== "string" || value.model.trim() === "")) {
|
|
477
|
-
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
478
|
-
}
|
|
479
|
-
if (value.variant !== undefined)
|
|
480
|
-
assertString(value.variant, `${label}.variant`);
|
|
481
|
-
if (value.agent !== undefined)
|
|
482
|
-
assertString(value.agent, `${label}.agent`);
|
|
483
|
-
if (value.provider === "cursor" && value.variant !== undefined)
|
|
484
|
-
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
485
|
-
if (value.provider === "codex" && value.agent !== undefined)
|
|
486
|
-
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
487
|
-
if (value.provider === "codewith" && value.agent !== undefined) {
|
|
488
|
-
throw new Error(`${label}.agent is not supported for provider codewith durable background-agent execution`);
|
|
489
|
-
}
|
|
490
1015
|
const extraArgs = optionalStringArray(value.extraArgs, `${label}.extraArgs`);
|
|
491
|
-
if (value.provider === "codewith") {
|
|
492
|
-
const unsafe = extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS.has(arg));
|
|
493
|
-
if (unsafe)
|
|
494
|
-
throw new Error(`${label}.extraArgs cannot include ${unsafe}; codewith agent steps use durable agent start, not exec/ephemeral flags`);
|
|
495
|
-
}
|
|
496
1016
|
optionalStringArray(value.addDirs, `${label}.addDirs`);
|
|
497
|
-
|
|
498
|
-
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
499
|
-
}
|
|
500
|
-
if (value.permissionMode !== undefined) {
|
|
501
|
-
assertString(value.permissionMode, `${label}.permissionMode`);
|
|
502
|
-
const permissionModes = ["default", "plan", "auto", "bypass"];
|
|
503
|
-
if (!permissionModes.includes(value.permissionMode)) {
|
|
504
|
-
throw new Error(`${label}.permissionMode must be one of ${permissionModes.join(", ")}`);
|
|
505
|
-
}
|
|
506
|
-
if (value.permissionMode === "plan" && !["claude", "cursor"].includes(value.provider)) {
|
|
507
|
-
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
508
|
-
}
|
|
509
|
-
if (value.permissionMode === "auto" && value.provider !== "claude") {
|
|
510
|
-
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
|
-
if (value.sandbox !== undefined) {
|
|
514
|
-
assertString(value.sandbox, `${label}.sandbox`);
|
|
515
|
-
const codexLike = ["read-only", "workspace-write", "danger-full-access"];
|
|
516
|
-
const cursorLike = ["enabled", "disabled"];
|
|
517
|
-
if (["codewith", "codex"].includes(value.provider)) {
|
|
518
|
-
if (!codexLike.includes(value.sandbox))
|
|
519
|
-
throw new Error(`${label}.sandbox must be one of ${codexLike.join(", ")}`);
|
|
520
|
-
} else if (value.provider === "cursor") {
|
|
521
|
-
if (!cursorLike.includes(value.sandbox))
|
|
522
|
-
throw new Error(`${label}.sandbox must be one of ${cursorLike.join(", ")}`);
|
|
523
|
-
} else {
|
|
524
|
-
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
525
|
-
}
|
|
526
|
-
}
|
|
1017
|
+
providerAdapter(value.provider).validate({ ...value, extraArgs, ...promptFields }, label);
|
|
527
1018
|
if (value.allowlist !== undefined) {
|
|
528
1019
|
assertObject(value.allowlist, `${label}.allowlist`);
|
|
529
1020
|
optionalStringArray(value.allowlist.tools, `${label}.allowlist.tools`);
|
|
@@ -653,8 +1144,8 @@ function workflowBodyFromJson(value, fallbackName, opts = {}) {
|
|
|
653
1144
|
|
|
654
1145
|
// src/lib/run-artifacts.ts
|
|
655
1146
|
import { createHash } from "crypto";
|
|
656
|
-
import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
|
|
657
|
-
import { basename, join as join2 } from "path";
|
|
1147
|
+
import { mkdirSync as mkdirSync2, renameSync, rmdirSync, rmSync, writeFileSync } from "fs";
|
|
1148
|
+
import { basename, dirname, join as join2 } from "path";
|
|
658
1149
|
function shortHash(value) {
|
|
659
1150
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
660
1151
|
}
|
|
@@ -675,13 +1166,14 @@ function workflowRunProjectSlug(projectKey) {
|
|
|
675
1166
|
return "global";
|
|
676
1167
|
return safeRunPathSlug(projectKey.startsWith("/") ? basename(projectKey) : projectKey, "project");
|
|
677
1168
|
}
|
|
678
|
-
function
|
|
1169
|
+
function stageWorkflowRunManifest(args) {
|
|
679
1170
|
const projectSlug = workflowRunProjectSlug(args.projectKey);
|
|
680
1171
|
const subjectKey = workflowRunSubjectKey(args.subjectKind, args.rawSubjectRef);
|
|
681
1172
|
const dir = join2(args.loopsDataDir, "runs", projectSlug, subjectKey, args.workflowRunId);
|
|
682
1173
|
mkdirSync2(dir, { recursive: true, mode: 448 });
|
|
683
1174
|
const manifestPath = join2(dir, "manifest.json");
|
|
684
|
-
|
|
1175
|
+
const tmpPath = `${manifestPath}.tmp`;
|
|
1176
|
+
writeFileSync(tmpPath, JSON.stringify({
|
|
685
1177
|
version: 1,
|
|
686
1178
|
workflowRunId: args.workflowRunId,
|
|
687
1179
|
workflowId: args.workflowId,
|
|
@@ -694,13 +1186,26 @@ function writeWorkflowRunManifest(args) {
|
|
|
694
1186
|
createdAt: new Date().toISOString(),
|
|
695
1187
|
...args.payload
|
|
696
1188
|
}, null, 2), { mode: 384 });
|
|
697
|
-
return manifestPath;
|
|
1189
|
+
return { manifestPath, tmpPath };
|
|
1190
|
+
}
|
|
1191
|
+
function commitWorkflowRunManifest(staged) {
|
|
1192
|
+
renameSync(staged.tmpPath, staged.manifestPath);
|
|
1193
|
+
return staged.manifestPath;
|
|
1194
|
+
}
|
|
1195
|
+
function discardWorkflowRunManifest(staged) {
|
|
1196
|
+
rmSync(staged.tmpPath, { force: true });
|
|
1197
|
+
try {
|
|
1198
|
+
rmdirSync(dirname(staged.manifestPath));
|
|
1199
|
+
} catch {}
|
|
698
1200
|
}
|
|
699
1201
|
|
|
700
1202
|
// src/lib/store.ts
|
|
701
1203
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
702
1204
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
703
1205
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1206
|
+
var SCHEMA_USER_VERSION = 6;
|
|
1207
|
+
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1208
|
+
var PRUNE_BATCH_SIZE = 400;
|
|
704
1209
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
705
1210
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
706
1211
|
function rowToLoop(row) {
|
|
@@ -741,6 +1246,8 @@ function rowToRun(row) {
|
|
|
741
1246
|
claimedBy: row.claimed_by ?? undefined,
|
|
742
1247
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
743
1248
|
pid: row.pid ?? undefined,
|
|
1249
|
+
pgid: row.pgid ?? undefined,
|
|
1250
|
+
processStartedAt: row.process_started_at ?? undefined,
|
|
744
1251
|
exitCode: row.exit_code ?? undefined,
|
|
745
1252
|
durationMs: row.duration_ms ?? undefined,
|
|
746
1253
|
stdout: row.stdout ?? undefined,
|
|
@@ -926,6 +1433,24 @@ function isProcessAlive(pid) {
|
|
|
926
1433
|
return false;
|
|
927
1434
|
}
|
|
928
1435
|
}
|
|
1436
|
+
function isRecordedProcessAlive(pid, processStartedAt) {
|
|
1437
|
+
if (!pid || !isProcessAlive(pid))
|
|
1438
|
+
return false;
|
|
1439
|
+
return sameProcessStart(processStartedAt ?? undefined, processStartTimeMs(pid));
|
|
1440
|
+
}
|
|
1441
|
+
function isoProcessStart(pid) {
|
|
1442
|
+
const startedMs = processStartTimeMs(pid);
|
|
1443
|
+
return startedMs === undefined ? undefined : new Date(startedMs).toISOString();
|
|
1444
|
+
}
|
|
1445
|
+
function isLiveStepProcess(pid, stepStartedAt) {
|
|
1446
|
+
if (!isProcessAlive(pid))
|
|
1447
|
+
return false;
|
|
1448
|
+
const actualMs = processStartTimeMs(pid);
|
|
1449
|
+
const stepStartMs = stepStartedAt ? Date.parse(stepStartedAt) : Number.NaN;
|
|
1450
|
+
if (actualMs === undefined || !Number.isFinite(stepStartMs))
|
|
1451
|
+
return true;
|
|
1452
|
+
return actualMs >= stepStartMs - START_TIME_TOLERANCE_MS;
|
|
1453
|
+
}
|
|
929
1454
|
function rowToLease(row) {
|
|
930
1455
|
return {
|
|
931
1456
|
id: row.id,
|
|
@@ -945,6 +1470,9 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
945
1470
|
}
|
|
946
1471
|
return;
|
|
947
1472
|
}
|
|
1473
|
+
function scrubbedOrNull(value) {
|
|
1474
|
+
return value == null ? null : scrubSecrets(value);
|
|
1475
|
+
}
|
|
948
1476
|
function chmodIfExists(path, mode) {
|
|
949
1477
|
try {
|
|
950
1478
|
if (existsSync(path))
|
|
@@ -952,7 +1480,7 @@ function chmodIfExists(path, mode) {
|
|
|
952
1480
|
} catch {}
|
|
953
1481
|
}
|
|
954
1482
|
function ensurePrivateStorePath(file) {
|
|
955
|
-
const dir =
|
|
1483
|
+
const dir = dirname2(file);
|
|
956
1484
|
mkdirSync3(dir, { recursive: true, mode: 448 });
|
|
957
1485
|
chmodIfExists(dir, 448);
|
|
958
1486
|
chmodIfExists(file, 384);
|
|
@@ -967,14 +1495,19 @@ class Store {
|
|
|
967
1495
|
const file = path ?? dbPath();
|
|
968
1496
|
if (file !== ":memory:")
|
|
969
1497
|
ensurePrivateStorePath(file);
|
|
970
|
-
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) :
|
|
1498
|
+
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
|
|
971
1499
|
this.db = new Database(file);
|
|
972
1500
|
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
973
1501
|
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
974
1502
|
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
975
1503
|
if (file !== ":memory:")
|
|
976
1504
|
ensurePrivateStorePath(file);
|
|
977
|
-
|
|
1505
|
+
try {
|
|
1506
|
+
this.migrate();
|
|
1507
|
+
} catch (error) {
|
|
1508
|
+
this.db.close();
|
|
1509
|
+
throw error;
|
|
1510
|
+
}
|
|
978
1511
|
}
|
|
979
1512
|
migrate() {
|
|
980
1513
|
this.db.exec(`
|
|
@@ -982,7 +1515,69 @@ class Store {
|
|
|
982
1515
|
id TEXT PRIMARY KEY,
|
|
983
1516
|
applied_at TEXT NOT NULL
|
|
984
1517
|
);
|
|
985
|
-
|
|
1518
|
+
`);
|
|
1519
|
+
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1520
|
+
const userVersion = versionRow?.user_version ?? 0;
|
|
1521
|
+
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1522
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1523
|
+
}
|
|
1524
|
+
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1525
|
+
for (const migration of this.migrations()) {
|
|
1526
|
+
if (!migration.baseline && applied.has(migration.id))
|
|
1527
|
+
continue;
|
|
1528
|
+
migration.apply();
|
|
1529
|
+
if (!applied.has(migration.id)) {
|
|
1530
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
if (userVersion !== SCHEMA_USER_VERSION)
|
|
1534
|
+
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
1535
|
+
}
|
|
1536
|
+
migrations() {
|
|
1537
|
+
return [
|
|
1538
|
+
{ id: "0001_initial_and_workflows", baseline: true, apply: () => this.createBaseSchema() },
|
|
1539
|
+
{ id: "0002_loop_machines", baseline: true, apply: () => this.addColumnIfMissing("loops", "machine_json", "TEXT") },
|
|
1540
|
+
{
|
|
1541
|
+
id: "0003_goals",
|
|
1542
|
+
baseline: true,
|
|
1543
|
+
apply: () => {
|
|
1544
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1545
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1546
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1547
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1548
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1549
|
+
}
|
|
1550
|
+
},
|
|
1551
|
+
{
|
|
1552
|
+
id: "0004_loop_archive_metadata",
|
|
1553
|
+
baseline: true,
|
|
1554
|
+
apply: () => {
|
|
1555
|
+
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1556
|
+
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1557
|
+
}
|
|
1558
|
+
},
|
|
1559
|
+
{
|
|
1560
|
+
id: "0005_workflow_invocations_and_admission",
|
|
1561
|
+
baseline: true,
|
|
1562
|
+
apply: () => {
|
|
1563
|
+
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1564
|
+
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1565
|
+
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1566
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1567
|
+
this.createWorkflowRunBackfillIndexes();
|
|
1568
|
+
}
|
|
1569
|
+
},
|
|
1570
|
+
{
|
|
1571
|
+
id: "0006_run_process_tracking",
|
|
1572
|
+
apply: () => {
|
|
1573
|
+
this.addColumnIfMissing("loop_runs", "pgid", "INTEGER");
|
|
1574
|
+
this.addColumnIfMissing("loop_runs", "process_started_at", "TEXT");
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
];
|
|
1578
|
+
}
|
|
1579
|
+
createBaseSchema() {
|
|
1580
|
+
this.db.exec(`
|
|
986
1581
|
CREATE TABLE IF NOT EXISTS loops (
|
|
987
1582
|
id TEXT PRIMARY KEY,
|
|
988
1583
|
name TEXT NOT NULL,
|
|
@@ -1021,6 +1616,8 @@ class Store {
|
|
|
1021
1616
|
claimed_by TEXT,
|
|
1022
1617
|
lease_expires_at TEXT,
|
|
1023
1618
|
pid INTEGER,
|
|
1619
|
+
pgid INTEGER,
|
|
1620
|
+
process_started_at TEXT,
|
|
1024
1621
|
exit_code INTEGER,
|
|
1025
1622
|
duration_ms INTEGER,
|
|
1026
1623
|
stdout TEXT,
|
|
@@ -1245,24 +1842,6 @@ class Store {
|
|
|
1245
1842
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
1246
1843
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
1247
1844
|
`);
|
|
1248
|
-
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
1249
|
-
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1250
|
-
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1251
|
-
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1252
|
-
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1253
|
-
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1254
|
-
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1255
|
-
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1256
|
-
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1257
|
-
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1258
|
-
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1259
|
-
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1260
|
-
this.createWorkflowRunBackfillIndexes();
|
|
1261
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
1262
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
1263
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
1264
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0004_loop_archive_metadata", nowIso());
|
|
1265
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0005_workflow_invocations_and_admission", nowIso());
|
|
1266
1845
|
}
|
|
1267
1846
|
addColumnIfMissing(table, column, definition) {
|
|
1268
1847
|
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
@@ -1276,6 +1855,9 @@ class Store {
|
|
|
1276
1855
|
CREATE INDEX IF NOT EXISTS idx_workflow_runs_work_item ON workflow_runs(work_item_id);
|
|
1277
1856
|
`);
|
|
1278
1857
|
}
|
|
1858
|
+
transact(fn) {
|
|
1859
|
+
return this.db.inTransaction ? fn() : this.writeTransaction(fn);
|
|
1860
|
+
}
|
|
1279
1861
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
1280
1862
|
if (!opts.daemonLeaseId)
|
|
1281
1863
|
return;
|
|
@@ -1357,14 +1939,14 @@ class Store {
|
|
|
1357
1939
|
return byId;
|
|
1358
1940
|
const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1359
1941
|
if (rows.length === 0)
|
|
1360
|
-
throw new
|
|
1942
|
+
throw new LoopNotFoundError(idOrName);
|
|
1361
1943
|
if (rows.length > 1)
|
|
1362
|
-
throw new
|
|
1944
|
+
throw new AmbiguousNameError(idOrName);
|
|
1363
1945
|
return rowToLoop(rows[0]);
|
|
1364
1946
|
}
|
|
1365
1947
|
requireLoop(idOrName) {
|
|
1366
1948
|
return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
|
|
1367
|
-
throw new
|
|
1949
|
+
throw new LoopNotFoundError(idOrName);
|
|
1368
1950
|
})();
|
|
1369
1951
|
}
|
|
1370
1952
|
listLoops(opts = {}) {
|
|
@@ -1401,7 +1983,9 @@ class Store {
|
|
|
1401
1983
|
try {
|
|
1402
1984
|
const current = this.getLoop(id);
|
|
1403
1985
|
if (!current)
|
|
1404
|
-
throw new
|
|
1986
|
+
throw new LoopNotFoundError(id);
|
|
1987
|
+
if (current.archivedAt)
|
|
1988
|
+
throw new LoopArchivedError(current.name || id);
|
|
1405
1989
|
const merged = { ...current, ...patch, updatedAt: updated };
|
|
1406
1990
|
const res = this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
1407
1991
|
expires_at=$expiresAt, updated_at=$updated
|
|
@@ -1633,10 +2217,10 @@ class Store {
|
|
|
1633
2217
|
renameLoop(id, name, opts = {}) {
|
|
1634
2218
|
const current = this.getLoop(id);
|
|
1635
2219
|
if (!current)
|
|
1636
|
-
throw new
|
|
2220
|
+
throw new LoopNotFoundError(id);
|
|
1637
2221
|
const trimmed = name.trim();
|
|
1638
2222
|
if (!trimmed)
|
|
1639
|
-
throw new
|
|
2223
|
+
throw new ValidationError("loop name must not be empty");
|
|
1640
2224
|
const updated = (opts.now ?? new Date).toISOString();
|
|
1641
2225
|
this.db.query(`UPDATE loops SET name=$name, updated_at=$updated
|
|
1642
2226
|
WHERE id=$id
|
|
@@ -1655,25 +2239,27 @@ class Store {
|
|
|
1655
2239
|
return after;
|
|
1656
2240
|
}
|
|
1657
2241
|
archiveLoop(idOrName) {
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
2242
|
+
return this.transact(() => {
|
|
2243
|
+
const loop = this.requireLoop(idOrName);
|
|
2244
|
+
if (loop.archivedAt)
|
|
2245
|
+
return loop;
|
|
2246
|
+
const updated = nowIso();
|
|
2247
|
+
const archivedStatus = loop.status === "active" ? "paused" : loop.status;
|
|
2248
|
+
this.db.query(`UPDATE loops
|
|
2249
|
+
SET status=$status, archived_at=$archivedAt, archived_from_status=$archivedFromStatus, updated_at=$updated
|
|
2250
|
+
WHERE id=$id`).run({
|
|
2251
|
+
$id: loop.id,
|
|
2252
|
+
$status: archivedStatus,
|
|
2253
|
+
$archivedAt: updated,
|
|
2254
|
+
$archivedFromStatus: loop.status,
|
|
2255
|
+
$updated: updated
|
|
2256
|
+
});
|
|
2257
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
2258
|
+
const archived = this.getLoop(loop.id);
|
|
2259
|
+
if (!archived)
|
|
2260
|
+
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
2261
|
+
return archived;
|
|
1671
2262
|
});
|
|
1672
|
-
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
1673
|
-
const archived = this.getLoop(loop.id);
|
|
1674
|
-
if (!archived)
|
|
1675
|
-
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
1676
|
-
return archived;
|
|
1677
2263
|
}
|
|
1678
2264
|
unarchiveLoop(idOrName) {
|
|
1679
2265
|
const loop = this.requireLoop(idOrName);
|
|
@@ -1694,10 +2280,12 @@ class Store {
|
|
|
1694
2280
|
return unarchived;
|
|
1695
2281
|
}
|
|
1696
2282
|
deleteLoop(idOrName) {
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
2283
|
+
return this.transact(() => {
|
|
2284
|
+
const loop = this.requireLoop(idOrName);
|
|
2285
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
|
|
2286
|
+
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
2287
|
+
return res.changes > 0;
|
|
2288
|
+
});
|
|
1701
2289
|
}
|
|
1702
2290
|
createWorkflow(input) {
|
|
1703
2291
|
const normalized = normalizeCreateWorkflowInput(input);
|
|
@@ -2214,26 +2802,7 @@ class Store {
|
|
|
2214
2802
|
try {
|
|
2215
2803
|
this.assertDaemonLeaseFence(opts, now);
|
|
2216
2804
|
for (const node of withReady) {
|
|
2217
|
-
this.
|
|
2218
|
-
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2219
|
-
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2220
|
-
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2221
|
-
$id: node.nodeId,
|
|
2222
|
-
$goalId: goal.goalId,
|
|
2223
|
-
$planId: goal.planId,
|
|
2224
|
-
$key: node.key,
|
|
2225
|
-
$sequence: node.sequence,
|
|
2226
|
-
$priority: node.priority,
|
|
2227
|
-
$objective: node.objective,
|
|
2228
|
-
$status: node.status,
|
|
2229
|
-
$ready: node.ready ? 1 : 0,
|
|
2230
|
-
$tokenBudget: node.tokenBudget ?? null,
|
|
2231
|
-
$tokensUsed: node.tokensUsed,
|
|
2232
|
-
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2233
|
-
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2234
|
-
$created: node.createdAt,
|
|
2235
|
-
$updated: node.updatedAt
|
|
2236
|
-
});
|
|
2805
|
+
this.insertGoalPlanNode(goal, node);
|
|
2237
2806
|
}
|
|
2238
2807
|
this.db.exec("COMMIT");
|
|
2239
2808
|
} catch (error) {
|
|
@@ -2244,6 +2813,38 @@ class Store {
|
|
|
2244
2813
|
}
|
|
2245
2814
|
return this.listGoalPlanNodes(goalId);
|
|
2246
2815
|
}
|
|
2816
|
+
insertGoalPlanNode(goal, node) {
|
|
2817
|
+
let id = node.nodeId;
|
|
2818
|
+
for (let attempt = 0;attempt < 3; attempt += 1) {
|
|
2819
|
+
const res = this.db.query(`INSERT OR IGNORE INTO goal_plan_nodes (id, goal_id, plan_id, key, sequence, priority, objective, status, ready,
|
|
2820
|
+
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2821
|
+
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2822
|
+
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2823
|
+
$id: id,
|
|
2824
|
+
$goalId: goal.goalId,
|
|
2825
|
+
$planId: goal.planId,
|
|
2826
|
+
$key: node.key,
|
|
2827
|
+
$sequence: node.sequence,
|
|
2828
|
+
$priority: node.priority,
|
|
2829
|
+
$objective: node.objective,
|
|
2830
|
+
$status: node.status,
|
|
2831
|
+
$ready: node.ready ? 1 : 0,
|
|
2832
|
+
$tokenBudget: node.tokenBudget ?? null,
|
|
2833
|
+
$tokensUsed: node.tokensUsed,
|
|
2834
|
+
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2835
|
+
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2836
|
+
$created: node.createdAt,
|
|
2837
|
+
$updated: node.updatedAt
|
|
2838
|
+
});
|
|
2839
|
+
if (res.changes === 1)
|
|
2840
|
+
return;
|
|
2841
|
+
const existingByKey = this.db.query("SELECT id FROM goal_plan_nodes WHERE plan_id = ? AND key = ? LIMIT 1").get(goal.planId, node.key);
|
|
2842
|
+
if (existingByKey)
|
|
2843
|
+
return;
|
|
2844
|
+
id = genId();
|
|
2845
|
+
}
|
|
2846
|
+
throw new Error(`goal plan node was not inserted after id retries: ${goal.planId}/${node.key}`);
|
|
2847
|
+
}
|
|
2247
2848
|
listGoalPlanNodes(goalIdOrPlanId) {
|
|
2248
2849
|
const rows = this.db.query("SELECT * FROM goal_plan_nodes WHERE goal_id = ? OR plan_id = ? ORDER BY sequence ASC").all(goalIdOrPlanId, goalIdOrPlanId);
|
|
2249
2850
|
return rows.map(rowToGoalPlanNode);
|
|
@@ -2335,8 +2936,8 @@ class Store {
|
|
|
2335
2936
|
$status: input.status,
|
|
2336
2937
|
$nodeKey: input.nodeKey ?? null,
|
|
2337
2938
|
$tokensUsed: input.tokensUsed ?? 0,
|
|
2338
|
-
$evidence: input.evidence ? JSON.stringify(input.evidence) : null,
|
|
2339
|
-
$rawResponse: input.rawResponse === undefined ? null : JSON.stringify(input.rawResponse),
|
|
2939
|
+
$evidence: input.evidence ? scrubSecrets(JSON.stringify(scrubSecretsDeep(input.evidence))) : null,
|
|
2940
|
+
$rawResponse: input.rawResponse === undefined ? null : scrubSecrets(JSON.stringify(scrubSecretsDeep(input.rawResponse))),
|
|
2340
2941
|
$created: now,
|
|
2341
2942
|
$updated: now
|
|
2342
2943
|
});
|
|
@@ -2374,7 +2975,6 @@ class Store {
|
|
|
2374
2975
|
const targetInput = input.loop?.target.type === "workflow" ? input.loop.target.input : undefined;
|
|
2375
2976
|
const invocationId = input.invocationId ?? targetInput?.workflowInvocationId ?? targetInput?.invocationId;
|
|
2376
2977
|
const workItemId = input.workItemId ?? targetInput?.workflowWorkItemId ?? targetInput?.workItemId;
|
|
2377
|
-
let manifestPath;
|
|
2378
2978
|
if (input.idempotencyKey) {
|
|
2379
2979
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2380
2980
|
if (existing) {
|
|
@@ -2382,6 +2982,28 @@ class Store {
|
|
|
2382
2982
|
return rowToWorkflowRun(existing);
|
|
2383
2983
|
}
|
|
2384
2984
|
}
|
|
2985
|
+
const runId = genId();
|
|
2986
|
+
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2987
|
+
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2988
|
+
const staged = stageWorkflowRunManifest({
|
|
2989
|
+
loopsDataDir: this.rootDir,
|
|
2990
|
+
workflowRunId: runId,
|
|
2991
|
+
workflowId: input.workflow.id,
|
|
2992
|
+
workflowName: input.workflow.name,
|
|
2993
|
+
invocationId,
|
|
2994
|
+
workItemId,
|
|
2995
|
+
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2996
|
+
subjectKind: invocation?.subjectRef.kind ?? (input.loop ? "loop" : "workflow"),
|
|
2997
|
+
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url ?? input.loop?.name ?? input.workflow.name,
|
|
2998
|
+
payload: {
|
|
2999
|
+
workflowInvocation: invocation,
|
|
3000
|
+
workflowWorkItem: workItem,
|
|
3001
|
+
loopId: input.loop?.id,
|
|
3002
|
+
loopRunId: input.loopRun?.id,
|
|
3003
|
+
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
3004
|
+
}
|
|
3005
|
+
});
|
|
3006
|
+
const manifestPath = staged.manifestPath;
|
|
2385
3007
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2386
3008
|
try {
|
|
2387
3009
|
this.assertDaemonLeaseFence(input, now);
|
|
@@ -2389,30 +3011,10 @@ class Store {
|
|
|
2389
3011
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2390
3012
|
if (existing) {
|
|
2391
3013
|
this.db.exec("COMMIT");
|
|
3014
|
+
discardWorkflowRunManifest(staged);
|
|
2392
3015
|
return rowToWorkflowRun(existing);
|
|
2393
3016
|
}
|
|
2394
3017
|
}
|
|
2395
|
-
const runId = genId();
|
|
2396
|
-
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2397
|
-
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2398
|
-
manifestPath = invocation || workItem ? writeWorkflowRunManifest({
|
|
2399
|
-
loopsDataDir: this.rootDir,
|
|
2400
|
-
workflowRunId: runId,
|
|
2401
|
-
workflowId: input.workflow.id,
|
|
2402
|
-
workflowName: input.workflow.name,
|
|
2403
|
-
invocationId,
|
|
2404
|
-
workItemId,
|
|
2405
|
-
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2406
|
-
subjectKind: invocation?.subjectRef.kind,
|
|
2407
|
-
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url,
|
|
2408
|
-
payload: {
|
|
2409
|
-
workflowInvocation: invocation,
|
|
2410
|
-
workflowWorkItem: workItem,
|
|
2411
|
-
loopId: input.loop?.id,
|
|
2412
|
-
loopRunId: input.loopRun?.id,
|
|
2413
|
-
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
2414
|
-
}
|
|
2415
|
-
}) : undefined;
|
|
2416
3018
|
this.db.query(`INSERT INTO workflow_runs (id, workflow_id, workflow_name, loop_id, loop_run_id, invocation_id, work_item_id,
|
|
2417
3019
|
scheduled_for, idempotency_key, manifest_path, status, started_at, finished_at, duration_ms, error,
|
|
2418
3020
|
created_at, updated_at)
|
|
@@ -2479,6 +3081,7 @@ class Store {
|
|
|
2479
3081
|
$created: now
|
|
2480
3082
|
});
|
|
2481
3083
|
this.db.exec("COMMIT");
|
|
3084
|
+
commitWorkflowRunManifest(staged);
|
|
2482
3085
|
const run = this.getWorkflowRun(runId);
|
|
2483
3086
|
if (!run)
|
|
2484
3087
|
throw new Error(`workflow run not found after create: ${runId}`);
|
|
@@ -2487,8 +3090,7 @@ class Store {
|
|
|
2487
3090
|
try {
|
|
2488
3091
|
this.db.exec("ROLLBACK");
|
|
2489
3092
|
} catch {}
|
|
2490
|
-
|
|
2491
|
-
rmSync(manifestPath, { force: true });
|
|
3093
|
+
discardWorkflowRunManifest(staged);
|
|
2492
3094
|
throw error;
|
|
2493
3095
|
}
|
|
2494
3096
|
}
|
|
@@ -2586,29 +3188,32 @@ class Store {
|
|
|
2586
3188
|
return run;
|
|
2587
3189
|
}
|
|
2588
3190
|
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
3191
|
+
return this.transact(() => {
|
|
3192
|
+
const now = nowIso();
|
|
3193
|
+
const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
|
|
3194
|
+
const live = before.filter((step) => step.pid !== undefined && isLiveStepProcess(step.pid, step.startedAt));
|
|
3195
|
+
if (live.length > 0) {
|
|
3196
|
+
throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
|
|
3197
|
+
}
|
|
3198
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
3199
|
+
SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
|
|
3200
|
+
stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
|
|
3201
|
+
WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
|
|
3202
|
+
if (before.length > 0) {
|
|
3203
|
+
this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
|
|
3204
|
+
reason,
|
|
3205
|
+
recoveredSteps: before.map((step) => step.stepId)
|
|
3206
|
+
});
|
|
3207
|
+
}
|
|
3208
|
+
return {
|
|
3209
|
+
run: this.requireWorkflowRun(workflowRunId),
|
|
3210
|
+
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
3211
|
+
};
|
|
3212
|
+
});
|
|
2609
3213
|
}
|
|
2610
3214
|
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
2611
3215
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3216
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2612
3217
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2613
3218
|
try {
|
|
2614
3219
|
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
@@ -2623,9 +3228,9 @@ class Store {
|
|
|
2623
3228
|
$finished: finishedAt,
|
|
2624
3229
|
$exitCode: patch.exitCode ?? null,
|
|
2625
3230
|
$durationMs: patch.durationMs ?? null,
|
|
2626
|
-
$stdout: patch.stdout
|
|
2627
|
-
$stderr: patch.stderr
|
|
2628
|
-
$error:
|
|
3231
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3232
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3233
|
+
$error: error ?? null,
|
|
2629
3234
|
$updated: finishedAt,
|
|
2630
3235
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2631
3236
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
@@ -2633,15 +3238,15 @@ class Store {
|
|
|
2633
3238
|
if (res.changes === 1) {
|
|
2634
3239
|
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
2635
3240
|
exitCode: patch.exitCode,
|
|
2636
|
-
error
|
|
3241
|
+
error
|
|
2637
3242
|
});
|
|
2638
3243
|
}
|
|
2639
3244
|
this.db.exec("COMMIT");
|
|
2640
|
-
} catch (
|
|
3245
|
+
} catch (error2) {
|
|
2641
3246
|
try {
|
|
2642
3247
|
this.db.exec("ROLLBACK");
|
|
2643
3248
|
} catch {}
|
|
2644
|
-
throw
|
|
3249
|
+
throw error2;
|
|
2645
3250
|
}
|
|
2646
3251
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
2647
3252
|
if (!run)
|
|
@@ -2681,6 +3286,7 @@ class Store {
|
|
|
2681
3286
|
}
|
|
2682
3287
|
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
2683
3288
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3289
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2684
3290
|
let changed = false;
|
|
2685
3291
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2686
3292
|
try {
|
|
@@ -2693,25 +3299,25 @@ class Store {
|
|
|
2693
3299
|
$status: status,
|
|
2694
3300
|
$finished: finishedAt,
|
|
2695
3301
|
$durationMs: patch.durationMs ?? null,
|
|
2696
|
-
$error:
|
|
3302
|
+
$error: error ?? null,
|
|
2697
3303
|
$updated: finishedAt,
|
|
2698
3304
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2699
3305
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
2700
3306
|
});
|
|
2701
3307
|
changed = res.changes === 1;
|
|
2702
3308
|
if (changed)
|
|
2703
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error
|
|
3309
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error });
|
|
2704
3310
|
if (changed) {
|
|
2705
3311
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
2706
|
-
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus,
|
|
3312
|
+
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
2707
3313
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
2708
3314
|
}
|
|
2709
3315
|
this.db.exec("COMMIT");
|
|
2710
|
-
} catch (
|
|
3316
|
+
} catch (error2) {
|
|
2711
3317
|
try {
|
|
2712
3318
|
this.db.exec("ROLLBACK");
|
|
2713
3319
|
} catch {}
|
|
2714
|
-
throw
|
|
3320
|
+
throw error2;
|
|
2715
3321
|
}
|
|
2716
3322
|
const run = this.getWorkflowRun(workflowRunId);
|
|
2717
3323
|
if (!run)
|
|
@@ -2744,24 +3350,26 @@ class Store {
|
|
|
2744
3350
|
}
|
|
2745
3351
|
}
|
|
2746
3352
|
appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
3353
|
+
return this.transact(() => {
|
|
3354
|
+
const now = nowIso();
|
|
3355
|
+
const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
|
|
3356
|
+
const sequence = (current?.sequence ?? 0) + 1;
|
|
3357
|
+
const id = genId();
|
|
3358
|
+
this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
|
|
3359
|
+
VALUES ($id, $workflowRunId, $sequence, $eventType, $stepId, $payload, $created)`).run({
|
|
3360
|
+
$id: id,
|
|
3361
|
+
$workflowRunId: workflowRunId,
|
|
3362
|
+
$sequence: sequence,
|
|
3363
|
+
$eventType: eventType,
|
|
3364
|
+
$stepId: stepId ?? null,
|
|
3365
|
+
$payload: payload ? JSON.stringify(payload) : null,
|
|
3366
|
+
$created: now
|
|
3367
|
+
});
|
|
3368
|
+
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
3369
|
+
if (!event)
|
|
3370
|
+
throw new Error(`workflow event not found after append: ${id}`);
|
|
3371
|
+
return rowToWorkflowEvent(event);
|
|
2760
3372
|
});
|
|
2761
|
-
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
2762
|
-
if (!event)
|
|
2763
|
-
throw new Error(`workflow event not found after append: ${id}`);
|
|
2764
|
-
return rowToWorkflowEvent(event);
|
|
2765
3373
|
}
|
|
2766
3374
|
listWorkflowEvents(workflowRunId, limit = 200) {
|
|
2767
3375
|
const rows = this.db.query("SELECT * FROM workflow_events WHERE workflow_run_id = ? ORDER BY sequence ASC LIMIT ?").all(workflowRunId, limit);
|
|
@@ -2777,35 +3385,64 @@ class Store {
|
|
|
2777
3385
|
}
|
|
2778
3386
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
2779
3387
|
const now = (opts.now ?? new Date).toISOString();
|
|
2780
|
-
const
|
|
3388
|
+
const startedMs = processStartTimeMs(pid);
|
|
3389
|
+
const processStartedAt = startedMs === undefined ? null : new Date(startedMs).toISOString();
|
|
3390
|
+
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2781
3391
|
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
2782
3392
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2783
3393
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2784
3394
|
))`).run({
|
|
2785
3395
|
$id: id,
|
|
2786
3396
|
$pid: pid,
|
|
3397
|
+
$processStartedAt: processStartedAt,
|
|
2787
3398
|
$updated: now,
|
|
2788
3399
|
$claimedBy: claimedBy,
|
|
2789
3400
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2790
3401
|
$now: now
|
|
2791
|
-
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
3402
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2792
3403
|
WHERE id=$id AND status='running'
|
|
2793
3404
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2794
3405
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2795
|
-
))`).run({
|
|
3406
|
+
))`).run({
|
|
3407
|
+
$id: id,
|
|
3408
|
+
$pid: pid,
|
|
3409
|
+
$processStartedAt: processStartedAt,
|
|
3410
|
+
$updated: now,
|
|
3411
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3412
|
+
$now: now
|
|
3413
|
+
});
|
|
2796
3414
|
if (res.changes !== 1)
|
|
2797
3415
|
return;
|
|
2798
3416
|
return this.getRun(id);
|
|
2799
3417
|
}
|
|
3418
|
+
recordRunProcess(runId, info, opts = {}) {
|
|
3419
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
3420
|
+
const res = this.db.query(`UPDATE loop_runs SET pid=$pid, pgid=$pgid, process_started_at=$processStartedAt, updated_at=$updated
|
|
3421
|
+
WHERE id=$id AND status='running'
|
|
3422
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3423
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3424
|
+
))`).run({
|
|
3425
|
+
$id: runId,
|
|
3426
|
+
$pid: info.pid,
|
|
3427
|
+
$pgid: info.pgid ?? null,
|
|
3428
|
+
$processStartedAt: info.processStartedAt ?? isoProcessStart(info.pid) ?? now,
|
|
3429
|
+
$updated: now,
|
|
3430
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3431
|
+
$now: now
|
|
3432
|
+
});
|
|
3433
|
+
if (res.changes !== 1)
|
|
3434
|
+
return;
|
|
3435
|
+
return this.getRun(runId);
|
|
3436
|
+
}
|
|
2800
3437
|
hasLiveWorkflowStepProcesses(loopRunId) {
|
|
2801
|
-
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
|
|
3438
|
+
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid, wsr.started_at AS started_at
|
|
2802
3439
|
FROM workflow_runs wr
|
|
2803
3440
|
JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
|
|
2804
3441
|
WHERE wr.loop_run_id = ?
|
|
2805
3442
|
AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
2806
3443
|
AND wsr.status = 'running'
|
|
2807
3444
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
2808
|
-
return liveWorkflowSteps.some((step) =>
|
|
3445
|
+
return liveWorkflowSteps.some((step) => isLiveStepProcess(step.pid, step.started_at));
|
|
2809
3446
|
}
|
|
2810
3447
|
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
2811
3448
|
const now = nowIso();
|
|
@@ -2879,7 +3516,7 @@ class Store {
|
|
|
2879
3516
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
2880
3517
|
if (existing) {
|
|
2881
3518
|
if (existing.status === "running") {
|
|
2882
|
-
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid
|
|
3519
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
2883
3520
|
this.db.exec("COMMIT");
|
|
2884
3521
|
return;
|
|
2885
3522
|
}
|
|
@@ -2888,7 +3525,7 @@ class Store {
|
|
|
2888
3525
|
return;
|
|
2889
3526
|
}
|
|
2890
3527
|
const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
|
|
2891
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3528
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2892
3529
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2893
3530
|
WHERE id=$id AND status='running' AND lease_expires_at <= $now`).run({
|
|
2894
3531
|
$id: existing.id,
|
|
@@ -2910,7 +3547,7 @@ class Store {
|
|
|
2910
3547
|
}
|
|
2911
3548
|
const attempt = existing.attempt + 1;
|
|
2912
3549
|
const res2 = this.db.query(`UPDATE loop_runs SET attempt=$attempt, status='running', started_at=$started, finished_at=NULL,
|
|
2913
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3550
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2914
3551
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2915
3552
|
WHERE id=$id
|
|
2916
3553
|
AND status IN ('failed', 'timed_out', 'abandoned')
|
|
@@ -2958,6 +3595,7 @@ class Store {
|
|
|
2958
3595
|
}
|
|
2959
3596
|
finalizeRun(id, patch, opts = {}) {
|
|
2960
3597
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3598
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2961
3599
|
const params = {
|
|
2962
3600
|
$id: id,
|
|
2963
3601
|
$status: patch.status,
|
|
@@ -2965,41 +3603,43 @@ class Store {
|
|
|
2965
3603
|
$pid: patch.pid ?? null,
|
|
2966
3604
|
$exitCode: patch.exitCode ?? null,
|
|
2967
3605
|
$durationMs: patch.durationMs ?? null,
|
|
2968
|
-
$stdout: patch.stdout
|
|
2969
|
-
$stderr: patch.stderr
|
|
2970
|
-
$error:
|
|
3606
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3607
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3608
|
+
$error: error ?? null,
|
|
2971
3609
|
$updated: finishedAt,
|
|
2972
3610
|
$claimedBy: opts.claimedBy ?? null,
|
|
2973
3611
|
$now: (opts.now ?? new Date).toISOString(),
|
|
2974
3612
|
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
2975
3613
|
};
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
AND
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3614
|
+
return this.transact(() => {
|
|
3615
|
+
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3616
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3617
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
3618
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3619
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3620
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3621
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
3622
|
+
const run = this.getRun(id);
|
|
3623
|
+
if (!run)
|
|
3624
|
+
throw new Error(`run not found after finalize: ${id}`);
|
|
3625
|
+
if (opts.claimedBy && res.changes !== 1)
|
|
3626
|
+
return run;
|
|
3627
|
+
if (res.changes === 1) {
|
|
3628
|
+
this.setWorkflowWorkItemsForLoopRun(run, error, finishedAt);
|
|
3629
|
+
const loop = this.getLoop(run.loopId);
|
|
3630
|
+
const itemStatus = workItemStatusForLoopRun(run.status, run.attempt, loop?.maxAttempts);
|
|
3631
|
+
if (loop?.target.type === "workflow" && itemStatus && itemStatus !== "admitted") {
|
|
3632
|
+
const workItemId = loop.target.input?.workflowWorkItemId ?? loop.target.input?.workItemId;
|
|
3633
|
+
this.maybeArchiveGeneratedRouteWorkflow({
|
|
3634
|
+
workflowId: loop.target.workflowId,
|
|
3635
|
+
loopId: loop.id,
|
|
3636
|
+
workItemId,
|
|
3637
|
+
updated: finishedAt
|
|
3638
|
+
});
|
|
3639
|
+
}
|
|
3000
3640
|
}
|
|
3001
|
-
|
|
3002
|
-
|
|
3641
|
+
return run;
|
|
3642
|
+
});
|
|
3003
3643
|
}
|
|
3004
3644
|
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
3005
3645
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
@@ -3049,6 +3689,9 @@ class Store {
|
|
|
3049
3689
|
});
|
|
3050
3690
|
}
|
|
3051
3691
|
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
3692
|
+
return this.recoverExpiredRunLeasesDetailed(now, opts).abandoned;
|
|
3693
|
+
}
|
|
3694
|
+
recoverExpiredRunLeasesDetailed(now = new Date, opts = {}) {
|
|
3052
3695
|
const limit = Math.max(1, Math.min(1000, Math.floor(opts.limit ?? DEFAULT_RECOVERY_BATCH_LIMIT)));
|
|
3053
3696
|
const scanLimit = Math.max(limit, Math.min(5000, Math.floor(opts.scanLimit ?? limit * DEFAULT_RECOVERY_SCAN_MULTIPLIER)));
|
|
3054
3697
|
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
@@ -3056,15 +3699,15 @@ class Store {
|
|
|
3056
3699
|
ORDER BY lease_expires_at ASC
|
|
3057
3700
|
LIMIT ?`).all(now.toISOString(), scanLimit);
|
|
3058
3701
|
const recovered = [];
|
|
3702
|
+
const deferred = [];
|
|
3059
3703
|
for (const row of rows) {
|
|
3060
3704
|
if (recovered.length >= limit)
|
|
3061
3705
|
break;
|
|
3062
|
-
if (row.pid
|
|
3063
|
-
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3064
|
-
continue;
|
|
3065
|
-
}
|
|
3066
|
-
if (this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3706
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at) || this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3067
3707
|
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3708
|
+
const deferredRun = this.getRun(row.id);
|
|
3709
|
+
if (deferredRun)
|
|
3710
|
+
deferred.push(deferredRun);
|
|
3068
3711
|
continue;
|
|
3069
3712
|
}
|
|
3070
3713
|
const finished = now.toISOString();
|
|
@@ -3148,7 +3791,7 @@ class Store {
|
|
|
3148
3791
|
if (run)
|
|
3149
3792
|
recovered.push(run);
|
|
3150
3793
|
}
|
|
3151
|
-
return recovered;
|
|
3794
|
+
return { abandoned: recovered, deferred };
|
|
3152
3795
|
}
|
|
3153
3796
|
expireLoops(now = new Date, opts = {}) {
|
|
3154
3797
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND archived_at IS NULL AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
@@ -3181,6 +3824,83 @@ class Store {
|
|
|
3181
3824
|
const row = status ? this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE status = ?").get(status) : this.db.query("SELECT COUNT(*) AS count FROM loop_runs").get();
|
|
3182
3825
|
return row?.count ?? 0;
|
|
3183
3826
|
}
|
|
3827
|
+
pruneHistory(opts) {
|
|
3828
|
+
const { maxAgeDays, keepPerLoop } = opts;
|
|
3829
|
+
if (maxAgeDays === undefined && keepPerLoop === undefined) {
|
|
3830
|
+
throw new ValidationError("pruneHistory requires maxAgeDays and/or keepPerLoop");
|
|
3831
|
+
}
|
|
3832
|
+
if (maxAgeDays !== undefined && (!Number.isFinite(maxAgeDays) || maxAgeDays < 0)) {
|
|
3833
|
+
throw new ValidationError(`pruneHistory maxAgeDays must be a non-negative number: ${maxAgeDays}`);
|
|
3834
|
+
}
|
|
3835
|
+
if (keepPerLoop !== undefined && (!Number.isInteger(keepPerLoop) || keepPerLoop < 0)) {
|
|
3836
|
+
throw new ValidationError(`pruneHistory keepPerLoop must be a non-negative integer: ${keepPerLoop}`);
|
|
3837
|
+
}
|
|
3838
|
+
const now = opts.now ?? new Date;
|
|
3839
|
+
const dryRun = opts.dryRun ?? false;
|
|
3840
|
+
const cutoff = maxAgeDays === undefined ? undefined : new Date(now.getTime() - maxAgeDays * 86400000).toISOString();
|
|
3841
|
+
const terminal = TERMINAL_RUN_STATUSES.map((status) => `'${status}'`).join(",");
|
|
3842
|
+
const candidateIds = this.db.query(`WITH ranked AS (
|
|
3843
|
+
SELECT id, status, created_at,
|
|
3844
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS recency
|
|
3845
|
+
FROM loop_runs
|
|
3846
|
+
)
|
|
3847
|
+
SELECT id FROM ranked
|
|
3848
|
+
WHERE status IN (${terminal})
|
|
3849
|
+
AND ($cutoff IS NULL OR created_at < $cutoff)
|
|
3850
|
+
AND ($keep IS NULL OR recency > $keep)`).all({ $cutoff: cutoff ?? null, $keep: keepPerLoop ?? null }).map((row) => row.id);
|
|
3851
|
+
const summary = {
|
|
3852
|
+
dryRun,
|
|
3853
|
+
cutoff,
|
|
3854
|
+
keepPerLoop,
|
|
3855
|
+
loopRuns: dryRun ? candidateIds.length : 0,
|
|
3856
|
+
workflowRuns: 0,
|
|
3857
|
+
goalRuns: 0
|
|
3858
|
+
};
|
|
3859
|
+
const manifestPaths = [];
|
|
3860
|
+
for (let offset = 0;offset < candidateIds.length; offset += PRUNE_BATCH_SIZE) {
|
|
3861
|
+
const batch = candidateIds.slice(offset, offset + PRUNE_BATCH_SIZE);
|
|
3862
|
+
const batchPlaceholders = batch.map(() => "?").join(",");
|
|
3863
|
+
if (dryRun) {
|
|
3864
|
+
const workflowRunIds = this.db.query(`SELECT id FROM workflow_runs WHERE loop_run_id IN (${batchPlaceholders})`).all(...batch).map((row) => row.id);
|
|
3865
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3866
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3867
|
+
summary.goalRuns += this.db.query(`SELECT COUNT(*) AS count FROM goal_runs
|
|
3868
|
+
WHERE loop_run_id IN (${batchPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).get(...batch, ...workflowRunIds)?.count ?? 0;
|
|
3869
|
+
continue;
|
|
3870
|
+
}
|
|
3871
|
+
this.transact(() => {
|
|
3872
|
+
const confirmed = this.db.query(`SELECT id FROM loop_runs WHERE id IN (${batchPlaceholders}) AND status IN (${terminal})`).all(...batch).map((row) => row.id);
|
|
3873
|
+
if (confirmed.length === 0)
|
|
3874
|
+
return;
|
|
3875
|
+
const runPlaceholders = confirmed.map(() => "?").join(",");
|
|
3876
|
+
const workflowRuns = this.db.query(`SELECT id, manifest_path FROM workflow_runs WHERE loop_run_id IN (${runPlaceholders})`).all(...confirmed);
|
|
3877
|
+
const workflowRunIds = workflowRuns.map((row) => row.id);
|
|
3878
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3879
|
+
summary.loopRuns += confirmed.length;
|
|
3880
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3881
|
+
summary.goalRuns += this.db.query(`DELETE FROM goal_runs WHERE loop_run_id IN (${runPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).run(...confirmed, ...workflowRunIds).changes;
|
|
3882
|
+
if (workflowRunIds.length > 0) {
|
|
3883
|
+
this.db.query(`DELETE FROM workflow_runs WHERE id IN (${workflowPlaceholders})`).run(...workflowRunIds);
|
|
3884
|
+
}
|
|
3885
|
+
this.db.query(`DELETE FROM loop_runs WHERE id IN (${runPlaceholders}) AND status IN (${terminal})`).run(...confirmed);
|
|
3886
|
+
for (const row of workflowRuns) {
|
|
3887
|
+
if (row.manifest_path)
|
|
3888
|
+
manifestPaths.push(row.manifest_path);
|
|
3889
|
+
}
|
|
3890
|
+
});
|
|
3891
|
+
}
|
|
3892
|
+
for (const manifestPath of manifestPaths) {
|
|
3893
|
+
const runDir = dirname2(manifestPath);
|
|
3894
|
+
try {
|
|
3895
|
+
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
3896
|
+
rmSync2(runDir, { recursive: true, force: true });
|
|
3897
|
+
} else {
|
|
3898
|
+
rmSync2(manifestPath, { force: true });
|
|
3899
|
+
}
|
|
3900
|
+
} catch {}
|
|
3901
|
+
}
|
|
3902
|
+
return summary;
|
|
3903
|
+
}
|
|
3184
3904
|
acquireDaemonLease(input) {
|
|
3185
3905
|
const now = input.now ?? new Date;
|
|
3186
3906
|
const expiresAt = new Date(now.getTime() + input.ttlMs).toISOString();
|
|
@@ -3243,16 +3963,245 @@ class Store {
|
|
|
3243
3963
|
}
|
|
3244
3964
|
}
|
|
3245
3965
|
|
|
3966
|
+
// src/lib/doctor.ts
|
|
3967
|
+
import { spawnSync as spawnSync5 } from "child_process";
|
|
3968
|
+
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
3969
|
+
|
|
3970
|
+
// src/daemon/control.ts
|
|
3971
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync3, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
3972
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
3973
|
+
import { hostname } from "os";
|
|
3974
|
+
import { dirname as dirname3, join as join4 } from "path";
|
|
3975
|
+
|
|
3976
|
+
// src/daemon/loop.ts
|
|
3977
|
+
function realSleep(ms) {
|
|
3978
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
3979
|
+
}
|
|
3980
|
+
async function runLoop(opts) {
|
|
3981
|
+
const sleep = opts.sleep ?? realSleep;
|
|
3982
|
+
const sliceMs = opts.sliceMs ?? 200;
|
|
3983
|
+
while (!opts.shouldStop()) {
|
|
3984
|
+
try {
|
|
3985
|
+
await opts.tickFn();
|
|
3986
|
+
} catch (err) {
|
|
3987
|
+
opts.onTickError?.(err);
|
|
3988
|
+
}
|
|
3989
|
+
let waited = 0;
|
|
3990
|
+
while (waited < opts.intervalMs && !opts.shouldStop()) {
|
|
3991
|
+
const chunk = Math.min(sliceMs, opts.intervalMs - waited);
|
|
3992
|
+
await sleep(chunk);
|
|
3993
|
+
waited += chunk;
|
|
3994
|
+
}
|
|
3995
|
+
}
|
|
3996
|
+
}
|
|
3997
|
+
|
|
3998
|
+
// src/daemon/control.ts
|
|
3999
|
+
function readPidRecord(path = pidFilePath()) {
|
|
4000
|
+
if (!existsSync2(path))
|
|
4001
|
+
return;
|
|
4002
|
+
let raw;
|
|
4003
|
+
try {
|
|
4004
|
+
raw = readFileSync3(path, "utf8").trim();
|
|
4005
|
+
} catch {
|
|
4006
|
+
return;
|
|
4007
|
+
}
|
|
4008
|
+
if (!raw)
|
|
4009
|
+
return;
|
|
4010
|
+
if (raw.startsWith("{")) {
|
|
4011
|
+
try {
|
|
4012
|
+
const parsed = JSON.parse(raw);
|
|
4013
|
+
const pid2 = Number(parsed.pid);
|
|
4014
|
+
if (!Number.isInteger(pid2) || pid2 <= 0)
|
|
4015
|
+
return;
|
|
4016
|
+
const startedAt = Number(parsed.startedAt);
|
|
4017
|
+
return { pid: pid2, startedAt: Number.isFinite(startedAt) ? startedAt : undefined };
|
|
4018
|
+
} catch {
|
|
4019
|
+
return;
|
|
4020
|
+
}
|
|
4021
|
+
}
|
|
4022
|
+
const pid = Number(raw);
|
|
4023
|
+
return Number.isInteger(pid) && pid > 0 ? { pid } : undefined;
|
|
4024
|
+
}
|
|
4025
|
+
function writePid(pid = process.pid, path = pidFilePath()) {
|
|
4026
|
+
mkdirSync4(dirname3(path), { recursive: true, mode: 448 });
|
|
4027
|
+
writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
4028
|
+
}
|
|
4029
|
+
function removePid(path = pidFilePath()) {
|
|
4030
|
+
rmSync3(path, { force: true });
|
|
4031
|
+
}
|
|
4032
|
+
function isAlive(pid, startedAt) {
|
|
4033
|
+
try {
|
|
4034
|
+
process.kill(pid, 0);
|
|
4035
|
+
} catch (err) {
|
|
4036
|
+
if (err.code !== "EPERM")
|
|
4037
|
+
return false;
|
|
4038
|
+
}
|
|
4039
|
+
return sameProcessStart(startedAt, processStartTimeMs(pid));
|
|
4040
|
+
}
|
|
4041
|
+
function isDaemonRunning(path = pidFilePath()) {
|
|
4042
|
+
const record = readPidRecord(path);
|
|
4043
|
+
if (!record)
|
|
4044
|
+
return { running: false, stale: false };
|
|
4045
|
+
if (isAlive(record.pid, record.startedAt))
|
|
4046
|
+
return { running: true, stale: false, pid: record.pid };
|
|
4047
|
+
return { running: false, stale: true, pid: record.pid };
|
|
4048
|
+
}
|
|
4049
|
+
function toReapableProcess(run) {
|
|
4050
|
+
return { pid: run.pid, pgid: run.pgid, processStartedAt: run.processStartedAt };
|
|
4051
|
+
}
|
|
4052
|
+
function ownProcessGroupId() {
|
|
4053
|
+
if (process.platform === "linux") {
|
|
4054
|
+
const fields = procStatFields("/proc/self/stat");
|
|
4055
|
+
const pgrp = fields ? Number(fields[2]) : Number.NaN;
|
|
4056
|
+
if (Number.isInteger(pgrp) && pgrp > 0)
|
|
4057
|
+
return pgrp;
|
|
4058
|
+
}
|
|
4059
|
+
try {
|
|
4060
|
+
const run = spawnSync2("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
4061
|
+
const pgid = Number(run.stdout.trim());
|
|
4062
|
+
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
4063
|
+
return pgid;
|
|
4064
|
+
} catch {}
|
|
4065
|
+
return;
|
|
4066
|
+
}
|
|
4067
|
+
function signalReapTarget(target, signal) {
|
|
4068
|
+
try {
|
|
4069
|
+
process.kill(target.group ? -target.id : target.id, signal);
|
|
4070
|
+
return true;
|
|
4071
|
+
} catch (err) {
|
|
4072
|
+
return err.code === "EPERM";
|
|
4073
|
+
}
|
|
4074
|
+
}
|
|
4075
|
+
function processExists(pid) {
|
|
4076
|
+
try {
|
|
4077
|
+
process.kill(pid, 0);
|
|
4078
|
+
return true;
|
|
4079
|
+
} catch (err) {
|
|
4080
|
+
return err.code === "EPERM";
|
|
4081
|
+
}
|
|
4082
|
+
}
|
|
4083
|
+
async function reapProcessGroups(entries, opts = {}) {
|
|
4084
|
+
const sleep = opts.sleep ?? realSleep;
|
|
4085
|
+
const graceMs = Math.max(100, opts.graceMs ?? 2000);
|
|
4086
|
+
const ownPgid = ownProcessGroupId();
|
|
4087
|
+
const targets = new Map;
|
|
4088
|
+
for (const entry of entries) {
|
|
4089
|
+
const leaderPid = entry.pid ?? entry.pgid;
|
|
4090
|
+
if (leaderPid !== undefined && processExists(leaderPid)) {
|
|
4091
|
+
if (entry.processStartedAt === undefined) {
|
|
4092
|
+
opts.log?.(`skipping reap of pid ${leaderPid}: no recorded start-time fingerprint to verify identity`);
|
|
4093
|
+
continue;
|
|
4094
|
+
}
|
|
4095
|
+
if (!verifiedProcessStart(entry.processStartedAt, processStartTimeMs(leaderPid))) {
|
|
4096
|
+
opts.log?.(`skipping reap of pid ${leaderPid}: start-time fingerprint mismatch or unverifiable (pid recycled)`);
|
|
4097
|
+
continue;
|
|
4098
|
+
}
|
|
4099
|
+
}
|
|
4100
|
+
const usableGroup = entry.pgid !== undefined && Number.isInteger(entry.pgid) && entry.pgid > 1 && entry.pgid !== process.pid && entry.pgid !== ownPgid;
|
|
4101
|
+
const usablePid = entry.pid !== undefined && Number.isInteger(entry.pid) && entry.pid > 1 && entry.pid !== process.pid;
|
|
4102
|
+
const target = usableGroup ? { id: entry.pgid, group: true } : usablePid ? { id: entry.pid, group: false } : undefined;
|
|
4103
|
+
if (!target)
|
|
4104
|
+
continue;
|
|
4105
|
+
targets.set(`${target.group ? "g" : "p"}:${target.id}`, target);
|
|
4106
|
+
}
|
|
4107
|
+
const live = [...targets.values()].filter((target) => signalReapTarget(target, "SIGTERM"));
|
|
4108
|
+
if (live.length === 0)
|
|
4109
|
+
return [];
|
|
4110
|
+
const steps = Math.max(1, Math.ceil(graceMs / 100));
|
|
4111
|
+
let remaining = live;
|
|
4112
|
+
for (let i = 0;i < steps && remaining.length > 0; i++) {
|
|
4113
|
+
await sleep(100);
|
|
4114
|
+
remaining = remaining.filter((target) => signalReapTarget(target, 0));
|
|
4115
|
+
}
|
|
4116
|
+
for (const target of remaining) {
|
|
4117
|
+
signalReapTarget(target, "SIGKILL");
|
|
4118
|
+
opts.log?.(`escalated to SIGKILL for ${target.group ? "pgid" : "pid"} ${target.id}`);
|
|
4119
|
+
}
|
|
4120
|
+
return live.map((target) => target.id);
|
|
4121
|
+
}
|
|
4122
|
+
function daemonStatus(store, path = pidFilePath()) {
|
|
4123
|
+
return {
|
|
4124
|
+
...isDaemonRunning(path),
|
|
4125
|
+
lease: store.getDaemonLease(),
|
|
4126
|
+
host: hostname(),
|
|
4127
|
+
loops: {
|
|
4128
|
+
total: store.countLoops(),
|
|
4129
|
+
active: store.countLoops("active"),
|
|
4130
|
+
paused: store.countLoops("paused"),
|
|
4131
|
+
stopped: store.countLoops("stopped"),
|
|
4132
|
+
expired: store.countLoops("expired"),
|
|
4133
|
+
archived: store.countLoops(undefined, { archived: true })
|
|
4134
|
+
},
|
|
4135
|
+
runs: {
|
|
4136
|
+
total: store.countRuns(),
|
|
4137
|
+
running: store.countRuns("running"),
|
|
4138
|
+
failed: store.countRuns("failed"),
|
|
4139
|
+
succeeded: store.countRuns("succeeded"),
|
|
4140
|
+
abandoned: store.countRuns("abandoned")
|
|
4141
|
+
},
|
|
4142
|
+
logPath: daemonLogPath()
|
|
4143
|
+
};
|
|
4144
|
+
}
|
|
4145
|
+
async function stopDaemon(opts = {}) {
|
|
4146
|
+
const path = opts.path ?? pidFilePath();
|
|
4147
|
+
const record = readPidRecord(path);
|
|
4148
|
+
const state = isDaemonRunning(path);
|
|
4149
|
+
if (state.stale) {
|
|
4150
|
+
removePid(path);
|
|
4151
|
+
return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
|
|
4152
|
+
}
|
|
4153
|
+
if (!state.running || !state.pid)
|
|
4154
|
+
return { wasRunning: false, stopped: false, forced: false };
|
|
4155
|
+
const sleep = opts.sleep ?? realSleep;
|
|
4156
|
+
const store = new Store(join4(dirname3(path), "loops.db"));
|
|
4157
|
+
try {
|
|
4158
|
+
const lease = store.getDaemonLease();
|
|
4159
|
+
if (!lease || lease.pid !== state.pid || new Date(lease.expiresAt).getTime() <= Date.now()) {
|
|
4160
|
+
removePid(path);
|
|
4161
|
+
return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
|
|
4162
|
+
}
|
|
4163
|
+
try {
|
|
4164
|
+
process.kill(state.pid, "SIGTERM");
|
|
4165
|
+
} catch {
|
|
4166
|
+
removePid(path);
|
|
4167
|
+
return { wasRunning: true, stopped: true, forced: false, pid: state.pid };
|
|
4168
|
+
}
|
|
4169
|
+
const steps = Math.max(1, Math.ceil((opts.timeoutMs ?? 6000) / 100));
|
|
4170
|
+
for (let i = 0;i < steps; i++) {
|
|
4171
|
+
await sleep(100);
|
|
4172
|
+
if (!isAlive(state.pid, record?.startedAt)) {
|
|
4173
|
+
removePid(path);
|
|
4174
|
+
return { wasRunning: true, stopped: true, forced: false, pid: state.pid };
|
|
4175
|
+
}
|
|
4176
|
+
}
|
|
4177
|
+
try {
|
|
4178
|
+
process.kill(state.pid, "SIGKILL");
|
|
4179
|
+
} catch {}
|
|
4180
|
+
await sleep(150);
|
|
4181
|
+
removePid(path);
|
|
4182
|
+
const ownedRuns = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.claimedBy !== undefined && run.claimedBy.endsWith(`:${lease.id}`));
|
|
4183
|
+
const reapedPgids = await reapProcessGroups(ownedRuns.map(toReapableProcess), {
|
|
4184
|
+
sleep,
|
|
4185
|
+
graceMs: opts.reapGraceMs
|
|
4186
|
+
});
|
|
4187
|
+
return { wasRunning: true, stopped: !isAlive(state.pid, record?.startedAt), forced: true, pid: state.pid, reapedPgids };
|
|
4188
|
+
} finally {
|
|
4189
|
+
store.close();
|
|
4190
|
+
}
|
|
4191
|
+
}
|
|
4192
|
+
|
|
3246
4193
|
// src/lib/executor.ts
|
|
3247
|
-
import { spawn, spawnSync as
|
|
4194
|
+
import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
|
|
3248
4195
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
3249
4196
|
import { once } from "events";
|
|
3250
|
-
import {
|
|
4197
|
+
import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
|
|
4198
|
+
import { dirname as dirname4, resolve as resolve2 } from "path";
|
|
3251
4199
|
|
|
3252
4200
|
// src/lib/accounts.ts
|
|
3253
|
-
import { spawnSync } from "child_process";
|
|
3254
|
-
import { existsSync as
|
|
4201
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
4202
|
+
import { existsSync as existsSync3 } from "fs";
|
|
3255
4203
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4204
|
+
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
3256
4205
|
function accountToolForProvider(provider) {
|
|
3257
4206
|
switch (provider) {
|
|
3258
4207
|
case "claude":
|
|
@@ -3314,21 +4263,17 @@ function accountDirEnvVar(tool) {
|
|
|
3314
4263
|
return;
|
|
3315
4264
|
}
|
|
3316
4265
|
}
|
|
3317
|
-
function
|
|
3318
|
-
if (!account)
|
|
3319
|
-
return {};
|
|
4266
|
+
function requiredAccountTool(account, toolHint) {
|
|
3320
4267
|
const tool = account.tool ?? toolHint;
|
|
3321
4268
|
if (!tool)
|
|
3322
4269
|
throw new Error("account.tool is required when no provider tool can be inferred");
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
3327
|
-
});
|
|
4270
|
+
return tool;
|
|
4271
|
+
}
|
|
4272
|
+
function accountEnvFromResult(account, tool, result) {
|
|
3328
4273
|
if (result.error) {
|
|
3329
|
-
throw new Error(`failed to run accounts env for ${account.profile}/${tool}: ${result.error
|
|
4274
|
+
throw new Error(`failed to run accounts env for ${account.profile}/${tool}: ${result.error}`);
|
|
3330
4275
|
}
|
|
3331
|
-
if ((result.status ??
|
|
4276
|
+
if ((result.status ?? 1) !== 0) {
|
|
3332
4277
|
const stderr = result.stderr.trim();
|
|
3333
4278
|
throw new Error(`accounts env failed for ${account.profile}/${tool}${stderr ? `: ${stderr}` : ""}`);
|
|
3334
4279
|
}
|
|
@@ -3336,7 +4281,7 @@ function resolveAccountEnv(account, toolHint, env) {
|
|
|
3336
4281
|
const profileDir = (accountDirEnvVar(tool) ? accountEnv[accountDirEnvVar(tool)] : undefined) ?? primaryAccountDir(result.stdout);
|
|
3337
4282
|
if (!profileDir)
|
|
3338
4283
|
throw new Error(`accounts env returned no profile directory for ${account.profile}/${tool}`);
|
|
3339
|
-
if (!
|
|
4284
|
+
if (!existsSync3(profileDir))
|
|
3340
4285
|
throw new Error(`account profile directory does not exist for ${account.profile}/${tool}: ${profileDir}`);
|
|
3341
4286
|
return {
|
|
3342
4287
|
...accountEnv,
|
|
@@ -3344,11 +4289,38 @@ function resolveAccountEnv(account, toolHint, env) {
|
|
|
3344
4289
|
LOOPS_ACCOUNT_TOOL: tool
|
|
3345
4290
|
};
|
|
3346
4291
|
}
|
|
4292
|
+
async function resolveAccountEnv(account, toolHint, env) {
|
|
4293
|
+
if (!account)
|
|
4294
|
+
return {};
|
|
4295
|
+
const tool = requiredAccountTool(account, toolHint);
|
|
4296
|
+
const result = await spawnCapture("accounts", ["env", account.profile, "--tool", tool], {
|
|
4297
|
+
env,
|
|
4298
|
+
timeoutMs: ACCOUNTS_ENV_TIMEOUT_MS
|
|
4299
|
+
});
|
|
4300
|
+
return accountEnvFromResult(account, tool, result);
|
|
4301
|
+
}
|
|
4302
|
+
function resolveAccountEnvSync(account, toolHint, env) {
|
|
4303
|
+
if (!account)
|
|
4304
|
+
return {};
|
|
4305
|
+
const tool = requiredAccountTool(account, toolHint);
|
|
4306
|
+
const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
|
|
4307
|
+
encoding: "utf8",
|
|
4308
|
+
env,
|
|
4309
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
4310
|
+
timeout: ACCOUNTS_ENV_TIMEOUT_MS
|
|
4311
|
+
});
|
|
4312
|
+
return accountEnvFromResult(account, tool, {
|
|
4313
|
+
status: result.status,
|
|
4314
|
+
stdout: result.stdout ?? "",
|
|
4315
|
+
stderr: result.stderr ?? "",
|
|
4316
|
+
error: result.error?.message
|
|
4317
|
+
});
|
|
4318
|
+
}
|
|
3347
4319
|
|
|
3348
4320
|
// src/lib/env.ts
|
|
3349
4321
|
import { accessSync, constants } from "fs";
|
|
3350
4322
|
import { homedir as homedir2 } from "os";
|
|
3351
|
-
import { delimiter, join as
|
|
4323
|
+
import { delimiter, join as join5 } from "path";
|
|
3352
4324
|
function compactPathParts(parts) {
|
|
3353
4325
|
const seen = new Set;
|
|
3354
4326
|
const result = [];
|
|
@@ -3364,14 +4336,14 @@ function compactPathParts(parts) {
|
|
|
3364
4336
|
function commonExecutableDirs(env = process.env) {
|
|
3365
4337
|
const home = env.HOME || homedir2();
|
|
3366
4338
|
return compactPathParts([
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
env.BUN_INSTALL ?
|
|
4339
|
+
join5(home, ".local", "bin"),
|
|
4340
|
+
join5(home, ".bun", "bin"),
|
|
4341
|
+
join5(home, ".cargo", "bin"),
|
|
4342
|
+
join5(home, ".npm-global", "bin"),
|
|
4343
|
+
join5(home, "bin"),
|
|
4344
|
+
env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
|
|
3373
4345
|
env.PNPM_HOME,
|
|
3374
|
-
env.NPM_CONFIG_PREFIX ?
|
|
4346
|
+
env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
3375
4347
|
"/opt/homebrew/bin",
|
|
3376
4348
|
"/usr/local/bin",
|
|
3377
4349
|
"/usr/bin",
|
|
@@ -3395,7 +4367,7 @@ function executableExists(command, env = process.env) {
|
|
|
3395
4367
|
if (command.includes("/"))
|
|
3396
4368
|
return isExecutable(command);
|
|
3397
4369
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
3398
|
-
if (dir && isExecutable(
|
|
4370
|
+
if (dir && isExecutable(join5(dir, command)))
|
|
3399
4371
|
return true;
|
|
3400
4372
|
}
|
|
3401
4373
|
return false;
|
|
@@ -3405,10 +4377,20 @@ function commandNotFoundMessage(command, env = process.env) {
|
|
|
3405
4377
|
}
|
|
3406
4378
|
|
|
3407
4379
|
// src/lib/machines.ts
|
|
3408
|
-
import {
|
|
3409
|
-
|
|
3410
|
-
|
|
3411
|
-
|
|
4380
|
+
import { createRequire } from "module";
|
|
4381
|
+
var consumerModule;
|
|
4382
|
+
function machinesConsumer() {
|
|
4383
|
+
if (!consumerModule) {
|
|
4384
|
+
try {
|
|
4385
|
+
const resolved = Bun.resolveSync("@hasna/machines/consumer", import.meta.dir);
|
|
4386
|
+
consumerModule = createRequire(import.meta.url)(resolved);
|
|
4387
|
+
} catch (error) {
|
|
4388
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
4389
|
+
throw new Error(`@hasna/machines is not available; install the optional dependency to use machine-assigned loops: ${detail}`);
|
|
4390
|
+
}
|
|
4391
|
+
}
|
|
4392
|
+
return consumerModule;
|
|
4393
|
+
}
|
|
3412
4394
|
function compact(value) {
|
|
3413
4395
|
const text = value?.trim();
|
|
3414
4396
|
return text ? text : undefined;
|
|
@@ -3445,14 +4427,18 @@ function machineFromRoute(route, topology) {
|
|
|
3445
4427
|
};
|
|
3446
4428
|
}
|
|
3447
4429
|
function listOpenMachines() {
|
|
3448
|
-
const topology = discoverMachineTopology();
|
|
4430
|
+
const topology = machinesConsumer().discoverMachineTopology();
|
|
3449
4431
|
return topology.machines.map((entry) => entryToSummary(entry, topology));
|
|
3450
4432
|
}
|
|
3451
4433
|
function resolveLoopMachine(machineId) {
|
|
3452
|
-
const
|
|
3453
|
-
const
|
|
4434
|
+
const consumer = machinesConsumer();
|
|
4435
|
+
const topology = consumer.discoverMachineTopology();
|
|
4436
|
+
const route = consumer.resolveMachineRoute(machineId, { topology });
|
|
3454
4437
|
return machineFromRoute(route, topology);
|
|
3455
4438
|
}
|
|
4439
|
+
function resolveMachineCommand(machineId, command) {
|
|
4440
|
+
return machinesConsumer().resolveMachineCommand(machineId, command);
|
|
4441
|
+
}
|
|
3456
4442
|
function refreshLoopMachine(machine) {
|
|
3457
4443
|
return resolveLoopMachine(machine.id);
|
|
3458
4444
|
}
|
|
@@ -3460,6 +4446,10 @@ function refreshLoopMachine(machine) {
|
|
|
3460
4446
|
// src/lib/executor.ts
|
|
3461
4447
|
var DEFAULT_TIMEOUT_MS = 30 * 60000;
|
|
3462
4448
|
var DEFAULT_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
4449
|
+
var DEFAULT_AGENT_IDLE_TIMEOUT_MS = 30 * 60000;
|
|
4450
|
+
var BUFFERED_OUTPUT_PROVIDERS = new Set(["claude", "codewith", "opencode", "aicopilot"]);
|
|
4451
|
+
var DEFAULT_BUFFERED_AGENT_IDLE_TIMEOUT_MS = 4 * 60 * 60000;
|
|
4452
|
+
var WORKTREE_GIT_TIMEOUT_MS = 5 * 60000;
|
|
3463
4453
|
var AUTH_ENV_KEYS = [
|
|
3464
4454
|
"CLAUDE_CONFIG_DIR",
|
|
3465
4455
|
"CODEWITH_HOME",
|
|
@@ -3495,34 +4485,39 @@ var TRANSPORT_ENV_KEYS = new Set([
|
|
|
3495
4485
|
"USER",
|
|
3496
4486
|
"XDG_RUNTIME_DIR"
|
|
3497
4487
|
]);
|
|
3498
|
-
function
|
|
3499
|
-
const
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
3517
|
-
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
4488
|
+
function boundedText(text, maxBytes) {
|
|
4489
|
+
const buffer = new BoundedOutputBuffer(maxBytes);
|
|
4490
|
+
buffer.append(text);
|
|
4491
|
+
return buffer.value();
|
|
4492
|
+
}
|
|
4493
|
+
function buildResult(status, startedAt, fields = {}) {
|
|
4494
|
+
const finishedAt = fields.finishedAt ?? nowIso();
|
|
4495
|
+
return {
|
|
4496
|
+
status,
|
|
4497
|
+
exitCode: fields.exitCode,
|
|
4498
|
+
stdout: fields.stdout ?? "",
|
|
4499
|
+
stderr: fields.stderr ?? "",
|
|
4500
|
+
error: fields.error,
|
|
4501
|
+
pid: fields.pid,
|
|
4502
|
+
startedAt,
|
|
4503
|
+
finishedAt,
|
|
4504
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4505
|
+
};
|
|
4506
|
+
}
|
|
4507
|
+
function failureResult(startedAt, error, fields = {}) {
|
|
4508
|
+
return buildResult("failed", startedAt, { ...fields, error });
|
|
4509
|
+
}
|
|
4510
|
+
function timeoutResult(startedAt, error, fields = {}) {
|
|
4511
|
+
return buildResult("timed_out", startedAt, { ...fields, error });
|
|
4512
|
+
}
|
|
4513
|
+
function successResult(startedAt, fields = {}) {
|
|
4514
|
+
return buildResult("succeeded", startedAt, fields);
|
|
4515
|
+
}
|
|
4516
|
+
function notifySpawn(pid, opts) {
|
|
4517
|
+
if (!pid)
|
|
4518
|
+
return;
|
|
4519
|
+
opts.onSpawn?.(pid);
|
|
4520
|
+
opts.onSpawnProcess?.({ pid, pgid: pid, processStartedAt: nowIso() });
|
|
3526
4521
|
}
|
|
3527
4522
|
function shellQuote(value) {
|
|
3528
4523
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
@@ -3578,229 +4573,21 @@ function allowlistEnv(allowlist) {
|
|
|
3578
4573
|
env.LOOPS_AGENT_ALLOWLIST_ENFORCEMENT = "metadata_only";
|
|
3579
4574
|
return env;
|
|
3580
4575
|
}
|
|
3581
|
-
function
|
|
3582
|
-
|
|
3583
|
-
case "claude":
|
|
3584
|
-
return "claude";
|
|
3585
|
-
case "cursor":
|
|
3586
|
-
return "sh";
|
|
3587
|
-
case "codewith":
|
|
3588
|
-
return "codewith";
|
|
3589
|
-
case "aicopilot":
|
|
3590
|
-
return "aicopilot";
|
|
3591
|
-
case "opencode":
|
|
3592
|
-
return "opencode";
|
|
3593
|
-
case "codex":
|
|
3594
|
-
return "codex";
|
|
3595
|
-
}
|
|
3596
|
-
}
|
|
3597
|
-
function codewithLikeSandbox(target) {
|
|
3598
|
-
const sandbox = target.sandbox ?? (target.permissionMode === "bypass" ? "danger-full-access" : "workspace-write");
|
|
3599
|
-
if (sandbox !== "read-only" && sandbox !== "workspace-write" && sandbox !== "danger-full-access") {
|
|
3600
|
-
throw new Error(`${target.provider} sandbox must be read-only, workspace-write, or danger-full-access`);
|
|
3601
|
-
}
|
|
3602
|
-
return sandbox;
|
|
3603
|
-
}
|
|
3604
|
-
function configStringValue(value) {
|
|
3605
|
-
return JSON.stringify(value);
|
|
3606
|
-
}
|
|
3607
|
-
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS2 = new Set([
|
|
3608
|
-
"e",
|
|
3609
|
-
"exec",
|
|
3610
|
-
"agent",
|
|
3611
|
-
"start",
|
|
3612
|
-
"--ephemeral",
|
|
3613
|
-
"--ignore-rules",
|
|
3614
|
-
"--skip-git-repo-check",
|
|
3615
|
-
"--json",
|
|
3616
|
-
"--output-last-message",
|
|
3617
|
-
"-o",
|
|
3618
|
-
"--output-schema",
|
|
3619
|
-
"--dangerously-bypass-approvals-and-sandbox"
|
|
3620
|
-
]);
|
|
3621
|
-
function assertStringOption(value, label) {
|
|
3622
|
-
if (value !== undefined && typeof value !== "string")
|
|
3623
|
-
throw new Error(`${label} must be a string`);
|
|
3624
|
-
}
|
|
3625
|
-
function assertSupportedAgentOptions(target) {
|
|
3626
|
-
assertStringOption(target.variant, `${target.provider}.variant`);
|
|
3627
|
-
assertStringOption(target.model, `${target.provider}.model`);
|
|
3628
|
-
assertStringOption(target.agent, `${target.provider}.agent`);
|
|
3629
|
-
assertStringOption(target.authProfile, `${target.provider}.authProfile`);
|
|
3630
|
-
assertStringOption(target.configIsolation, `${target.provider}.configIsolation`);
|
|
3631
|
-
if (target.provider === "opencode" && (target.model === undefined || target.model.trim() === "")) {
|
|
3632
|
-
throw new Error("opencode.model is required; pass a provider/model id such as openrouter/google/gemini-2.5-flash");
|
|
3633
|
-
}
|
|
3634
|
-
if (target.configIsolation !== undefined && target.configIsolation !== "safe" && target.configIsolation !== "none") {
|
|
3635
|
-
throw new Error(`${target.provider}.configIsolation must be safe or none`);
|
|
3636
|
-
}
|
|
3637
|
-
if (target.authProfile !== undefined && target.provider !== "codewith") {
|
|
3638
|
-
throw new Error(`${target.provider}.authProfile is supported only for codewith`);
|
|
3639
|
-
}
|
|
3640
|
-
if (target.addDirs?.length && !["codewith", "codex"].includes(target.provider)) {
|
|
3641
|
-
throw new Error(`${target.provider}.addDirs is currently supported only for codewith or codex`);
|
|
3642
|
-
}
|
|
3643
|
-
if (target.permissionMode && !["default", "plan", "auto", "bypass"].includes(target.permissionMode)) {
|
|
3644
|
-
throw new Error(`${target.provider}.permissionMode must be default, plan, auto, or bypass`);
|
|
3645
|
-
}
|
|
3646
|
-
if (target.sandbox && !["read-only", "workspace-write", "danger-full-access", "enabled", "disabled"].includes(target.sandbox)) {
|
|
3647
|
-
throw new Error(`${target.provider}.sandbox is not supported: ${target.sandbox}`);
|
|
3648
|
-
}
|
|
3649
|
-
if (target.provider === "codex" && target.agent !== undefined)
|
|
3650
|
-
throw new Error("codex.agent is not supported");
|
|
3651
|
-
if (target.provider === "codewith" && target.agent !== undefined) {
|
|
3652
|
-
throw new Error("codewith.agent is not supported by the durable background-agent adapter");
|
|
3653
|
-
}
|
|
3654
|
-
if (target.provider === "codewith") {
|
|
3655
|
-
const unsafe = target.extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS2.has(arg));
|
|
3656
|
-
if (unsafe)
|
|
3657
|
-
throw new Error(`codewith.extraArgs cannot include ${unsafe}; durable agent steps use codewith agent start, not exec/ephemeral flags`);
|
|
3658
|
-
}
|
|
3659
|
-
if (["codewith", "codex"].includes(target.provider)) {
|
|
3660
|
-
if (target.permissionMode && !["default", "bypass"].includes(target.permissionMode)) {
|
|
3661
|
-
throw new Error(`${target.provider}.permissionMode supports only default or bypass`);
|
|
3662
|
-
}
|
|
3663
|
-
if (target.sandbox)
|
|
3664
|
-
codewithLikeSandbox(target);
|
|
3665
|
-
return;
|
|
3666
|
-
}
|
|
3667
|
-
if (target.provider === "claude") {
|
|
3668
|
-
if (target.sandbox !== undefined)
|
|
3669
|
-
throw new Error("claude.sandbox is not supported");
|
|
4576
|
+
function defaultAgentIdleTimeoutMs(target, opts) {
|
|
4577
|
+
if (target.timeoutMs !== undefined || target.idleTimeoutMs !== undefined)
|
|
3670
4578
|
return;
|
|
4579
|
+
const raw = opts.env?.LOOPS_AGENT_IDLE_TIMEOUT_MS ?? process.env.LOOPS_AGENT_IDLE_TIMEOUT_MS;
|
|
4580
|
+
if (raw !== undefined && raw !== "") {
|
|
4581
|
+
const normalized = raw.trim().toLowerCase();
|
|
4582
|
+
if (normalized === "0" || normalized === "none" || normalized === "off")
|
|
4583
|
+
return;
|
|
4584
|
+
const parsed = Number(normalized);
|
|
4585
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
4586
|
+
return parsed;
|
|
3671
4587
|
}
|
|
3672
|
-
|
|
3673
|
-
if (target.variant !== undefined)
|
|
3674
|
-
throw new Error("cursor.variant is not supported");
|
|
3675
|
-
if (target.permissionMode === "auto")
|
|
3676
|
-
throw new Error("cursor.permissionMode auto is not supported; use provider-specific extraArgs for Cursor auto-review");
|
|
3677
|
-
if (target.sandbox !== undefined && target.sandbox !== "enabled" && target.sandbox !== "disabled") {
|
|
3678
|
-
throw new Error("cursor.sandbox must be enabled or disabled");
|
|
3679
|
-
}
|
|
3680
|
-
return;
|
|
3681
|
-
}
|
|
3682
|
-
if (target.permissionMode && !["default", "bypass"].includes(target.permissionMode)) {
|
|
3683
|
-
throw new Error(`${target.provider}.permissionMode supports only default or bypass`);
|
|
3684
|
-
}
|
|
3685
|
-
if (target.sandbox !== undefined)
|
|
3686
|
-
throw new Error(`${target.provider}.sandbox is not supported`);
|
|
3687
|
-
}
|
|
3688
|
-
function agentArgs(target) {
|
|
3689
|
-
assertSupportedAgentOptions(target);
|
|
3690
|
-
const isolation = target.configIsolation ?? "safe";
|
|
3691
|
-
const permissionMode = target.permissionMode ?? "default";
|
|
3692
|
-
const args = [];
|
|
3693
|
-
switch (target.provider) {
|
|
3694
|
-
case "claude":
|
|
3695
|
-
if (isolation === "safe")
|
|
3696
|
-
args.push("--safe-mode", "--setting-sources", "local", "--no-session-persistence");
|
|
3697
|
-
if (permissionMode !== "default") {
|
|
3698
|
-
const mode = permissionMode === "bypass" ? "bypassPermissions" : permissionMode === "plan" || permissionMode === "auto" ? permissionMode : undefined;
|
|
3699
|
-
if (mode)
|
|
3700
|
-
args.push("--permission-mode", mode);
|
|
3701
|
-
}
|
|
3702
|
-
args.push("-p", "--output-format", "json");
|
|
3703
|
-
if (target.model)
|
|
3704
|
-
args.push("--model", target.model);
|
|
3705
|
-
if (target.variant)
|
|
3706
|
-
args.push("--effort", target.variant);
|
|
3707
|
-
if (target.agent)
|
|
3708
|
-
args.push("--agent", target.agent);
|
|
3709
|
-
args.push(...target.extraArgs ?? []);
|
|
3710
|
-
return args;
|
|
3711
|
-
case "cursor":
|
|
3712
|
-
args.push("-c", [
|
|
3713
|
-
"set -eu",
|
|
3714
|
-
"if command -v agent >/dev/null 2>&1; then",
|
|
3715
|
-
' exec agent "$@"',
|
|
3716
|
-
"else",
|
|
3717
|
-
" echo 'Executable not found in PATH: agent' >&2",
|
|
3718
|
-
" exit 127",
|
|
3719
|
-
"fi"
|
|
3720
|
-
].join(`
|
|
3721
|
-
`), "openloops-cursor", "-p");
|
|
3722
|
-
if (permissionMode === "plan")
|
|
3723
|
-
args.push("--mode", "plan");
|
|
3724
|
-
if (permissionMode === "bypass")
|
|
3725
|
-
args.push("--force");
|
|
3726
|
-
const cursorSandbox = target.sandbox ?? (isolation === "safe" ? "enabled" : undefined);
|
|
3727
|
-
if (cursorSandbox) {
|
|
3728
|
-
if (cursorSandbox !== "enabled" && cursorSandbox !== "disabled")
|
|
3729
|
-
throw new Error("cursor sandbox must be enabled or disabled");
|
|
3730
|
-
args.push("--sandbox", cursorSandbox);
|
|
3731
|
-
}
|
|
3732
|
-
if (target.model)
|
|
3733
|
-
args.push("--model", target.model);
|
|
3734
|
-
if (target.agent)
|
|
3735
|
-
args.push("--agent", target.agent);
|
|
3736
|
-
args.push(...target.extraArgs ?? []);
|
|
3737
|
-
return args;
|
|
3738
|
-
case "codewith":
|
|
3739
|
-
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : []);
|
|
3740
|
-
if (target.variant)
|
|
3741
|
-
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
3742
|
-
args.push("--ask-for-approval", "never", "--sandbox", codewithLikeSandbox(target));
|
|
3743
|
-
if (target.cwd)
|
|
3744
|
-
args.push("--cd", target.cwd);
|
|
3745
|
-
for (const dir of target.addDirs ?? [])
|
|
3746
|
-
args.push("--add-dir", dir);
|
|
3747
|
-
if (target.model)
|
|
3748
|
-
args.push("--model", target.model);
|
|
3749
|
-
args.push(...target.extraArgs ?? []);
|
|
3750
|
-
args.push("agent", "start");
|
|
3751
|
-
if (target.cwd)
|
|
3752
|
-
args.push("--cwd", target.cwd);
|
|
3753
|
-
args.push(target.prompt);
|
|
3754
|
-
return args;
|
|
3755
|
-
case "codex":
|
|
3756
|
-
if (target.variant)
|
|
3757
|
-
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
3758
|
-
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
3759
|
-
if (isolation === "safe")
|
|
3760
|
-
args.push("--ignore-rules");
|
|
3761
|
-
if (target.cwd)
|
|
3762
|
-
args.push("--cd", target.cwd);
|
|
3763
|
-
for (const dir of target.addDirs ?? [])
|
|
3764
|
-
args.push("--add-dir", dir);
|
|
3765
|
-
if (target.model)
|
|
3766
|
-
args.push("--model", target.model);
|
|
3767
|
-
args.push(...target.extraArgs ?? []);
|
|
3768
|
-
return args;
|
|
3769
|
-
case "aicopilot":
|
|
3770
|
-
args.push("run", "--format", "json");
|
|
3771
|
-
if (isolation === "safe")
|
|
3772
|
-
args.push("--pure");
|
|
3773
|
-
if (permissionMode === "bypass")
|
|
3774
|
-
args.push("--dangerously-skip-permissions");
|
|
3775
|
-
if (target.cwd)
|
|
3776
|
-
args.push("--dir", target.cwd);
|
|
3777
|
-
if (target.model)
|
|
3778
|
-
args.push("--model", target.model);
|
|
3779
|
-
if (target.variant)
|
|
3780
|
-
args.push("--variant", target.variant);
|
|
3781
|
-
if (target.agent)
|
|
3782
|
-
args.push("--agent", target.agent);
|
|
3783
|
-
args.push(...target.extraArgs ?? []);
|
|
3784
|
-
return args;
|
|
3785
|
-
case "opencode":
|
|
3786
|
-
args.push("run", "--format", "json");
|
|
3787
|
-
if (isolation === "safe")
|
|
3788
|
-
args.push("--pure");
|
|
3789
|
-
if (permissionMode === "bypass")
|
|
3790
|
-
args.push("--dangerously-skip-permissions");
|
|
3791
|
-
if (target.cwd)
|
|
3792
|
-
args.push("--dir", target.cwd);
|
|
3793
|
-
if (target.model)
|
|
3794
|
-
args.push("--model", target.model);
|
|
3795
|
-
if (target.variant)
|
|
3796
|
-
args.push("--variant", target.variant);
|
|
3797
|
-
if (target.agent)
|
|
3798
|
-
args.push("--agent", target.agent);
|
|
3799
|
-
args.push(...target.extraArgs ?? []);
|
|
3800
|
-
return args;
|
|
3801
|
-
}
|
|
4588
|
+
return BUFFERED_OUTPUT_PROVIDERS.has(target.provider) ? DEFAULT_BUFFERED_AGENT_IDLE_TIMEOUT_MS : DEFAULT_AGENT_IDLE_TIMEOUT_MS;
|
|
3802
4589
|
}
|
|
3803
|
-
function commandSpec(target) {
|
|
4590
|
+
function commandSpec(target, opts) {
|
|
3804
4591
|
if (target.type === "command") {
|
|
3805
4592
|
const commandTarget = target;
|
|
3806
4593
|
return {
|
|
@@ -3816,25 +4603,27 @@ function commandSpec(target) {
|
|
|
3816
4603
|
};
|
|
3817
4604
|
}
|
|
3818
4605
|
const agentTarget = target;
|
|
4606
|
+
const adapter = providerAdapter(agentTarget.provider);
|
|
4607
|
+
const invocation = adapter.buildInvocation(agentTarget);
|
|
3819
4608
|
return {
|
|
3820
|
-
command:
|
|
3821
|
-
args:
|
|
4609
|
+
command: invocation.command,
|
|
4610
|
+
args: invocation.args,
|
|
3822
4611
|
cwd: agentTarget.cwd,
|
|
3823
4612
|
timeoutMs: agentTarget.timeoutMs ?? null,
|
|
3824
|
-
idleTimeoutMs: agentTarget.idleTimeoutMs,
|
|
4613
|
+
idleTimeoutMs: agentTarget.idleTimeoutMs ?? defaultAgentIdleTimeoutMs(agentTarget, opts),
|
|
3825
4614
|
account: agentTarget.account,
|
|
3826
4615
|
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
|
|
3827
4616
|
nativeAuthProfile: agentTarget.authProfile ? { provider: agentTarget.provider, profile: agentTarget.authProfile } : undefined,
|
|
3828
|
-
preflightAnyOf:
|
|
3829
|
-
stdin:
|
|
4617
|
+
preflightAnyOf: invocation.preflightAnyOf,
|
|
4618
|
+
stdin: invocation.stdin,
|
|
3830
4619
|
allowlist: agentTarget.allowlist,
|
|
3831
|
-
|
|
4620
|
+
worktree: agentTarget.worktree,
|
|
4621
|
+
codewithDurableAgent: adapter.capabilities.durable ? { target: agentTarget } : undefined
|
|
3832
4622
|
};
|
|
3833
4623
|
}
|
|
3834
|
-
function
|
|
4624
|
+
function composeExecutionEnv(spec, metadata, opts, accountEnv) {
|
|
3835
4625
|
const env = { ...opts.env ?? process.env };
|
|
3836
|
-
if (
|
|
3837
|
-
const accountEnv = resolveAccountEnv(spec.account, spec.accountTool, env);
|
|
4626
|
+
if (accountEnv) {
|
|
3838
4627
|
for (const key of AUTH_ENV_KEYS)
|
|
3839
4628
|
delete env[key];
|
|
3840
4629
|
Object.assign(env, accountEnv);
|
|
@@ -3845,6 +4634,14 @@ function executionEnv(spec, metadata, opts) {
|
|
|
3845
4634
|
Object.assign(env, metadataEnv(metadata));
|
|
3846
4635
|
return env;
|
|
3847
4636
|
}
|
|
4637
|
+
async function executionEnv(spec, metadata, opts) {
|
|
4638
|
+
const accountEnv = spec.account ? await resolveAccountEnv(spec.account, spec.accountTool, { ...opts.env ?? process.env }) : undefined;
|
|
4639
|
+
return composeExecutionEnv(spec, metadata, opts, accountEnv);
|
|
4640
|
+
}
|
|
4641
|
+
function executionEnvSync(spec, metadata, opts) {
|
|
4642
|
+
const accountEnv = spec.account ? resolveAccountEnvSync(spec.account, spec.accountTool, { ...opts.env ?? process.env }) : undefined;
|
|
4643
|
+
return composeExecutionEnv(spec, metadata, opts, accountEnv);
|
|
4644
|
+
}
|
|
3848
4645
|
function resolvedMachine(opts) {
|
|
3849
4646
|
if (!opts.machine)
|
|
3850
4647
|
return;
|
|
@@ -3862,13 +4659,21 @@ function hereDoc(value) {
|
|
|
3862
4659
|
}
|
|
3863
4660
|
return [`cat > "$__OPENLOOPS_STDIN" <<'${delimiter2}'`, value, delimiter2];
|
|
3864
4661
|
}
|
|
3865
|
-
function remoteBootstrapLines(spec, metadata) {
|
|
4662
|
+
function remoteBootstrapLines(spec, metadata, opts = {}) {
|
|
3866
4663
|
const lines = [
|
|
3867
4664
|
"set -e",
|
|
3868
4665
|
'export PATH="$HOME/.local/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$HOME/.npm-global/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${PATH:+:$PATH}"'
|
|
3869
4666
|
];
|
|
3870
|
-
|
|
4667
|
+
const worktree = spec.worktree;
|
|
4668
|
+
const worktreeManaged = worktree?.enabled && (worktree.mode === "auto" || worktree.mode === "required");
|
|
4669
|
+
if (opts.worktree && worktree && worktreeManaged) {
|
|
4670
|
+
lines.push(...remoteWorktreePrepareLines(worktree));
|
|
4671
|
+
lines.push(...remoteWorktreeEnterLines(worktree, spec.cwd));
|
|
4672
|
+
} else if (worktree && worktreeManaged) {
|
|
4673
|
+
lines.push(`cd ${shellQuote(worktree.originalCwd)}`);
|
|
4674
|
+
} else if (spec.cwd) {
|
|
3871
4675
|
lines.push(`cd ${shellQuote(spec.cwd)}`);
|
|
4676
|
+
}
|
|
3872
4677
|
if (spec.account) {
|
|
3873
4678
|
if (!spec.accountTool)
|
|
3874
4679
|
throw new Error("account.tool is required when no provider tool can be inferred");
|
|
@@ -3884,16 +4689,79 @@ function remoteBootstrapLines(spec, metadata) {
|
|
|
3884
4689
|
}
|
|
3885
4690
|
return lines;
|
|
3886
4691
|
}
|
|
3887
|
-
function
|
|
3888
|
-
const
|
|
4692
|
+
function remoteWorktreePrepareLines(worktree) {
|
|
4693
|
+
const { repoRoot, path, branch } = worktree;
|
|
4694
|
+
if (!repoRoot || !path || !branch) {
|
|
4695
|
+
return [
|
|
4696
|
+
"__openloops_prepare_worktree() {",
|
|
4697
|
+
` echo ${shellQuote("worktree preparation requires repoRoot, path, and branch metadata")} >&2`,
|
|
4698
|
+
" return 1",
|
|
4699
|
+
"}"
|
|
4700
|
+
];
|
|
4701
|
+
}
|
|
4702
|
+
return [
|
|
4703
|
+
"__openloops_prepare_worktree() {",
|
|
4704
|
+
` local repo=${shellQuote(repoRoot)} path=${shellQuote(path)} branch=${shellQuote(branch)}`,
|
|
4705
|
+
" local top expected_common actual_common current",
|
|
4706
|
+
' if [ -L "$path" ]; then echo "refusing symlinked worktree path $path" >&2; return 1; fi',
|
|
4707
|
+
' if [ -e "$path" ]; then',
|
|
4708
|
+
' top="$(git -C "$path" rev-parse --show-toplevel 2>/dev/null)" || { echo "refusing to reuse non-worktree path: $path" >&2; return 1; }',
|
|
4709
|
+
' if [ "$(cd "$top" 2>/dev/null && pwd -P)" != "$(cd "$path" 2>/dev/null && pwd -P)" ]; then echo "existing worktree top-level mismatch for $path: $top" >&2; return 1; fi',
|
|
4710
|
+
' expected_common="$(cd "$repo" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
|
|
4711
|
+
' actual_common="$(cd "$path" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
|
|
4712
|
+
' if [ -z "$expected_common" ] || [ "$expected_common" != "$actual_common" ]; then echo "existing worktree $path belongs to a different git common dir" >&2; return 1; fi',
|
|
4713
|
+
' current="$(git -C "$path" branch --show-current 2>/dev/null)"',
|
|
4714
|
+
' if [ "$current" != "$branch" ]; then echo "existing worktree $path is on branch ${current:-unknown}, expected $branch" >&2; return 1; fi',
|
|
4715
|
+
" return 0",
|
|
4716
|
+
" fi",
|
|
4717
|
+
' git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
|
|
4718
|
+
' mkdir -p "$(dirname "$path")" || return 1',
|
|
4719
|
+
" # Preparation chatter goes to stderr so run stdout stays the agent's.",
|
|
4720
|
+
' if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
|
|
4721
|
+
' git -C "$repo" worktree add "$path" "$branch" 1>&2 || return 1',
|
|
4722
|
+
" else",
|
|
4723
|
+
' git -C "$repo" worktree add -b "$branch" "$path" HEAD 1>&2 || return 1',
|
|
4724
|
+
" fi",
|
|
4725
|
+
"}"
|
|
4726
|
+
];
|
|
4727
|
+
}
|
|
4728
|
+
function remoteWorktreeEnterLines(worktree, cwd) {
|
|
4729
|
+
const workdir = cwd ?? worktree.cwd;
|
|
4730
|
+
if (worktree.mode === "required") {
|
|
4731
|
+
return [
|
|
4732
|
+
"if ! __openloops_prepare_worktree; then",
|
|
4733
|
+
` echo ${shellQuote("worktree preparation failed (mode=required)")} >&2`,
|
|
4734
|
+
" exit 1",
|
|
4735
|
+
"fi",
|
|
4736
|
+
"__OPENLOOPS_WORKTREE_OK=1",
|
|
4737
|
+
`cd ${shellQuote(workdir)}`
|
|
4738
|
+
];
|
|
4739
|
+
}
|
|
4740
|
+
return [
|
|
4741
|
+
"if __openloops_prepare_worktree; then",
|
|
4742
|
+
" __OPENLOOPS_WORKTREE_OK=1",
|
|
4743
|
+
` cd ${shellQuote(workdir)}`,
|
|
4744
|
+
"else",
|
|
4745
|
+
` echo ${shellQuote(`worktree preparation failed (mode=${worktree.mode}); falling back to ${worktree.originalCwd}`)} >&2`,
|
|
4746
|
+
" __OPENLOOPS_WORKTREE_OK=0",
|
|
4747
|
+
` cd ${shellQuote(worktree.originalCwd)}`,
|
|
4748
|
+
"fi"
|
|
4749
|
+
];
|
|
4750
|
+
}
|
|
4751
|
+
function remoteScript(spec, metadata, fallbackSpec) {
|
|
4752
|
+
const lines = remoteBootstrapLines(spec, metadata, { worktree: true });
|
|
3889
4753
|
let stdinRedirect = "";
|
|
3890
4754
|
if (spec.stdin !== undefined) {
|
|
3891
4755
|
lines.push('__OPENLOOPS_STDIN="$(mktemp -t openloops-stdin.XXXXXX)"', `trap 'rm -f "$__OPENLOOPS_STDIN"' EXIT`);
|
|
3892
4756
|
lines.push(...hereDoc(spec.stdin));
|
|
3893
4757
|
stdinRedirect = ' < "$__OPENLOOPS_STDIN"';
|
|
3894
4758
|
}
|
|
3895
|
-
const
|
|
3896
|
-
|
|
4759
|
+
const invocationFor = (invocationSpec) => invocationSpec.shell ? `sh -c ${shellQuote(commandForShell(invocationSpec))}${stdinRedirect}` : `${[invocationSpec.command, ...invocationSpec.args].map(shellQuote).join(" ")}${stdinRedirect}`;
|
|
4760
|
+
if (spec.worktree?.enabled && spec.worktree.mode === "auto" && fallbackSpec) {
|
|
4761
|
+
lines.push('if [ "${__OPENLOOPS_WORKTREE_OK:-0}" = 1 ]; then', ` ${invocationFor(spec)}`, "else", ` ${invocationFor(fallbackSpec)}`, "fi");
|
|
4762
|
+
} else {
|
|
4763
|
+
lines.push(invocationFor(spec));
|
|
4764
|
+
}
|
|
3897
4765
|
return `${lines.join(`
|
|
3898
4766
|
`)}
|
|
3899
4767
|
`;
|
|
@@ -3925,31 +4793,43 @@ function transportEnv(opts) {
|
|
|
3925
4793
|
env.PATH = normalizeExecutionPath(env);
|
|
3926
4794
|
return env;
|
|
3927
4795
|
}
|
|
3928
|
-
function
|
|
3929
|
-
if (!spec.nativeAuthProfile)
|
|
3930
|
-
return;
|
|
3931
|
-
if (spec.nativeAuthProfile.provider !== "codewith")
|
|
3932
|
-
return;
|
|
3933
|
-
const result = spawnSync2(spec.command, ["profile", "list"], {
|
|
3934
|
-
encoding: "utf8",
|
|
3935
|
-
env,
|
|
3936
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
3937
|
-
timeout: 15000
|
|
3938
|
-
});
|
|
4796
|
+
function assertCodewithProfileListed(profile, result) {
|
|
3939
4797
|
if (result.error) {
|
|
3940
|
-
throw new Error(`codewith auth profile preflight failed: ${result.error
|
|
4798
|
+
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
3941
4799
|
}
|
|
3942
4800
|
if ((result.status ?? 1) !== 0) {
|
|
3943
4801
|
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
3944
4802
|
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
3945
4803
|
}
|
|
3946
4804
|
const profiles = new Set((result.stdout || "").split(/\r?\n/).slice(1).map((line) => line.trim().split(/\s+/)[0]).filter(Boolean));
|
|
3947
|
-
if (!profiles.has(
|
|
3948
|
-
throw new Error(`codewith auth profile not found: ${
|
|
4805
|
+
if (!profiles.has(profile)) {
|
|
4806
|
+
throw new Error(`codewith auth profile not found: ${profile}`);
|
|
3949
4807
|
}
|
|
3950
4808
|
}
|
|
4809
|
+
function preflightNativeAuthProfileSync(spec, env) {
|
|
4810
|
+
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
4811
|
+
return;
|
|
4812
|
+
const result = spawnSync4(spec.command, ["profile", "list"], {
|
|
4813
|
+
encoding: "utf8",
|
|
4814
|
+
env,
|
|
4815
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
4816
|
+
timeout: 15000
|
|
4817
|
+
});
|
|
4818
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, {
|
|
4819
|
+
status: result.status,
|
|
4820
|
+
stdout: result.stdout ?? "",
|
|
4821
|
+
stderr: result.stderr ?? "",
|
|
4822
|
+
error: result.error?.message
|
|
4823
|
+
});
|
|
4824
|
+
}
|
|
4825
|
+
async function preflightNativeAuthProfile(spec, env) {
|
|
4826
|
+
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
4827
|
+
return;
|
|
4828
|
+
const result = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
4829
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, result);
|
|
4830
|
+
}
|
|
3951
4831
|
function codewithAgentStartArgs(target, idempotencyKey) {
|
|
3952
|
-
const args =
|
|
4832
|
+
const args = providerAdapter(target.provider).buildInvocation(target).args;
|
|
3953
4833
|
const startIndex = args.findIndex((arg, index) => arg === "start" && args[index - 1] === "agent");
|
|
3954
4834
|
if (startIndex === -1)
|
|
3955
4835
|
throw new Error("internal error: codewith durable agent args missing agent start");
|
|
@@ -4023,7 +4903,7 @@ function codewithAgentEvidence(startJson, readJson, logsJson) {
|
|
|
4023
4903
|
}, null, 2);
|
|
4024
4904
|
}
|
|
4025
4905
|
function sleep(ms) {
|
|
4026
|
-
return new Promise((
|
|
4906
|
+
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
4027
4907
|
}
|
|
4028
4908
|
async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt) {
|
|
4029
4909
|
const target = spec.codewithDurableAgent?.target;
|
|
@@ -4031,116 +4911,84 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
|
|
|
4031
4911
|
throw new Error("internal error: missing codewith durable target");
|
|
4032
4912
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
4033
4913
|
const idempotencyKey = codewithAgentIdempotencyKey(metadata);
|
|
4034
|
-
const start =
|
|
4914
|
+
const start = await spawnCapture(spec.command, codewithAgentStartArgs(target, idempotencyKey), {
|
|
4035
4915
|
cwd: spec.cwd,
|
|
4036
4916
|
env,
|
|
4037
|
-
|
|
4038
|
-
|
|
4039
|
-
timeout: 30000
|
|
4917
|
+
timeoutMs: 30000,
|
|
4918
|
+
maxOutputBytes
|
|
4040
4919
|
});
|
|
4041
|
-
|
|
4920
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
4921
|
+
stderr.append(start.stderr);
|
|
4042
4922
|
if (start.error || (start.status ?? 1) !== 0) {
|
|
4043
|
-
|
|
4044
|
-
return {
|
|
4045
|
-
status: "failed",
|
|
4923
|
+
return failureResult(startedAt, start.error ?? `codewith agent start exited with code ${start.status ?? "unknown"}`, {
|
|
4046
4924
|
exitCode: start.status ?? undefined,
|
|
4047
|
-
stdout: start.stdout
|
|
4048
|
-
stderr
|
|
4049
|
-
|
|
4050
|
-
startedAt,
|
|
4051
|
-
finishedAt,
|
|
4052
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4053
|
-
};
|
|
4925
|
+
stdout: start.stdout,
|
|
4926
|
+
stderr: stderr.value()
|
|
4927
|
+
});
|
|
4054
4928
|
}
|
|
4055
4929
|
let startJson;
|
|
4056
4930
|
try {
|
|
4057
4931
|
startJson = parseJsonOutput(start.stdout || "{}", "codewith agent start");
|
|
4058
4932
|
} catch (error) {
|
|
4059
|
-
|
|
4060
|
-
return {
|
|
4061
|
-
status: "failed",
|
|
4062
|
-
stdout: start.stdout || "",
|
|
4063
|
-
stderr,
|
|
4064
|
-
error: error instanceof Error ? error.message : String(error),
|
|
4065
|
-
startedAt,
|
|
4066
|
-
finishedAt,
|
|
4067
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4068
|
-
};
|
|
4933
|
+
return failureResult(startedAt, error instanceof Error ? error.message : String(error), { stdout: start.stdout, stderr: stderr.value() });
|
|
4069
4934
|
}
|
|
4070
4935
|
const agentId = stringField(recordField(startJson, "agent"), "agentId");
|
|
4071
4936
|
if (!agentId) {
|
|
4072
|
-
|
|
4073
|
-
return {
|
|
4074
|
-
status: "failed",
|
|
4075
|
-
stdout: start.stdout || "",
|
|
4076
|
-
stderr,
|
|
4077
|
-
error: "codewith agent start did not return agent.agentId",
|
|
4078
|
-
startedAt,
|
|
4079
|
-
finishedAt,
|
|
4080
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4081
|
-
};
|
|
4937
|
+
return failureResult(startedAt, "codewith agent start did not return agent.agentId", { stdout: start.stdout, stderr: stderr.value() });
|
|
4082
4938
|
}
|
|
4939
|
+
const stopAgent = async () => {
|
|
4940
|
+
await spawnCapture(spec.command, codewithAgentControlArgs(target, "stop", agentId), {
|
|
4941
|
+
cwd: spec.cwd,
|
|
4942
|
+
env,
|
|
4943
|
+
timeoutMs: 15000,
|
|
4944
|
+
maxOutputBytes
|
|
4945
|
+
});
|
|
4946
|
+
};
|
|
4083
4947
|
const pollMs = Math.max(100, Number(opts.env?.LOOPS_CODEWITH_AGENT_POLL_MS ?? process.env.LOOPS_CODEWITH_AGENT_POLL_MS ?? 2000) || 2000);
|
|
4084
4948
|
let lastReadJson = startJson;
|
|
4085
4949
|
let lastLogsJson;
|
|
4086
|
-
let
|
|
4950
|
+
let lastFingerprint;
|
|
4951
|
+
let lastProgressAt = Date.now();
|
|
4952
|
+
const evidence = () => boundedText(codewithAgentEvidence(startJson, lastReadJson, lastLogsJson), maxOutputBytes);
|
|
4087
4953
|
while (true) {
|
|
4088
4954
|
if (opts.signal?.aborted) {
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
return {
|
|
4092
|
-
status: "failed",
|
|
4093
|
-
stdout: appendBoundedText(stdout, codewithAgentEvidence(startJson, lastReadJson, lastLogsJson), maxOutputBytes),
|
|
4094
|
-
stderr,
|
|
4095
|
-
error: "cancelled",
|
|
4096
|
-
startedAt,
|
|
4097
|
-
finishedAt,
|
|
4098
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4099
|
-
};
|
|
4955
|
+
await stopAgent();
|
|
4956
|
+
return failureResult(startedAt, "cancelled", { stdout: evidence(), stderr: stderr.value() });
|
|
4100
4957
|
}
|
|
4101
|
-
const read =
|
|
4958
|
+
const read = await spawnCapture(spec.command, codewithAgentControlArgs(target, "read", agentId), {
|
|
4102
4959
|
cwd: spec.cwd,
|
|
4103
4960
|
env,
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
timeout: 30000
|
|
4961
|
+
timeoutMs: 30000,
|
|
4962
|
+
maxOutputBytes
|
|
4107
4963
|
});
|
|
4108
|
-
stderr
|
|
4964
|
+
stderr.append(read.stderr);
|
|
4109
4965
|
if (read.error || (read.status ?? 1) !== 0) {
|
|
4110
|
-
|
|
4111
|
-
return {
|
|
4112
|
-
status: "failed",
|
|
4966
|
+
return failureResult(startedAt, read.error ?? `codewith agent read exited with code ${read.status ?? "unknown"}`, {
|
|
4113
4967
|
exitCode: read.status ?? undefined,
|
|
4114
|
-
stdout:
|
|
4115
|
-
stderr
|
|
4116
|
-
|
|
4117
|
-
startedAt,
|
|
4118
|
-
finishedAt,
|
|
4119
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4120
|
-
};
|
|
4968
|
+
stdout: evidence(),
|
|
4969
|
+
stderr: stderr.value()
|
|
4970
|
+
});
|
|
4121
4971
|
}
|
|
4122
4972
|
try {
|
|
4123
4973
|
lastReadJson = parseJsonOutput(read.stdout || "{}", "codewith agent read");
|
|
4124
4974
|
} catch (error) {
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
stderr,
|
|
4130
|
-
error: error instanceof Error ? error.message : String(error),
|
|
4131
|
-
startedAt,
|
|
4132
|
-
finishedAt,
|
|
4133
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4134
|
-
};
|
|
4975
|
+
return failureResult(startedAt, error instanceof Error ? error.message : String(error), {
|
|
4976
|
+
stdout: boundedText(read.stdout, maxOutputBytes),
|
|
4977
|
+
stderr: stderr.value()
|
|
4978
|
+
});
|
|
4135
4979
|
}
|
|
4136
4980
|
const status = codewithAgentStatus(lastReadJson);
|
|
4981
|
+
const fingerprint = JSON.stringify({ status, snapshot: recordField(lastReadJson, "statusSnapshot") ?? null });
|
|
4982
|
+
if (fingerprint !== lastFingerprint) {
|
|
4983
|
+
lastFingerprint = fingerprint;
|
|
4984
|
+
lastProgressAt = Date.now();
|
|
4985
|
+
}
|
|
4137
4986
|
if (status === "completed" || status === "failed" || status === "cancelled") {
|
|
4138
|
-
const logs =
|
|
4987
|
+
const logs = await spawnCapture(spec.command, codewithAgentControlArgs(target, "logs", agentId), {
|
|
4139
4988
|
cwd: spec.cwd,
|
|
4140
4989
|
env,
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
timeout: 30000
|
|
4990
|
+
timeoutMs: 30000,
|
|
4991
|
+
maxOutputBytes
|
|
4144
4992
|
});
|
|
4145
4993
|
if (!logs.error && (logs.status ?? 1) === 0) {
|
|
4146
4994
|
try {
|
|
@@ -4149,40 +4997,129 @@ async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt)
|
|
|
4149
4997
|
lastLogsJson = undefined;
|
|
4150
4998
|
}
|
|
4151
4999
|
} else {
|
|
4152
|
-
stderr
|
|
5000
|
+
stderr.append(logs.stderr || logs.error || "");
|
|
4153
5001
|
}
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
exitCode: resultStatus === "succeeded" ? 0 : 1,
|
|
4159
|
-
stdout: appendBoundedText(stdout, codewithAgentEvidence(startJson, lastReadJson, lastLogsJson), maxOutputBytes),
|
|
4160
|
-
stderr,
|
|
4161
|
-
error: resultStatus === "succeeded" ? undefined : stringField(recordField(lastReadJson, "agent"), "statusReason") ?? `codewith agent ${status}`,
|
|
4162
|
-
startedAt,
|
|
4163
|
-
finishedAt,
|
|
4164
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4165
|
-
};
|
|
5002
|
+
if (status === "completed") {
|
|
5003
|
+
return successResult(startedAt, { exitCode: 0, stdout: evidence(), stderr: stderr.value() });
|
|
5004
|
+
}
|
|
5005
|
+
return failureResult(startedAt, stringField(recordField(lastReadJson, "agent"), "statusReason") ?? `codewith agent ${status}`, { exitCode: 1, stdout: evidence(), stderr: stderr.value() });
|
|
4166
5006
|
}
|
|
4167
5007
|
if (typeof spec.timeoutMs === "number" && new Date(nowIso()).getTime() - new Date(startedAt).getTime() >= spec.timeoutMs) {
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4178
|
-
};
|
|
5008
|
+
await stopAgent();
|
|
5009
|
+
return timeoutResult(startedAt, `timed out after ${spec.timeoutMs}ms`, { stdout: evidence(), stderr: stderr.value() });
|
|
5010
|
+
}
|
|
5011
|
+
if (typeof spec.idleTimeoutMs === "number" && Date.now() - lastProgressAt >= spec.idleTimeoutMs) {
|
|
5012
|
+
await stopAgent();
|
|
5013
|
+
return timeoutResult(startedAt, `idle timed out after ${spec.idleTimeoutMs}ms without agent progress`, {
|
|
5014
|
+
stdout: evidence(),
|
|
5015
|
+
stderr: stderr.value()
|
|
5016
|
+
});
|
|
4179
5017
|
}
|
|
4180
5018
|
await sleep(pollMs);
|
|
4181
5019
|
}
|
|
4182
5020
|
}
|
|
5021
|
+
function resolvedDirEquals(left, right) {
|
|
5022
|
+
try {
|
|
5023
|
+
return realpathSync(left) === realpathSync(right);
|
|
5024
|
+
} catch {
|
|
5025
|
+
return false;
|
|
5026
|
+
}
|
|
5027
|
+
}
|
|
5028
|
+
async function ensureLocalWorktree(worktree, env) {
|
|
5029
|
+
const { repoRoot, path, branch } = worktree;
|
|
5030
|
+
if (!repoRoot || !path || !branch) {
|
|
5031
|
+
return { error: "worktree preparation requires repoRoot, path, and branch metadata" };
|
|
5032
|
+
}
|
|
5033
|
+
const git = (args) => spawnCapture("git", args, { env, timeoutMs: WORKTREE_GIT_TIMEOUT_MS });
|
|
5034
|
+
let stats;
|
|
5035
|
+
try {
|
|
5036
|
+
stats = lstatSync(path);
|
|
5037
|
+
} catch {
|
|
5038
|
+
stats = undefined;
|
|
5039
|
+
}
|
|
5040
|
+
if (stats?.isSymbolicLink())
|
|
5041
|
+
return { error: `refusing symlinked worktree path ${path}` };
|
|
5042
|
+
const commonDir = async (base) => {
|
|
5043
|
+
const result = await git(["-C", base, "rev-parse", "--git-common-dir"]);
|
|
5044
|
+
if (result.error || (result.status ?? 1) !== 0)
|
|
5045
|
+
return;
|
|
5046
|
+
const raw = result.stdout.trim();
|
|
5047
|
+
if (!raw)
|
|
5048
|
+
return;
|
|
5049
|
+
try {
|
|
5050
|
+
return realpathSync(resolve2(base, raw));
|
|
5051
|
+
} catch {
|
|
5052
|
+
return resolve2(base, raw);
|
|
5053
|
+
}
|
|
5054
|
+
};
|
|
5055
|
+
if (stats) {
|
|
5056
|
+
const top = await git(["-C", path, "rev-parse", "--show-toplevel"]);
|
|
5057
|
+
if (top.error || (top.status ?? 1) !== 0) {
|
|
5058
|
+
return { error: `refusing to reuse non-worktree path: ${path}` };
|
|
5059
|
+
}
|
|
5060
|
+
if (!resolvedDirEquals(top.stdout.trim(), path)) {
|
|
5061
|
+
return { error: `existing worktree top-level mismatch for ${path}: ${top.stdout.trim()}` };
|
|
5062
|
+
}
|
|
5063
|
+
const expectedCommon = await commonDir(repoRoot);
|
|
5064
|
+
const actualCommon = await commonDir(path);
|
|
5065
|
+
if (!expectedCommon || expectedCommon !== actualCommon) {
|
|
5066
|
+
return { error: `existing worktree ${path} belongs to a different git common dir` };
|
|
5067
|
+
}
|
|
5068
|
+
const current = await git(["-C", path, "branch", "--show-current"]);
|
|
5069
|
+
const actualBranch = current.stdout.trim();
|
|
5070
|
+
if (current.error || (current.status ?? 1) !== 0 || actualBranch !== branch) {
|
|
5071
|
+
return { error: `existing worktree ${path} is on branch ${actualBranch || "unknown"}, expected ${branch}` };
|
|
5072
|
+
}
|
|
5073
|
+
return { cwd: worktree.cwd };
|
|
5074
|
+
}
|
|
5075
|
+
const inside = await git(["-C", repoRoot, "rev-parse", "--is-inside-work-tree"]);
|
|
5076
|
+
if (inside.error || (inside.status ?? 1) !== 0) {
|
|
5077
|
+
return { error: `worktree repoRoot is not a git repository: ${repoRoot}` };
|
|
5078
|
+
}
|
|
5079
|
+
try {
|
|
5080
|
+
mkdirSync5(dirname4(path), { recursive: true });
|
|
5081
|
+
} catch (error) {
|
|
5082
|
+
return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
|
|
5083
|
+
}
|
|
5084
|
+
const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
5085
|
+
const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
|
|
5086
|
+
if (add.error || (add.status ?? 1) !== 0) {
|
|
5087
|
+
const detail = (add.stderr || add.stdout || add.error || "").toString().trim();
|
|
5088
|
+
return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
|
|
5089
|
+
}
|
|
5090
|
+
return { cwd: worktree.cwd };
|
|
5091
|
+
}
|
|
5092
|
+
async function enterWorktree(spec, opts, env, startedAt) {
|
|
5093
|
+
const worktree = spec.worktree;
|
|
5094
|
+
if (!worktree?.enabled || worktree.mode === "off" || worktree.mode === "main")
|
|
5095
|
+
return;
|
|
5096
|
+
const prepared = await ensureLocalWorktree(worktree, env);
|
|
5097
|
+
if (prepared.error) {
|
|
5098
|
+
if (worktree.mode === "required") {
|
|
5099
|
+
return { failure: failureResult(startedAt, `worktree preparation failed (mode=required): ${prepared.error}`) };
|
|
5100
|
+
}
|
|
5101
|
+
opts.log?.(`worktree preparation failed (mode=${worktree.mode}); falling back to ${worktree.originalCwd}: ${prepared.error}`);
|
|
5102
|
+
spec.cwd = worktree.originalCwd;
|
|
5103
|
+
return { fallbackCwd: worktree.originalCwd };
|
|
5104
|
+
}
|
|
5105
|
+
spec.cwd = prepared.cwd ?? spec.cwd;
|
|
5106
|
+
opts.log?.(`entered worktree ${worktree.path ?? spec.cwd}${worktree.branch ? ` branch ${worktree.branch}` : ""}`);
|
|
5107
|
+
return;
|
|
5108
|
+
}
|
|
5109
|
+
function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
5110
|
+
if (target.type !== "agent")
|
|
5111
|
+
return;
|
|
5112
|
+
const agentTarget = target;
|
|
5113
|
+
const fallbackTarget = {
|
|
5114
|
+
...agentTarget,
|
|
5115
|
+
cwd: fallbackCwd,
|
|
5116
|
+
worktree: agentTarget.worktree ? { ...agentTarget.worktree, enabled: false, cwd: fallbackCwd, reason: "auto worktree preparation failed" } : undefined
|
|
5117
|
+
};
|
|
5118
|
+
return commandSpec(fallbackTarget, opts);
|
|
5119
|
+
}
|
|
4183
5120
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
4184
5121
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
4185
|
-
const result =
|
|
5122
|
+
const result = spawnSync4(plan.command, plan.args, {
|
|
4186
5123
|
encoding: "utf8",
|
|
4187
5124
|
env: transportEnv(opts),
|
|
4188
5125
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -4196,11 +5133,11 @@ function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
|
4196
5133
|
throw new Error(`remote preflight failed on ${machine.id}${detail ? `: ${detail}` : ""}`);
|
|
4197
5134
|
}
|
|
4198
5135
|
}
|
|
4199
|
-
async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
5136
|
+
async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
4200
5137
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
4201
5138
|
const startedAt = nowIso();
|
|
4202
|
-
|
|
4203
|
-
|
|
5139
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
5140
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
4204
5141
|
let timedOut = false;
|
|
4205
5142
|
let idleTimedOut = false;
|
|
4206
5143
|
let exitCode;
|
|
@@ -4209,25 +5146,16 @@ async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
|
4209
5146
|
let script;
|
|
4210
5147
|
try {
|
|
4211
5148
|
plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
4212
|
-
script = remoteScript(spec, metadata);
|
|
5149
|
+
script = remoteScript(spec, metadata, fallbackSpec);
|
|
4213
5150
|
} catch (err) {
|
|
4214
|
-
return
|
|
4215
|
-
status: "failed",
|
|
4216
|
-
stdout: "",
|
|
4217
|
-
stderr: "",
|
|
4218
|
-
error: err instanceof Error ? err.message : String(err),
|
|
4219
|
-
startedAt,
|
|
4220
|
-
finishedAt: nowIso(),
|
|
4221
|
-
durationMs: 0
|
|
4222
|
-
};
|
|
5151
|
+
return failureResult(startedAt, err instanceof Error ? err.message : String(err));
|
|
4223
5152
|
}
|
|
4224
|
-
const child =
|
|
5153
|
+
const child = spawn2(plan.command, plan.args, {
|
|
4225
5154
|
env: transportEnv(opts),
|
|
4226
5155
|
detached: true,
|
|
4227
5156
|
stdio: ["pipe", "pipe", "pipe"]
|
|
4228
5157
|
});
|
|
4229
|
-
|
|
4230
|
-
opts.onSpawn?.(child.pid);
|
|
5158
|
+
notifySpawn(child.pid, opts);
|
|
4231
5159
|
child.stdin?.on("error", (err) => {
|
|
4232
5160
|
if (err.code !== "EPIPE")
|
|
4233
5161
|
error = err.message;
|
|
@@ -4261,12 +5189,14 @@ async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
|
4261
5189
|
idleTimer.unref();
|
|
4262
5190
|
};
|
|
4263
5191
|
resetIdleTimer();
|
|
5192
|
+
child.stdout?.setEncoding("utf8");
|
|
5193
|
+
child.stderr?.setEncoding("utf8");
|
|
4264
5194
|
child.stdout?.on("data", (chunk) => {
|
|
4265
|
-
stdout
|
|
5195
|
+
stdout.append(chunk);
|
|
4266
5196
|
resetIdleTimer();
|
|
4267
5197
|
});
|
|
4268
5198
|
child.stderr?.on("data", (chunk) => {
|
|
4269
|
-
stderr
|
|
5199
|
+
stderr.append(chunk);
|
|
4270
5200
|
resetIdleTimer();
|
|
4271
5201
|
});
|
|
4272
5202
|
try {
|
|
@@ -4284,47 +5214,17 @@ async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
|
4284
5214
|
clearTimeout(idleTimer);
|
|
4285
5215
|
opts.signal?.removeEventListener("abort", abortHandler);
|
|
4286
5216
|
}
|
|
4287
|
-
const
|
|
4288
|
-
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
5217
|
+
const fields = { exitCode, stdout: stdout.value(), stderr: stderr.value(), pid: child.pid };
|
|
4289
5218
|
if (timedOut || idleTimedOut) {
|
|
4290
|
-
return {
|
|
4291
|
-
status: "timed_out",
|
|
4292
|
-
exitCode,
|
|
4293
|
-
stdout,
|
|
4294
|
-
stderr,
|
|
4295
|
-
error: idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`,
|
|
4296
|
-
pid: child.pid,
|
|
4297
|
-
startedAt,
|
|
4298
|
-
finishedAt,
|
|
4299
|
-
durationMs
|
|
4300
|
-
};
|
|
5219
|
+
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
4301
5220
|
}
|
|
4302
5221
|
if (error || exitCode !== 0) {
|
|
4303
|
-
return {
|
|
4304
|
-
status: "failed",
|
|
4305
|
-
exitCode,
|
|
4306
|
-
stdout,
|
|
4307
|
-
stderr,
|
|
4308
|
-
error: error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`,
|
|
4309
|
-
pid: child.pid,
|
|
4310
|
-
startedAt,
|
|
4311
|
-
finishedAt,
|
|
4312
|
-
durationMs
|
|
4313
|
-
};
|
|
5222
|
+
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
4314
5223
|
}
|
|
4315
|
-
return
|
|
4316
|
-
status: "succeeded",
|
|
4317
|
-
exitCode,
|
|
4318
|
-
stdout,
|
|
4319
|
-
stderr,
|
|
4320
|
-
pid: child.pid,
|
|
4321
|
-
startedAt,
|
|
4322
|
-
finishedAt,
|
|
4323
|
-
durationMs
|
|
4324
|
-
};
|
|
5224
|
+
return successResult(startedAt, fields);
|
|
4325
5225
|
}
|
|
4326
5226
|
function preflightTarget(target, metadata = {}, opts = {}) {
|
|
4327
|
-
const spec = commandSpec(target);
|
|
5227
|
+
const spec = commandSpec(target, opts);
|
|
4328
5228
|
const machine = resolvedMachine(opts);
|
|
4329
5229
|
if (machine && !machine.local) {
|
|
4330
5230
|
preflightRemoteSpec(spec, machine, metadata, opts);
|
|
@@ -4334,14 +5234,14 @@ function preflightTarget(target, metadata = {}, opts = {}) {
|
|
|
4334
5234
|
accountTool: spec.accountTool
|
|
4335
5235
|
};
|
|
4336
5236
|
}
|
|
4337
|
-
const env =
|
|
5237
|
+
const env = executionEnvSync(spec, metadata, opts);
|
|
4338
5238
|
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
4339
5239
|
throw new Error(commandNotFoundMessage(spec.command, env));
|
|
4340
5240
|
}
|
|
4341
5241
|
if (spec.preflightAnyOf?.length && !spec.preflightAnyOf.some((command) => executableExists(command, env))) {
|
|
4342
5242
|
throw new Error(`none of required executables found: ${spec.preflightAnyOf.join(", ")}`);
|
|
4343
5243
|
}
|
|
4344
|
-
|
|
5244
|
+
preflightNativeAuthProfileSync(spec, env);
|
|
4345
5245
|
return {
|
|
4346
5246
|
command: spec.command,
|
|
4347
5247
|
accountProfile: spec.account?.profile,
|
|
@@ -4349,79 +5249,52 @@ function preflightTarget(target, metadata = {}, opts = {}) {
|
|
|
4349
5249
|
};
|
|
4350
5250
|
}
|
|
4351
5251
|
async function executeTarget(target, metadata = {}, opts = {}) {
|
|
4352
|
-
|
|
5252
|
+
let spec = commandSpec(target, opts);
|
|
4353
5253
|
const machine = resolvedMachine(opts);
|
|
4354
5254
|
if (machine && !machine.local && spec.codewithDurableAgent) {
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
stderr: "",
|
|
4361
|
-
error: "remote Codewith durable background-agent steps require remote status polling support; run this Codewith step locally or add remote durable readback before dispatch",
|
|
4362
|
-
startedAt: startedAt2,
|
|
4363
|
-
finishedAt: finishedAt2,
|
|
4364
|
-
durationMs: new Date(finishedAt2).getTime() - new Date(startedAt2).getTime()
|
|
4365
|
-
};
|
|
5255
|
+
return failureResult(nowIso(), "remote Codewith durable background-agent steps require remote status polling support; run this Codewith step locally or add remote durable readback before dispatch");
|
|
5256
|
+
}
|
|
5257
|
+
if (machine && !machine.local) {
|
|
5258
|
+
const remoteFallbackSpec = spec.worktree?.enabled && spec.worktree.mode === "auto" ? worktreeFallbackSpec(target, opts, spec.worktree.originalCwd) : undefined;
|
|
5259
|
+
return executeRemoteSpec(spec, machine, metadata, opts, remoteFallbackSpec);
|
|
4366
5260
|
}
|
|
4367
|
-
if (machine && !machine.local)
|
|
4368
|
-
return executeRemoteSpec(spec, machine, metadata, opts);
|
|
4369
5261
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
4370
5262
|
const startedAt = nowIso();
|
|
4371
|
-
|
|
4372
|
-
|
|
5263
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
5264
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
4373
5265
|
let timedOut = false;
|
|
4374
5266
|
let idleTimedOut = false;
|
|
4375
5267
|
let exitCode;
|
|
4376
5268
|
let error;
|
|
4377
|
-
const env = executionEnv(spec, metadata, opts);
|
|
5269
|
+
const env = await executionEnv(spec, metadata, opts);
|
|
4378
5270
|
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
4379
|
-
return
|
|
4380
|
-
status: "failed",
|
|
4381
|
-
stdout: "",
|
|
4382
|
-
stderr: "",
|
|
4383
|
-
error: commandNotFoundMessage(spec.command, env),
|
|
4384
|
-
startedAt,
|
|
4385
|
-
finishedAt: nowIso(),
|
|
4386
|
-
durationMs: 0
|
|
4387
|
-
};
|
|
5271
|
+
return failureResult(startedAt, commandNotFoundMessage(spec.command, env));
|
|
4388
5272
|
}
|
|
4389
5273
|
if (spec.preflightAnyOf?.length && !spec.preflightAnyOf.some((command) => executableExists(command, env))) {
|
|
4390
|
-
return {
|
|
4391
|
-
status: "failed",
|
|
4392
|
-
stdout,
|
|
4393
|
-
stderr,
|
|
4394
|
-
error: `none of required executables found: ${spec.preflightAnyOf.join(", ")}`,
|
|
4395
|
-
startedAt,
|
|
4396
|
-
finishedAt: nowIso(),
|
|
4397
|
-
durationMs: 0
|
|
4398
|
-
};
|
|
5274
|
+
return failureResult(startedAt, `none of required executables found: ${spec.preflightAnyOf.join(", ")}`);
|
|
4399
5275
|
}
|
|
4400
5276
|
try {
|
|
4401
|
-
preflightNativeAuthProfile(spec, env);
|
|
5277
|
+
await preflightNativeAuthProfile(spec, env);
|
|
4402
5278
|
} catch (err) {
|
|
4403
|
-
return
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
durationMs: 0
|
|
4411
|
-
};
|
|
5279
|
+
return failureResult(startedAt, err instanceof Error ? err.message : String(err));
|
|
5280
|
+
}
|
|
5281
|
+
const worktreeEntry = await enterWorktree(spec, opts, env, startedAt);
|
|
5282
|
+
if (worktreeEntry?.failure)
|
|
5283
|
+
return worktreeEntry.failure;
|
|
5284
|
+
if (worktreeEntry?.fallbackCwd) {
|
|
5285
|
+
spec = worktreeFallbackSpec(target, opts, worktreeEntry.fallbackCwd) ?? spec;
|
|
4412
5286
|
}
|
|
4413
5287
|
if (spec.codewithDurableAgent) {
|
|
4414
5288
|
return executeCodewithDurableAgent(spec, metadata, opts, env, startedAt);
|
|
4415
5289
|
}
|
|
4416
|
-
const child =
|
|
5290
|
+
const child = spawn2(spec.command, spec.args, {
|
|
4417
5291
|
cwd: spec.cwd,
|
|
4418
5292
|
env,
|
|
4419
5293
|
shell: spec.shell ?? false,
|
|
4420
5294
|
detached: true,
|
|
4421
5295
|
stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
|
|
4422
5296
|
});
|
|
4423
|
-
|
|
4424
|
-
opts.onSpawn?.(child.pid);
|
|
5297
|
+
notifySpawn(child.pid, opts);
|
|
4425
5298
|
if (spec.stdin !== undefined && child.stdin) {
|
|
4426
5299
|
child.stdin.on("error", (err) => {
|
|
4427
5300
|
if (err.code !== "EPIPE")
|
|
@@ -4457,12 +5330,14 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
4457
5330
|
idleTimer.unref();
|
|
4458
5331
|
};
|
|
4459
5332
|
resetIdleTimer();
|
|
5333
|
+
child.stdout?.setEncoding("utf8");
|
|
5334
|
+
child.stderr?.setEncoding("utf8");
|
|
4460
5335
|
child.stdout?.on("data", (chunk) => {
|
|
4461
|
-
stdout
|
|
5336
|
+
stdout.append(chunk);
|
|
4462
5337
|
resetIdleTimer();
|
|
4463
5338
|
});
|
|
4464
5339
|
child.stderr?.on("data", (chunk) => {
|
|
4465
|
-
stderr
|
|
5340
|
+
stderr.append(chunk);
|
|
4466
5341
|
resetIdleTimer();
|
|
4467
5342
|
});
|
|
4468
5343
|
try {
|
|
@@ -4480,97 +5355,706 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
4480
5355
|
clearTimeout(idleTimer);
|
|
4481
5356
|
opts.signal?.removeEventListener("abort", abortHandler);
|
|
4482
5357
|
}
|
|
4483
|
-
const
|
|
4484
|
-
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
5358
|
+
const fields = { exitCode, stdout: stdout.value(), stderr: stderr.value(), pid: child.pid };
|
|
4485
5359
|
if (timedOut || idleTimedOut) {
|
|
5360
|
+
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5361
|
+
}
|
|
5362
|
+
if (error || exitCode !== 0) {
|
|
5363
|
+
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5364
|
+
}
|
|
5365
|
+
return successResult(startedAt, fields);
|
|
5366
|
+
}
|
|
5367
|
+
async function executeLoop(loop, run, opts = {}) {
|
|
5368
|
+
if (loop.target.type === "workflow") {
|
|
5369
|
+
throw new Error("workflow loop targets must be executed with executeLoopTarget");
|
|
5370
|
+
}
|
|
5371
|
+
if (loop.target.preflight?.beforeRun) {
|
|
5372
|
+
const startedAt = nowIso();
|
|
5373
|
+
try {
|
|
5374
|
+
preflightTarget(loop.target, {
|
|
5375
|
+
loopId: loop.id,
|
|
5376
|
+
loopName: loop.name,
|
|
5377
|
+
runId: run.id,
|
|
5378
|
+
scheduledFor: run.scheduledFor
|
|
5379
|
+
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5380
|
+
} catch (error) {
|
|
5381
|
+
return failureResult(startedAt, `runtime preflight failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
5382
|
+
}
|
|
5383
|
+
}
|
|
5384
|
+
return executeTarget(loop.target, {
|
|
5385
|
+
loopId: loop.id,
|
|
5386
|
+
loopName: loop.name,
|
|
5387
|
+
runId: run.id,
|
|
5388
|
+
scheduledFor: run.scheduledFor
|
|
5389
|
+
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5390
|
+
}
|
|
5391
|
+
|
|
5392
|
+
// src/lib/doctor.ts
|
|
5393
|
+
var PROVIDER_COMMANDS = [
|
|
5394
|
+
"claude",
|
|
5395
|
+
"agent",
|
|
5396
|
+
"codewith",
|
|
5397
|
+
"aicopilot",
|
|
5398
|
+
"opencode",
|
|
5399
|
+
"codex"
|
|
5400
|
+
];
|
|
5401
|
+
function hasCommand(command) {
|
|
5402
|
+
const result = spawnSync5("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
5403
|
+
return (result.status ?? 1) === 0;
|
|
5404
|
+
}
|
|
5405
|
+
function commandVersion(command) {
|
|
5406
|
+
const result = spawnSync5(command, ["--version"], {
|
|
5407
|
+
encoding: "utf8",
|
|
5408
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
5409
|
+
});
|
|
5410
|
+
if ((result.status ?? 1) !== 0)
|
|
5411
|
+
return;
|
|
5412
|
+
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
5413
|
+
}
|
|
5414
|
+
function runDoctor(store) {
|
|
5415
|
+
const checks = [];
|
|
5416
|
+
try {
|
|
5417
|
+
const dir = ensureDataDir();
|
|
5418
|
+
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
5419
|
+
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
5420
|
+
} catch (error) {
|
|
5421
|
+
checks.push({
|
|
5422
|
+
id: "data-dir",
|
|
5423
|
+
status: "fail",
|
|
5424
|
+
message: "data directory is not writable",
|
|
5425
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
5426
|
+
});
|
|
5427
|
+
}
|
|
5428
|
+
const bunVersion = commandVersion("bun");
|
|
5429
|
+
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
5430
|
+
const accountsVersion = commandVersion("accounts");
|
|
5431
|
+
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
5432
|
+
try {
|
|
5433
|
+
const machines = listOpenMachines();
|
|
5434
|
+
const local = machines.find((machine) => machine.local);
|
|
5435
|
+
checks.push({
|
|
5436
|
+
id: "machines",
|
|
5437
|
+
status: "ok",
|
|
5438
|
+
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
5439
|
+
detail: local ? `local=${local.id}` : undefined
|
|
5440
|
+
});
|
|
5441
|
+
} catch (error) {
|
|
5442
|
+
checks.push({
|
|
5443
|
+
id: "machines",
|
|
5444
|
+
status: "warn",
|
|
5445
|
+
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
5446
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
5447
|
+
});
|
|
5448
|
+
}
|
|
5449
|
+
for (const command of PROVIDER_COMMANDS) {
|
|
5450
|
+
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
5451
|
+
}
|
|
5452
|
+
const status = daemonStatus(store);
|
|
5453
|
+
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
5454
|
+
const failedRuns = store.countRuns("failed");
|
|
5455
|
+
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
5456
|
+
for (const loop of store.listLoops({ status: "active" })) {
|
|
5457
|
+
try {
|
|
5458
|
+
if (loop.target.type === "workflow") {
|
|
5459
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
5460
|
+
for (const step of workflowExecutionOrder(workflow)) {
|
|
5461
|
+
preflightTarget({
|
|
5462
|
+
...step.target,
|
|
5463
|
+
account: step.account ?? step.target.account,
|
|
5464
|
+
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
5465
|
+
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
5466
|
+
}
|
|
5467
|
+
} else {
|
|
5468
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
5469
|
+
}
|
|
5470
|
+
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
5471
|
+
} catch (error) {
|
|
5472
|
+
checks.push({
|
|
5473
|
+
id: `loop:${loop.id}:preflight`,
|
|
5474
|
+
status: "fail",
|
|
5475
|
+
message: `active loop target preflight failed: ${loop.name}`,
|
|
5476
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
5477
|
+
});
|
|
5478
|
+
}
|
|
5479
|
+
}
|
|
5480
|
+
return {
|
|
5481
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
5482
|
+
checks
|
|
5483
|
+
};
|
|
5484
|
+
}
|
|
5485
|
+
|
|
5486
|
+
// src/lib/health.ts
|
|
5487
|
+
import { createHash as createHash2 } from "crypto";
|
|
5488
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
|
|
5489
|
+
|
|
5490
|
+
// src/lib/format.ts
|
|
5491
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
5492
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
5493
|
+
function redact(value, visible = 0) {
|
|
5494
|
+
if (!value)
|
|
5495
|
+
return value;
|
|
5496
|
+
if (value.length <= visible)
|
|
5497
|
+
return value;
|
|
5498
|
+
if (visible <= 0)
|
|
5499
|
+
return `[redacted ${value.length} chars]`;
|
|
5500
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
5501
|
+
}
|
|
5502
|
+
function truncateTextOutput(value) {
|
|
5503
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
5504
|
+
return value;
|
|
5505
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
5506
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
5507
|
+
}
|
|
5508
|
+
function redactSensitivePayload(value, key) {
|
|
5509
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
5510
|
+
if (typeof value === "string")
|
|
5511
|
+
return redact(value);
|
|
5512
|
+
if (value === undefined || value === null)
|
|
5513
|
+
return value;
|
|
5514
|
+
return "[redacted]";
|
|
5515
|
+
}
|
|
5516
|
+
if (Array.isArray(value))
|
|
5517
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
5518
|
+
if (value && typeof value === "object") {
|
|
5519
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
5520
|
+
}
|
|
5521
|
+
return value;
|
|
5522
|
+
}
|
|
5523
|
+
function textOutputBlocks(value, opts = {}) {
|
|
5524
|
+
const indent = opts.indent ?? "";
|
|
5525
|
+
const nested = `${indent} `;
|
|
5526
|
+
const blocks = [];
|
|
5527
|
+
for (const [label, output] of [
|
|
5528
|
+
["stdout", value.stdout],
|
|
5529
|
+
["stderr", value.stderr]
|
|
5530
|
+
]) {
|
|
5531
|
+
if (!output)
|
|
5532
|
+
continue;
|
|
5533
|
+
blocks.push(`${indent}${label}:`);
|
|
5534
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
5535
|
+
blocks.push(`${nested}${line}`);
|
|
5536
|
+
}
|
|
5537
|
+
}
|
|
5538
|
+
return blocks;
|
|
5539
|
+
}
|
|
5540
|
+
function publicLoop(loop) {
|
|
5541
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
5542
|
+
return {
|
|
5543
|
+
...loop,
|
|
5544
|
+
target
|
|
5545
|
+
};
|
|
5546
|
+
}
|
|
5547
|
+
function publicRun(run, showOutput = false) {
|
|
5548
|
+
return {
|
|
5549
|
+
...run,
|
|
5550
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
5551
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined
|
|
5552
|
+
};
|
|
5553
|
+
}
|
|
5554
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
5555
|
+
return {
|
|
5556
|
+
...result,
|
|
5557
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
5558
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
5559
|
+
error: redact(result.error)
|
|
5560
|
+
};
|
|
5561
|
+
}
|
|
5562
|
+
function publicWorkflow(workflow) {
|
|
5563
|
+
return {
|
|
5564
|
+
...workflow,
|
|
5565
|
+
steps: workflow.steps.map((step) => ({
|
|
5566
|
+
...step,
|
|
5567
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
5568
|
+
}))
|
|
5569
|
+
};
|
|
5570
|
+
}
|
|
5571
|
+
function publicWorkflowRun(run) {
|
|
5572
|
+
return { ...run, error: redact(run.error) };
|
|
5573
|
+
}
|
|
5574
|
+
function publicWorkflowInvocation(invocation) {
|
|
5575
|
+
return redactSensitivePayload(invocation);
|
|
5576
|
+
}
|
|
5577
|
+
function publicWorkflowWorkItem(item) {
|
|
5578
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
5579
|
+
}
|
|
5580
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
5581
|
+
return {
|
|
5582
|
+
...run,
|
|
5583
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
5584
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
5585
|
+
error: redact(run.error)
|
|
5586
|
+
};
|
|
5587
|
+
}
|
|
5588
|
+
function publicWorkflowEvent(event) {
|
|
5589
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
5590
|
+
}
|
|
5591
|
+
function publicGoal(goal) {
|
|
5592
|
+
return {
|
|
5593
|
+
...goal,
|
|
5594
|
+
objective: redact(goal.objective, 120)
|
|
5595
|
+
};
|
|
5596
|
+
}
|
|
5597
|
+
function publicGoalRun(run) {
|
|
5598
|
+
return {
|
|
5599
|
+
...run,
|
|
5600
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
5601
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
5602
|
+
};
|
|
5603
|
+
}
|
|
5604
|
+
|
|
5605
|
+
// src/lib/health.ts
|
|
5606
|
+
var EVIDENCE_CHARS = 2000;
|
|
5607
|
+
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
5608
|
+
var CLASSIFICATIONS = [
|
|
5609
|
+
"rate_limit",
|
|
5610
|
+
"auth",
|
|
5611
|
+
"model_not_found",
|
|
5612
|
+
"context_length",
|
|
5613
|
+
"schema_response_format",
|
|
5614
|
+
"node_init",
|
|
5615
|
+
"preflight",
|
|
5616
|
+
"route_functional",
|
|
5617
|
+
"timeout",
|
|
5618
|
+
"sigsegv",
|
|
5619
|
+
"skipped_previous_active",
|
|
5620
|
+
"circuit_breaker",
|
|
5621
|
+
"unknown"
|
|
5622
|
+
];
|
|
5623
|
+
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
5624
|
+
if (!value)
|
|
5625
|
+
return;
|
|
5626
|
+
if (value.length <= limit)
|
|
5627
|
+
return value;
|
|
5628
|
+
return `${value.slice(0, limit)}
|
|
5629
|
+
[truncated ${value.length - limit} chars]`;
|
|
5630
|
+
}
|
|
5631
|
+
function redactedEvidence(value) {
|
|
5632
|
+
return redact(bounded(value));
|
|
5633
|
+
}
|
|
5634
|
+
function searchableText(run) {
|
|
5635
|
+
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
5636
|
+
`).toLowerCase();
|
|
5637
|
+
}
|
|
5638
|
+
function isRecord(value) {
|
|
5639
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
5640
|
+
}
|
|
5641
|
+
function stringValue(value) {
|
|
5642
|
+
return typeof value === "string" && value.trim() ? value : undefined;
|
|
5643
|
+
}
|
|
5644
|
+
function objectField(value, key) {
|
|
5645
|
+
if (!value)
|
|
5646
|
+
return;
|
|
5647
|
+
const field = value[key];
|
|
5648
|
+
return isRecord(field) ? field : undefined;
|
|
5649
|
+
}
|
|
5650
|
+
function tagsFromValue(value) {
|
|
5651
|
+
if (Array.isArray(value))
|
|
5652
|
+
return value.map((entry) => String(entry).trim()).filter(Boolean);
|
|
5653
|
+
if (typeof value === "string")
|
|
5654
|
+
return value.split(",").map((entry) => entry.trim()).filter(Boolean);
|
|
5655
|
+
return [];
|
|
5656
|
+
}
|
|
5657
|
+
function stableFingerprint(parts) {
|
|
5658
|
+
return createHash2("sha256").update(parts.join(`
|
|
5659
|
+
`)).digest("hex").slice(0, 16);
|
|
5660
|
+
}
|
|
5661
|
+
function stableFailureFingerprint(run, classification) {
|
|
5662
|
+
return stableFingerprint([
|
|
5663
|
+
run.loopId,
|
|
5664
|
+
classification,
|
|
5665
|
+
String(run.status),
|
|
5666
|
+
String(run.exitCode ?? ""),
|
|
5667
|
+
(run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
5668
|
+
]);
|
|
5669
|
+
}
|
|
5670
|
+
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
5671
|
+
return stableFingerprint([
|
|
5672
|
+
loop.id,
|
|
5673
|
+
"route_functional",
|
|
5674
|
+
reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
5675
|
+
]);
|
|
5676
|
+
}
|
|
5677
|
+
function healthRun(run) {
|
|
5678
|
+
return {
|
|
5679
|
+
...run,
|
|
5680
|
+
error: redactedEvidence(run.error),
|
|
5681
|
+
stdout: redactedEvidence(run.stdout),
|
|
5682
|
+
stderr: redactedEvidence(run.stderr)
|
|
5683
|
+
};
|
|
5684
|
+
}
|
|
5685
|
+
function classifyRunFailure(run) {
|
|
5686
|
+
if (run.status === "succeeded" || run.status === "running")
|
|
5687
|
+
return;
|
|
5688
|
+
const text = searchableText(run);
|
|
5689
|
+
let classification = "unknown";
|
|
5690
|
+
if (run.status === "timed_out")
|
|
5691
|
+
classification = "timeout";
|
|
5692
|
+
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
5693
|
+
classification = "circuit_breaker";
|
|
5694
|
+
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
5695
|
+
classification = "skipped_previous_active";
|
|
5696
|
+
else if (/runtime preflight failed|preflight failed|executable not found in path|none of required executables found|auth profile preflight failed|profile not found/.test(text))
|
|
5697
|
+
classification = "preflight";
|
|
5698
|
+
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
5699
|
+
classification = "rate_limit";
|
|
5700
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
5701
|
+
classification = "auth";
|
|
5702
|
+
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
5703
|
+
classification = "model_not_found";
|
|
5704
|
+
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
5705
|
+
classification = "context_length";
|
|
5706
|
+
else if (/response_format|json schema|schema validation|invalid schema|structured output/.test(text))
|
|
5707
|
+
classification = "schema_response_format";
|
|
5708
|
+
else if (/cannot find module|module not found|node:internal|bun: command not found|node: command not found|npm err!|err_module_not_found/.test(text))
|
|
5709
|
+
classification = "node_init";
|
|
5710
|
+
else if (/sigsegv|segmentation fault|signal 11/.test(text))
|
|
5711
|
+
classification = "sigsegv";
|
|
5712
|
+
return {
|
|
5713
|
+
classification,
|
|
5714
|
+
fingerprint: stableFailureFingerprint(run, classification),
|
|
5715
|
+
evidence: {
|
|
5716
|
+
error: redactedEvidence(run.error),
|
|
5717
|
+
stdout: redactedEvidence(run.stdout),
|
|
5718
|
+
stderr: redactedEvidence(run.stderr),
|
|
5719
|
+
exitCode: run.exitCode
|
|
5720
|
+
}
|
|
5721
|
+
};
|
|
5722
|
+
}
|
|
5723
|
+
var ROUTE_FUNCTIONAL_DISALLOWED_TAGS = new Set([
|
|
5724
|
+
"no-auto",
|
|
5725
|
+
"manual",
|
|
5726
|
+
"manual-required",
|
|
5727
|
+
"approval-required",
|
|
5728
|
+
"blocked",
|
|
5729
|
+
"completed",
|
|
5730
|
+
"done",
|
|
5731
|
+
"cancelled",
|
|
5732
|
+
"canceled",
|
|
5733
|
+
"failed",
|
|
5734
|
+
"archived"
|
|
5735
|
+
]);
|
|
5736
|
+
var ROUTE_FUNCTIONAL_DISALLOWED_STATUSES = new Set([
|
|
5737
|
+
"blocked",
|
|
5738
|
+
"completed",
|
|
5739
|
+
"done",
|
|
5740
|
+
"cancelled",
|
|
5741
|
+
"canceled",
|
|
5742
|
+
"failed",
|
|
5743
|
+
"archived"
|
|
5744
|
+
]);
|
|
5745
|
+
function parseJsonObject(raw) {
|
|
5746
|
+
if (!raw?.trim())
|
|
5747
|
+
return;
|
|
5748
|
+
try {
|
|
5749
|
+
const parsed = JSON.parse(raw);
|
|
5750
|
+
return isRecord(parsed) ? parsed : undefined;
|
|
5751
|
+
} catch {
|
|
5752
|
+
return;
|
|
5753
|
+
}
|
|
5754
|
+
}
|
|
5755
|
+
function routeEvidenceReport(run) {
|
|
5756
|
+
const stdoutReport = parseJsonObject(run.stdout);
|
|
5757
|
+
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
5758
|
+
if (evidencePath && existsSync4(evidencePath)) {
|
|
5759
|
+
try {
|
|
5760
|
+
return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
|
|
5761
|
+
} catch {
|
|
5762
|
+
return stdoutReport;
|
|
5763
|
+
}
|
|
5764
|
+
}
|
|
5765
|
+
return stdoutReport;
|
|
5766
|
+
}
|
|
5767
|
+
function commandName(command) {
|
|
5768
|
+
return command.split(/[\\/]/).at(-1) ?? command;
|
|
5769
|
+
}
|
|
5770
|
+
function argsContainSequence(args, sequence) {
|
|
5771
|
+
for (let index = 0;index <= args.length - sequence.length; index += 1) {
|
|
5772
|
+
if (sequence.every((part, offset) => args[index + offset] === part))
|
|
5773
|
+
return true;
|
|
5774
|
+
}
|
|
5775
|
+
return false;
|
|
5776
|
+
}
|
|
5777
|
+
function isRouteDrainLoop(loop) {
|
|
5778
|
+
if (loop.target.type !== "command")
|
|
5779
|
+
return false;
|
|
5780
|
+
if (commandName(loop.target.command) !== "loops")
|
|
5781
|
+
return false;
|
|
5782
|
+
const args = loop.target.args ?? [];
|
|
5783
|
+
return argsContainSequence(args, ["events", "drain", "todos-task"]) || argsContainSequence(args, ["routes", "drain", "todos-task"]) || argsContainSequence(args, ["route", "drain", "todos-task"]);
|
|
5784
|
+
}
|
|
5785
|
+
function routeResultTaskState(result) {
|
|
5786
|
+
const event = objectField(result, "event");
|
|
5787
|
+
const data = objectField(event, "data");
|
|
5788
|
+
const task = objectField(data, "task");
|
|
5789
|
+
const payload = objectField(data, "payload");
|
|
5790
|
+
const payloadTask = objectField(payload, "task");
|
|
5791
|
+
const metadata = objectField(data, "metadata");
|
|
5792
|
+
const records = [data, task, payload, payloadTask, metadata].filter(isRecord);
|
|
5793
|
+
const tags = new Set;
|
|
5794
|
+
for (const record of records) {
|
|
5795
|
+
for (const tag of tagsFromValue(record.tags ?? record.task_tags ?? record.taskTags)) {
|
|
5796
|
+
tags.add(tag.toLowerCase());
|
|
5797
|
+
}
|
|
5798
|
+
}
|
|
5799
|
+
const status = records.map((record) => stringValue(record.status ?? record.task_status ?? record.taskStatus)?.toLowerCase()).find(Boolean);
|
|
5800
|
+
return {
|
|
5801
|
+
taskId: stringValue(event?.subject) ?? stringValue(data?.id) ?? stringValue(task?.id) ?? stringValue(payloadTask?.id),
|
|
5802
|
+
tags: [...tags],
|
|
5803
|
+
status
|
|
5804
|
+
};
|
|
5805
|
+
}
|
|
5806
|
+
function detectRouteFunctionalFailure(store, loop, run) {
|
|
5807
|
+
if (run.status !== "succeeded")
|
|
5808
|
+
return;
|
|
5809
|
+
if (!isRouteDrainLoop(loop))
|
|
5810
|
+
return;
|
|
5811
|
+
const report = routeEvidenceReport(run);
|
|
5812
|
+
const rawResults = Array.isArray(report?.results) ? report.results.filter(isRecord) : [];
|
|
5813
|
+
for (const result of rawResults) {
|
|
5814
|
+
const kind = stringValue(result.kind);
|
|
5815
|
+
const task = routeResultTaskState(result);
|
|
5816
|
+
const disallowedTag = task.tags.find((tag) => ROUTE_FUNCTIONAL_DISALLOWED_TAGS.has(tag));
|
|
5817
|
+
if (kind && kind !== "skipped" && disallowedTag) {
|
|
5818
|
+
const reason2 = `route drain ${kind} task ${task.taskId ?? "unknown"} with disallowed tag ${disallowedTag}`;
|
|
5819
|
+
return {
|
|
5820
|
+
classification: "route_functional",
|
|
5821
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, reason2),
|
|
5822
|
+
evidence: { error: reason2, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5823
|
+
};
|
|
5824
|
+
}
|
|
5825
|
+
if (kind && kind !== "skipped" && task.status && ROUTE_FUNCTIONAL_DISALLOWED_STATUSES.has(task.status)) {
|
|
5826
|
+
const reason2 = `route drain ${kind} task ${task.taskId ?? "unknown"} with non-routable status ${task.status}`;
|
|
5827
|
+
return {
|
|
5828
|
+
classification: "route_functional",
|
|
5829
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, reason2),
|
|
5830
|
+
evidence: { error: reason2, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5831
|
+
};
|
|
5832
|
+
}
|
|
5833
|
+
const reason = stringValue(result.reason);
|
|
5834
|
+
if (kind === "skipped" && reason === "task metadata requires manual or approval-gated handling") {
|
|
5835
|
+
const message = `route drain skipped task ${task.taskId ?? "unknown"} with ambiguous manual-gate reason`;
|
|
5836
|
+
return {
|
|
5837
|
+
classification: "route_functional",
|
|
5838
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, message),
|
|
5839
|
+
evidence: { error: message, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5840
|
+
};
|
|
5841
|
+
}
|
|
5842
|
+
const sourceTaskUpdate = objectField(result, "sourceTaskUpdate");
|
|
5843
|
+
if (kind === "skipped" && sourceTaskUpdate && sourceTaskUpdate.ok === false) {
|
|
5844
|
+
const updateError = stringValue(sourceTaskUpdate.error);
|
|
5845
|
+
const message = `route drain skipped task ${task.taskId ?? "unknown"} but failed to update source task${updateError ? `: ${updateError}` : ""}`;
|
|
5846
|
+
return {
|
|
5847
|
+
classification: "route_functional",
|
|
5848
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, message),
|
|
5849
|
+
evidence: { error: message, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5850
|
+
};
|
|
5851
|
+
}
|
|
5852
|
+
const childLoopId = stringValue(objectField(result, "loop")?.id) ?? stringValue(result.loopId);
|
|
5853
|
+
const childRun = childLoopId ? store.listRuns({ loopId: childLoopId, limit: 1 })[0] : undefined;
|
|
5854
|
+
if (childRun && !["succeeded", "running"].includes(childRun.status)) {
|
|
5855
|
+
const message = `route drain ${kind ?? "handled"} task ${task.taskId ?? "unknown"} but child loop ${childLoopId} latest run is ${childRun.status}`;
|
|
5856
|
+
return {
|
|
5857
|
+
classification: "route_functional",
|
|
5858
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, message),
|
|
5859
|
+
evidence: { error: message, stdout: redactedEvidence(run.stdout), stderr: redactedEvidence(childRun.stderr), exitCode: childRun.exitCode }
|
|
5860
|
+
};
|
|
5861
|
+
}
|
|
5862
|
+
}
|
|
5863
|
+
return;
|
|
5864
|
+
}
|
|
5865
|
+
function targetRoute(loop) {
|
|
5866
|
+
if (loop.target.type === "agent") {
|
|
4486
5867
|
return {
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4493
|
-
startedAt,
|
|
4494
|
-
finishedAt,
|
|
4495
|
-
durationMs
|
|
5868
|
+
source: "openloops",
|
|
5869
|
+
kind: "loop_expectation",
|
|
5870
|
+
loopId: loop.id,
|
|
5871
|
+
loopName: loop.name,
|
|
5872
|
+
cwd: loop.target.cwd,
|
|
5873
|
+
provider: loop.target.provider
|
|
4496
5874
|
};
|
|
4497
5875
|
}
|
|
4498
|
-
if (
|
|
5876
|
+
if (loop.target.type === "command") {
|
|
4499
5877
|
return {
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
pid: child.pid,
|
|
4506
|
-
startedAt,
|
|
4507
|
-
finishedAt,
|
|
4508
|
-
durationMs
|
|
5878
|
+
source: "openloops",
|
|
5879
|
+
kind: "loop_expectation",
|
|
5880
|
+
loopId: loop.id,
|
|
5881
|
+
loopName: loop.name,
|
|
5882
|
+
cwd: loop.target.cwd
|
|
4509
5883
|
};
|
|
4510
5884
|
}
|
|
4511
5885
|
return {
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
pid: child.pid,
|
|
4517
|
-
startedAt,
|
|
4518
|
-
finishedAt,
|
|
4519
|
-
durationMs
|
|
5886
|
+
source: "openloops",
|
|
5887
|
+
kind: "loop_expectation",
|
|
5888
|
+
loopId: loop.id,
|
|
5889
|
+
loopName: loop.name
|
|
4520
5890
|
};
|
|
4521
5891
|
}
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
5892
|
+
function recommendedTask(loop, run, failure, route) {
|
|
5893
|
+
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
5894
|
+
const description = [
|
|
5895
|
+
`OpenLoops expectation failed for loop ${loop.name} (${loop.id}).`,
|
|
5896
|
+
`Run: ${run.id}`,
|
|
5897
|
+
`Status: ${run.status}`,
|
|
5898
|
+
`Classification: ${failure.classification}`,
|
|
5899
|
+
`Fingerprint: ${failure.fingerprint}`,
|
|
5900
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
5901
|
+
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
5902
|
+
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
5903
|
+
failure.evidence.error ? `Error:
|
|
5904
|
+
${failure.evidence.error}` : undefined,
|
|
5905
|
+
failure.evidence.stderr ? `Stderr:
|
|
5906
|
+
${failure.evidence.stderr}` : undefined
|
|
5907
|
+
].filter(Boolean).join(`
|
|
5908
|
+
|
|
5909
|
+
`);
|
|
5910
|
+
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
5911
|
+
const tags = ["bug", "openloops", "loop-health", failure.classification];
|
|
5912
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
5913
|
+
return {
|
|
5914
|
+
title,
|
|
5915
|
+
description,
|
|
5916
|
+
priority,
|
|
5917
|
+
tags,
|
|
5918
|
+
dedupeKey,
|
|
5919
|
+
search: { query: dedupeKey },
|
|
5920
|
+
compatibilityFallback: {
|
|
5921
|
+
search: ["todos", "search", dedupeKey, "--json"],
|
|
5922
|
+
add: ["todos", "add", title, "--description", description, "--tag", tags.join(","), "--priority", priority],
|
|
5923
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
5924
|
+
},
|
|
5925
|
+
futureNativeUpsert: {
|
|
5926
|
+
command: "todos upsert",
|
|
5927
|
+
fields: {
|
|
5928
|
+
title,
|
|
5929
|
+
description,
|
|
5930
|
+
priority,
|
|
5931
|
+
tags,
|
|
5932
|
+
dedupeKey,
|
|
5933
|
+
routeSource: route.source,
|
|
5934
|
+
routeKind: route.kind,
|
|
5935
|
+
routeLoopId: route.loopId,
|
|
5936
|
+
routeLoopName: route.loopName
|
|
5937
|
+
}
|
|
5938
|
+
}
|
|
5939
|
+
};
|
|
5940
|
+
}
|
|
5941
|
+
function expectationForLoop(store, loop) {
|
|
5942
|
+
const latestRun = store.listRuns({ loopId: loop.id, limit: 1 })[0];
|
|
5943
|
+
const route = targetRoute(loop);
|
|
5944
|
+
if (!latestRun) {
|
|
5945
|
+
return {
|
|
5946
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5947
|
+
ok: true,
|
|
5948
|
+
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
5949
|
+
route
|
|
5950
|
+
};
|
|
4525
5951
|
}
|
|
4526
|
-
|
|
4527
|
-
|
|
4528
|
-
|
|
4529
|
-
|
|
4530
|
-
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
5952
|
+
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
5953
|
+
if (routeFailure) {
|
|
5954
|
+
return {
|
|
5955
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5956
|
+
ok: false,
|
|
5957
|
+
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
5958
|
+
latestRun: healthRun(latestRun),
|
|
5959
|
+
failure: routeFailure,
|
|
5960
|
+
route,
|
|
5961
|
+
recommendedTask: recommendedTask(loop, latestRun, routeFailure, route)
|
|
5962
|
+
};
|
|
5963
|
+
}
|
|
5964
|
+
if (latestRun.status === "succeeded") {
|
|
5965
|
+
return {
|
|
5966
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5967
|
+
ok: true,
|
|
5968
|
+
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
5969
|
+
latestRun: healthRun(latestRun),
|
|
5970
|
+
route
|
|
5971
|
+
};
|
|
5972
|
+
}
|
|
5973
|
+
const failure = classifyRunFailure(latestRun);
|
|
5974
|
+
if (failure?.classification === "circuit_breaker") {
|
|
5975
|
+
if (loop.status !== "paused") {
|
|
4537
5976
|
return {
|
|
4538
|
-
status:
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
finishedAt,
|
|
4544
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
5977
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5978
|
+
ok: true,
|
|
5979
|
+
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
5980
|
+
latestRun: healthRun(latestRun),
|
|
5981
|
+
route
|
|
4545
5982
|
};
|
|
4546
5983
|
}
|
|
5984
|
+
return {
|
|
5985
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5986
|
+
ok: false,
|
|
5987
|
+
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
5988
|
+
latestRun: healthRun(latestRun),
|
|
5989
|
+
failure,
|
|
5990
|
+
route,
|
|
5991
|
+
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
5992
|
+
};
|
|
4547
5993
|
}
|
|
4548
|
-
return
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
5994
|
+
return {
|
|
5995
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5996
|
+
ok: false,
|
|
5997
|
+
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
5998
|
+
latestRun: healthRun(latestRun),
|
|
5999
|
+
failure,
|
|
6000
|
+
route,
|
|
6001
|
+
recommendedTask: failure ? recommendedTask(loop, latestRun, failure, route) : undefined
|
|
6002
|
+
};
|
|
4554
6003
|
}
|
|
4555
|
-
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
|
|
4563
|
-
function resolveGoalModel(opts = {}) {
|
|
4564
|
-
const env = opts.env ?? process.env;
|
|
4565
|
-
const apiKey = opts.apiKey ?? env.OPENROUTER_API_KEY;
|
|
4566
|
-
if (!apiKey) {
|
|
4567
|
-
throw new Error("OPENROUTER_API_KEY is required to run goals with the OpenRouter AI SDK provider");
|
|
6004
|
+
function buildHealthReport(store, opts = {}) {
|
|
6005
|
+
const loops = store.listLoops({ includeArchived: opts.includeArchived, limit: opts.limit ?? 200 }).filter((loop) => opts.includeInactive || loop.status === "active" || loop.status === "paused");
|
|
6006
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
6007
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
6008
|
+
for (const expectation of expectations) {
|
|
6009
|
+
if (expectation.failure)
|
|
6010
|
+
classifications[expectation.failure.classification] += 1;
|
|
4568
6011
|
}
|
|
4569
|
-
const
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
6012
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
6013
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
6014
|
+
return {
|
|
6015
|
+
ok: unhealthy === 0,
|
|
6016
|
+
generatedAt: new Date().toISOString(),
|
|
6017
|
+
summary: {
|
|
6018
|
+
loops: expectations.length,
|
|
6019
|
+
healthy: expectations.length - unhealthy,
|
|
6020
|
+
unhealthy,
|
|
6021
|
+
warnings
|
|
6022
|
+
},
|
|
6023
|
+
classifications,
|
|
6024
|
+
expectations
|
|
6025
|
+
};
|
|
6026
|
+
}
|
|
6027
|
+
|
|
6028
|
+
// src/lib/goal/metadata.ts
|
|
6029
|
+
function goalExecutionContext(parts) {
|
|
6030
|
+
return {
|
|
6031
|
+
loopId: parts.loop?.id,
|
|
6032
|
+
loopName: parts.loop?.name,
|
|
6033
|
+
loopRunId: parts.loopRun?.id,
|
|
6034
|
+
scheduledFor: parts.loopRun?.scheduledFor ?? parts.scheduledFor,
|
|
6035
|
+
workflowId: parts.workflow?.id,
|
|
6036
|
+
workflowName: parts.workflow?.name,
|
|
6037
|
+
workflowRunId: parts.workflowRunId,
|
|
6038
|
+
workflowStepId: parts.workflowStepId
|
|
6039
|
+
};
|
|
6040
|
+
}
|
|
6041
|
+
function executionMetadata(context, goal, node) {
|
|
6042
|
+
return {
|
|
6043
|
+
loopId: context?.loopId,
|
|
6044
|
+
loopName: context?.loopName,
|
|
6045
|
+
runId: context?.loopRunId,
|
|
6046
|
+
scheduledFor: context?.scheduledFor,
|
|
6047
|
+
workflowId: context?.workflowId,
|
|
6048
|
+
workflowName: context?.workflowName,
|
|
6049
|
+
workflowRunId: context?.workflowRunId,
|
|
6050
|
+
workflowStepId: context?.workflowStepId,
|
|
6051
|
+
goalId: goal?.goalId,
|
|
6052
|
+
goalObjective: goal?.objective,
|
|
6053
|
+
goalNodeKey: node?.key
|
|
6054
|
+
};
|
|
6055
|
+
}
|
|
6056
|
+
function withGoalNodeEnv(env, node) {
|
|
6057
|
+
return { ...env ?? process.env, LOOPS_GOAL_NODE_KEY: node.key, LOOPS_GOAL_NODE_OBJECTIVE: node.objective };
|
|
4574
6058
|
}
|
|
4575
6059
|
|
|
4576
6060
|
// src/lib/goal/prompts.ts
|
|
@@ -4586,6 +6070,23 @@ function planPrompt(spec) {
|
|
|
4586
6070
|
].join(`
|
|
4587
6071
|
`);
|
|
4588
6072
|
}
|
|
6073
|
+
function iterationPrompt(goal, node) {
|
|
6074
|
+
return [
|
|
6075
|
+
"Continue executing the goal plan without losing budget discipline.",
|
|
6076
|
+
`Goal: ${goal.objective}`,
|
|
6077
|
+
`Plan node: ${node.key}`,
|
|
6078
|
+
`Node objective: ${node.objective}`,
|
|
6079
|
+
"Work only on this node's objective in this run; other plan nodes run in their own iterations."
|
|
6080
|
+
].join(`
|
|
6081
|
+
`);
|
|
6082
|
+
}
|
|
6083
|
+
function goalNodeTarget(target, goal, node) {
|
|
6084
|
+
if (target.type !== "agent")
|
|
6085
|
+
return target;
|
|
6086
|
+
return { ...target, prompt: `${target.prompt}
|
|
6087
|
+
|
|
6088
|
+
${iterationPrompt(goal, node)}` };
|
|
6089
|
+
}
|
|
4589
6090
|
function achievementPrompt(goal, nodes, evidence) {
|
|
4590
6091
|
return [
|
|
4591
6092
|
"Run an adversarial achievement audit.",
|
|
@@ -4605,6 +6106,79 @@ ${evidence.join(`
|
|
|
4605
6106
|
`);
|
|
4606
6107
|
}
|
|
4607
6108
|
|
|
6109
|
+
// src/lib/goal/runner.ts
|
|
6110
|
+
import { generateObject } from "ai";
|
|
6111
|
+
import { z } from "zod";
|
|
6112
|
+
|
|
6113
|
+
// src/lib/run-envelope.ts
|
|
6114
|
+
var ENVELOPE_EXCERPT_CHARS = 2048;
|
|
6115
|
+
var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
|
|
6116
|
+
function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
|
|
6117
|
+
if (!text)
|
|
6118
|
+
return;
|
|
6119
|
+
const scrubbed = scrubSecrets(text);
|
|
6120
|
+
if (scrubbed.length <= limit * 2)
|
|
6121
|
+
return scrubbed;
|
|
6122
|
+
const omitted = scrubbed.length - limit * 2;
|
|
6123
|
+
return `${scrubbed.slice(0, limit)}
|
|
6124
|
+
[... ${omitted} chars omitted ...]
|
|
6125
|
+
${scrubbed.slice(-limit)}`;
|
|
6126
|
+
}
|
|
6127
|
+
function summarizeOutput(stdout, stderr) {
|
|
6128
|
+
return {
|
|
6129
|
+
stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
|
|
6130
|
+
stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
|
|
6131
|
+
stdoutExcerpt: boundedExcerpt(stdout),
|
|
6132
|
+
stderrExcerpt: boundedExcerpt(stderr)
|
|
6133
|
+
};
|
|
6134
|
+
}
|
|
6135
|
+
function summarizeExecutorResult(result) {
|
|
6136
|
+
return {
|
|
6137
|
+
...summarizeOutput(result.stdout, result.stderr),
|
|
6138
|
+
status: result.status,
|
|
6139
|
+
exitCode: result.exitCode,
|
|
6140
|
+
durationMs: result.durationMs,
|
|
6141
|
+
error: boundedExcerpt(result.error)
|
|
6142
|
+
};
|
|
6143
|
+
}
|
|
6144
|
+
function isBlockedStepRun(step) {
|
|
6145
|
+
return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
|
|
6146
|
+
}
|
|
6147
|
+
function summarizeWorkflowStepRun(step) {
|
|
6148
|
+
return {
|
|
6149
|
+
...summarizeOutput(step.stdout, step.stderr),
|
|
6150
|
+
stepId: step.stepId,
|
|
6151
|
+
status: step.status,
|
|
6152
|
+
exitCode: step.exitCode,
|
|
6153
|
+
durationMs: step.durationMs,
|
|
6154
|
+
error: boundedExcerpt(step.error),
|
|
6155
|
+
blocked: isBlockedStepRun(step)
|
|
6156
|
+
};
|
|
6157
|
+
}
|
|
6158
|
+
function workflowRunEnvelope(run, steps) {
|
|
6159
|
+
return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
|
|
6160
|
+
}
|
|
6161
|
+
|
|
6162
|
+
// src/lib/goal/model-factory.ts
|
|
6163
|
+
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
|
|
6164
|
+
var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
|
|
6165
|
+
function resolveGoalModel(opts = {}) {
|
|
6166
|
+
const env = opts.env ?? process.env;
|
|
6167
|
+
const apiKey = opts.apiKey ?? env.OPENROUTER_API_KEY;
|
|
6168
|
+
if (!apiKey) {
|
|
6169
|
+
throw new Error("OPENROUTER_API_KEY is required to run goals with the OpenRouter AI SDK provider");
|
|
6170
|
+
}
|
|
6171
|
+
const provider = createOpenRouter({
|
|
6172
|
+
apiKey,
|
|
6173
|
+
baseURL: opts.baseURL ?? env.LOOPS_GOAL_BASE_URL ?? env.OPENROUTER_BASE_URL
|
|
6174
|
+
});
|
|
6175
|
+
return provider.chat(opts.model ?? env.LOOPS_GOAL_MODEL ?? DEFAULT_GOAL_MODEL);
|
|
6176
|
+
}
|
|
6177
|
+
function resolveGoalVerifierModel(opts = {}) {
|
|
6178
|
+
const env = opts.env ?? process.env;
|
|
6179
|
+
return resolveGoalModel({ ...opts, model: opts.model ?? env.LOOPS_GOAL_VERIFIER_MODEL });
|
|
6180
|
+
}
|
|
6181
|
+
|
|
4608
6182
|
// src/lib/goal/runner.ts
|
|
4609
6183
|
var DEFAULT_MAX_TURNS = 10;
|
|
4610
6184
|
var PlanNodeSchema = z.object({
|
|
@@ -4732,34 +6306,39 @@ function noReadyDiagnostic(goal, nodes) {
|
|
|
4732
6306
|
incompleteNodes: incompleteDetails
|
|
4733
6307
|
};
|
|
4734
6308
|
}
|
|
4735
|
-
function metadataFor(goal, node, context) {
|
|
4736
|
-
return {
|
|
4737
|
-
loopId: context?.loopId,
|
|
4738
|
-
loopName: context?.loopName,
|
|
4739
|
-
runId: context?.loopRunId,
|
|
4740
|
-
scheduledFor: context?.scheduledFor,
|
|
4741
|
-
workflowId: context?.workflowId,
|
|
4742
|
-
workflowName: context?.workflowName,
|
|
4743
|
-
workflowRunId: context?.workflowRunId,
|
|
4744
|
-
workflowStepId: context?.workflowStepId,
|
|
4745
|
-
goalId: goal.goalId,
|
|
4746
|
-
goalObjective: goal.objective,
|
|
4747
|
-
goalNodeKey: node.key
|
|
4748
|
-
};
|
|
4749
|
-
}
|
|
4750
6309
|
async function executeUnderlyingTarget(target, goal, node, opts) {
|
|
4751
|
-
const metadata =
|
|
6310
|
+
const metadata = executionMetadata(opts.context, goal, node);
|
|
4752
6311
|
if (opts.executeNode)
|
|
4753
6312
|
return opts.executeNode(node, metadata);
|
|
4754
6313
|
if (!target)
|
|
4755
6314
|
throw new Error("runGoal requires either target or executeNode");
|
|
4756
|
-
return executeTarget(target, metadata, {
|
|
4757
|
-
env: opts.env,
|
|
6315
|
+
return executeTarget(goalNodeTarget(target, goal, node), metadata, {
|
|
6316
|
+
env: withGoalNodeEnv(opts.env, node),
|
|
4758
6317
|
daemonLeaseId: opts.daemonLeaseId,
|
|
4759
6318
|
beforePersist: opts.beforePersist,
|
|
4760
6319
|
signal: opts.signal
|
|
4761
6320
|
});
|
|
4762
6321
|
}
|
|
6322
|
+
function verifierModelFor(spec, opts, planner) {
|
|
6323
|
+
if (opts.verifierModel)
|
|
6324
|
+
return opts.verifierModel;
|
|
6325
|
+
const env = opts.env ?? process.env;
|
|
6326
|
+
const configured = spec.verifierModel ?? env.LOOPS_GOAL_VERIFIER_MODEL;
|
|
6327
|
+
if (!configured)
|
|
6328
|
+
return planner;
|
|
6329
|
+
return resolveGoalVerifierModel({ model: configured, env: opts.env });
|
|
6330
|
+
}
|
|
6331
|
+
function nodeEvidence(node, result) {
|
|
6332
|
+
const summary = summarizeExecutorResult(result);
|
|
6333
|
+
return [
|
|
6334
|
+
`node ${node.key} ${summary.status} (exit ${summary.exitCode ?? "unknown"}, ${summary.durationMs}ms, ` + `stdout ${summary.stdoutBytes}B, stderr ${summary.stderrBytes}B)`,
|
|
6335
|
+
summary.stdoutExcerpt ? `stdout excerpt:
|
|
6336
|
+
${summary.stdoutExcerpt}` : undefined,
|
|
6337
|
+
summary.stderrExcerpt ? `stderr excerpt:
|
|
6338
|
+
${summary.stderrExcerpt}` : undefined
|
|
6339
|
+
].filter(Boolean).join(`
|
|
6340
|
+
`);
|
|
6341
|
+
}
|
|
4763
6342
|
async function planGoal(store, goal, spec, model, opts) {
|
|
4764
6343
|
const existing = store.listGoalPlanNodes(goal.goalId);
|
|
4765
6344
|
if (existing.length > 0)
|
|
@@ -4793,18 +6372,19 @@ async function planGoal(store, goal, spec, model, opts) {
|
|
|
4793
6372
|
return store.createGoalPlanNodes(goal.goalId, rawNodes, { daemonLeaseId: opts.daemonLeaseId });
|
|
4794
6373
|
}
|
|
4795
6374
|
function stdoutFor(goal, nodes, evidence, validation, diagnostics) {
|
|
4796
|
-
return JSON.stringify({
|
|
6375
|
+
return scrubSecrets(JSON.stringify(scrubSecretsDeep({
|
|
4797
6376
|
goal,
|
|
4798
6377
|
rollup: rollupSummary(nodes),
|
|
4799
6378
|
nodes,
|
|
4800
6379
|
evidence,
|
|
4801
6380
|
validation,
|
|
4802
6381
|
diagnostics
|
|
4803
|
-
}, null, 2);
|
|
6382
|
+
}), null, 2));
|
|
4804
6383
|
}
|
|
4805
6384
|
async function runGoal(store, input, opts = {}) {
|
|
4806
6385
|
const spec = normalizeGoalSpec2(input);
|
|
4807
6386
|
const model = opts.model ?? resolveGoalModel({ model: spec.model, env: opts.env });
|
|
6387
|
+
const verifier = verifierModelFor(spec, opts, model);
|
|
4808
6388
|
const startedAt = nowIso();
|
|
4809
6389
|
const existing = store.findGoalByContext({
|
|
4810
6390
|
loopRunId: opts.context?.loopRunId,
|
|
@@ -4837,6 +6417,13 @@ async function runGoal(store, input, opts = {}) {
|
|
|
4837
6417
|
goal = store.updateGoalStatus(goal.goalId, "budgetLimited", { daemonLeaseId: opts.daemonLeaseId });
|
|
4838
6418
|
return resultFromGoal(goal, "failed", stdoutFor(goal, nodes, evidence), "goal token budget exhausted after planning", startedAt);
|
|
4839
6419
|
}
|
|
6420
|
+
if ((goal.autoExecute ?? spec.autoExecute) === "off") {
|
|
6421
|
+
nodes = syncReadyFlags(store, goal, nodes, opts);
|
|
6422
|
+
return resultFromGoal(goal, "succeeded", stdoutFor(goal, nodes, evidence, undefined, {
|
|
6423
|
+
autoExecute: "off",
|
|
6424
|
+
note: "autoExecute is off: goal plan persisted without executing any nodes"
|
|
6425
|
+
}), undefined, startedAt);
|
|
6426
|
+
}
|
|
4840
6427
|
for (let turn = 1;turn <= (spec.maxTurns ?? DEFAULT_MAX_TURNS); turn++) {
|
|
4841
6428
|
if (opts.signal?.aborted) {
|
|
4842
6429
|
goal = store.updateGoalStatus(goal.goalId, "cancelled", { daemonLeaseId: opts.daemonLeaseId });
|
|
@@ -4875,11 +6462,7 @@ async function runGoal(store, input, opts = {}) {
|
|
|
4875
6462
|
}
|
|
4876
6463
|
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
4877
6464
|
if (result.status === "succeeded") {
|
|
4878
|
-
evidence.push(
|
|
4879
|
-
stdout:
|
|
4880
|
-
${result.stdout}
|
|
4881
|
-
stderr:
|
|
4882
|
-
${result.stderr}`);
|
|
6465
|
+
evidence.push(nodeEvidence(node, result));
|
|
4883
6466
|
store.updateGoalPlanNode(goal.goalId, node.key, {
|
|
4884
6467
|
status: "complete",
|
|
4885
6468
|
timeUsedSeconds: Math.round(result.durationMs / 1000)
|
|
@@ -4906,7 +6489,7 @@ ${result.stderr}`);
|
|
|
4906
6489
|
}
|
|
4907
6490
|
if (nodes.every((node) => node.status === "complete")) {
|
|
4908
6491
|
const judged = await generateObject({
|
|
4909
|
-
model,
|
|
6492
|
+
model: verifier,
|
|
4910
6493
|
schema: AchievementSchema,
|
|
4911
6494
|
temperature: 0,
|
|
4912
6495
|
prompt: achievementPrompt(goal, nodes, evidence),
|
|
@@ -4975,19 +6558,31 @@ ${result.stderr}`);
|
|
|
4975
6558
|
}
|
|
4976
6559
|
|
|
4977
6560
|
// src/lib/workflow-runner.ts
|
|
4978
|
-
|
|
6561
|
+
var DEFAULT_BLOCKED_EXIT_CODES = [12];
|
|
6562
|
+
var GATE_STEP_PATTERN = /(^|[^a-z])gate([^a-z]|$)/i;
|
|
6563
|
+
function blockedExitCodesForStep(step) {
|
|
6564
|
+
const explicit = step.blockedExitCodes;
|
|
6565
|
+
if (explicit !== undefined)
|
|
6566
|
+
return explicit.filter((code) => Number.isInteger(code));
|
|
6567
|
+
return GATE_STEP_PATTERN.test(step.name ? `${step.id} ${step.name}` : step.id) ? DEFAULT_BLOCKED_EXIT_CODES : [];
|
|
6568
|
+
}
|
|
6569
|
+
function targetWithStepAccount(step, goalNodePrompt) {
|
|
4979
6570
|
const account = step.account ?? step.target.account;
|
|
4980
6571
|
const timeoutMs = step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs;
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
6572
|
+
const target = !account && timeoutMs === step.target.timeoutMs ? step.target : { ...step.target, account, timeoutMs };
|
|
6573
|
+
if (goalNodePrompt && target.type === "agent") {
|
|
6574
|
+
return { ...target, prompt: `${target.prompt}
|
|
6575
|
+
|
|
6576
|
+
${goalNodePrompt}` };
|
|
6577
|
+
}
|
|
6578
|
+
return target;
|
|
4984
6579
|
}
|
|
4985
|
-
function workflowResult(workflowRun, status, startedAt, finishedAt,
|
|
6580
|
+
function workflowResult(workflowRun, status, startedAt, finishedAt, steps, error) {
|
|
4986
6581
|
const executorStatus = status === "succeeded" ? "succeeded" : status === "timed_out" ? "timed_out" : "failed";
|
|
4987
6582
|
return {
|
|
4988
6583
|
status: executorStatus,
|
|
4989
6584
|
exitCode: executorStatus === "succeeded" ? 0 : 1,
|
|
4990
|
-
stdout,
|
|
6585
|
+
stdout: workflowRunEnvelope(workflowRun, steps),
|
|
4991
6586
|
stderr: "",
|
|
4992
6587
|
error,
|
|
4993
6588
|
startedAt,
|
|
@@ -4997,20 +6592,16 @@ function workflowResult(workflowRun, status, startedAt, finishedAt, stdout, erro
|
|
|
4997
6592
|
}
|
|
4998
6593
|
async function executeWorkflow(store, workflow, opts = {}) {
|
|
4999
6594
|
if (workflow.goal) {
|
|
6595
|
+
const goalSpec = workflow.goal;
|
|
5000
6596
|
const workflowWithoutGoal = { ...workflow, goal: undefined };
|
|
5001
|
-
return runGoal(store,
|
|
6597
|
+
return runGoal(store, goalSpec, {
|
|
5002
6598
|
...opts,
|
|
5003
6599
|
model: opts.goalModel,
|
|
5004
|
-
context: {
|
|
5005
|
-
loopId: opts.loop?.id,
|
|
5006
|
-
loopName: opts.loop?.name,
|
|
5007
|
-
loopRunId: opts.loopRun?.id,
|
|
5008
|
-
scheduledFor: opts.loopRun?.scheduledFor ?? opts.scheduledFor,
|
|
5009
|
-
workflowId: workflow.id,
|
|
5010
|
-
workflowName: workflow.name
|
|
5011
|
-
},
|
|
6600
|
+
context: goalExecutionContext({ loop: opts.loop, loopRun: opts.loopRun, scheduledFor: opts.scheduledFor, workflow }),
|
|
5012
6601
|
executeNode: async (node) => executeWorkflow(store, workflowWithoutGoal, {
|
|
5013
6602
|
...opts,
|
|
6603
|
+
env: withGoalNodeEnv(opts.env, node),
|
|
6604
|
+
goalNodePrompt: iterationPrompt(goalSpec, node),
|
|
5014
6605
|
idempotencyKey: `${opts.idempotencyKey ?? workflow.id}:goal:${node.key}`
|
|
5015
6606
|
})
|
|
5016
6607
|
});
|
|
@@ -5025,8 +6616,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5025
6616
|
});
|
|
5026
6617
|
const startedAt = run.startedAt ?? nowIso();
|
|
5027
6618
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
5028
|
-
|
|
5029
|
-
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), JSON.stringify({ workflowRun: run, steps: steps2 }, null, 2), run.error);
|
|
6619
|
+
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), store.listWorkflowStepRuns(run.id), run.error);
|
|
5030
6620
|
}
|
|
5031
6621
|
const ordered = workflowExecutionOrder(workflow);
|
|
5032
6622
|
const byId = new Map(workflow.steps.map((step) => [step.id, step]));
|
|
@@ -5056,6 +6646,12 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5056
6646
|
});
|
|
5057
6647
|
if (blockedBy) {
|
|
5058
6648
|
opts.beforePersist?.();
|
|
6649
|
+
if (isBlockedStepRun(store.getWorkflowStepRun(run.id, blockedBy))) {
|
|
6650
|
+
store.skipWorkflowStepRun(run.id, step.id, `${BLOCKED_STEP_ERROR_PREFIX} upstream step ${blockedBy} was blocked`, {
|
|
6651
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6652
|
+
});
|
|
6653
|
+
continue;
|
|
6654
|
+
}
|
|
5059
6655
|
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
5060
6656
|
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
5061
6657
|
terminalStatus = "failed";
|
|
@@ -5068,16 +6664,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5068
6664
|
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
5069
6665
|
break;
|
|
5070
6666
|
}
|
|
5071
|
-
const
|
|
5072
|
-
|
|
5073
|
-
|
|
5074
|
-
|
|
5075
|
-
|
|
5076
|
-
workflowId: workflow.id,
|
|
5077
|
-
workflowName: workflow.name,
|
|
6667
|
+
const stepContext = goalExecutionContext({
|
|
6668
|
+
loop: opts.loop,
|
|
6669
|
+
loopRun: opts.loopRun,
|
|
6670
|
+
scheduledFor: opts.scheduledFor,
|
|
6671
|
+
workflow,
|
|
5078
6672
|
workflowRunId: run.id,
|
|
5079
6673
|
workflowStepId: step.id
|
|
5080
|
-
};
|
|
6674
|
+
});
|
|
5081
6675
|
let result;
|
|
5082
6676
|
const controller = new AbortController;
|
|
5083
6677
|
const externalAbort = () => controller.abort();
|
|
@@ -5096,19 +6690,10 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5096
6690
|
model: opts.goalModel,
|
|
5097
6691
|
target: targetWithStepAccount(step),
|
|
5098
6692
|
signal: controller.signal,
|
|
5099
|
-
context:
|
|
5100
|
-
loopId: opts.loop?.id,
|
|
5101
|
-
loopName: opts.loop?.name,
|
|
5102
|
-
loopRunId: opts.loopRun?.id,
|
|
5103
|
-
scheduledFor: opts.loopRun?.scheduledFor ?? opts.scheduledFor,
|
|
5104
|
-
workflowId: workflow.id,
|
|
5105
|
-
workflowName: workflow.name,
|
|
5106
|
-
workflowRunId: run.id,
|
|
5107
|
-
workflowStepId: step.id
|
|
5108
|
-
}
|
|
6693
|
+
context: stepContext
|
|
5109
6694
|
});
|
|
5110
6695
|
} else {
|
|
5111
|
-
result = await executeTarget(targetWithStepAccount(step),
|
|
6696
|
+
result = await executeTarget(targetWithStepAccount(step, opts.goalNodePrompt), executionMetadata(stepContext), {
|
|
5112
6697
|
...opts,
|
|
5113
6698
|
machine: opts.machine ?? opts.loop?.machine,
|
|
5114
6699
|
signal: controller.signal,
|
|
@@ -5144,6 +6729,21 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5144
6729
|
break;
|
|
5145
6730
|
}
|
|
5146
6731
|
opts.beforePersist?.();
|
|
6732
|
+
const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
|
|
6733
|
+
if (blockedExit) {
|
|
6734
|
+
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
6735
|
+
status: "skipped",
|
|
6736
|
+
finishedAt: result.finishedAt,
|
|
6737
|
+
durationMs: result.durationMs,
|
|
6738
|
+
stdout: result.stdout,
|
|
6739
|
+
stderr: result.stderr,
|
|
6740
|
+
exitCode: result.exitCode,
|
|
6741
|
+
error: `${BLOCKED_STEP_ERROR_PREFIX} step ${step.id} exited with blocked exit code ${result.exitCode}`
|
|
6742
|
+
}, {
|
|
6743
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6744
|
+
});
|
|
6745
|
+
continue;
|
|
6746
|
+
}
|
|
5147
6747
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
5148
6748
|
status: result.status,
|
|
5149
6749
|
finishedAt: result.finishedAt,
|
|
@@ -5174,8 +6774,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5174
6774
|
const finishedAt = nowIso();
|
|
5175
6775
|
if (store.isWorkflowRunTerminal(run.id)) {
|
|
5176
6776
|
const terminalRun = store.requireWorkflowRun(run.id);
|
|
5177
|
-
|
|
5178
|
-
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
6777
|
+
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
|
|
5179
6778
|
}
|
|
5180
6779
|
opts.beforePersist?.();
|
|
5181
6780
|
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
@@ -5185,8 +6784,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
5185
6784
|
}, {
|
|
5186
6785
|
daemonLeaseId: opts.daemonLeaseId
|
|
5187
6786
|
});
|
|
5188
|
-
|
|
5189
|
-
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
6787
|
+
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
|
|
5190
6788
|
}
|
|
5191
6789
|
function preflightWorkflow(workflow, opts = {}) {
|
|
5192
6790
|
return workflowExecutionOrder(workflow).map((step) => {
|
|
@@ -5237,12 +6835,7 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
5237
6835
|
...opts,
|
|
5238
6836
|
model: opts.goalModel,
|
|
5239
6837
|
target: loop.target,
|
|
5240
|
-
context: {
|
|
5241
|
-
loopId: loop.id,
|
|
5242
|
-
loopName: loop.name,
|
|
5243
|
-
loopRunId: run.id,
|
|
5244
|
-
scheduledFor: run.scheduledFor
|
|
5245
|
-
}
|
|
6838
|
+
context: goalExecutionContext({ loop, loopRun: run })
|
|
5246
6839
|
});
|
|
5247
6840
|
}
|
|
5248
6841
|
return executeLoop(loop, run, opts);
|
|
@@ -5257,23 +6850,19 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
5257
6850
|
}
|
|
5258
6851
|
}
|
|
5259
6852
|
if (loop.goal) {
|
|
6853
|
+
const loopGoal = loop.goal;
|
|
5260
6854
|
const workflowForLoopGoal = workflow.goal ? { ...workflow, goal: undefined } : workflow;
|
|
5261
|
-
return runGoal(store,
|
|
6855
|
+
return runGoal(store, loopGoal, {
|
|
5262
6856
|
...opts,
|
|
5263
6857
|
model: opts.goalModel,
|
|
5264
|
-
context: {
|
|
5265
|
-
loopId: loop.id,
|
|
5266
|
-
loopName: loop.name,
|
|
5267
|
-
loopRunId: run.id,
|
|
5268
|
-
scheduledFor: run.scheduledFor,
|
|
5269
|
-
workflowId: workflow.id,
|
|
5270
|
-
workflowName: workflow.name
|
|
5271
|
-
},
|
|
6858
|
+
context: goalExecutionContext({ loop, loopRun: run, workflow }),
|
|
5272
6859
|
executeNode: async (node) => executeWorkflow(store, workflowForLoopGoal, {
|
|
5273
6860
|
...opts,
|
|
5274
6861
|
loop,
|
|
5275
6862
|
loopRun: run,
|
|
5276
6863
|
scheduledFor: run.scheduledFor,
|
|
6864
|
+
env: withGoalNodeEnv(opts.env, node),
|
|
6865
|
+
goalNodePrompt: iterationPrompt(loopGoal, node),
|
|
5277
6866
|
idempotencyKey: `${loop.id}:${run.scheduledFor}:attempt:${run.attempt}:goal:${node.key}`
|
|
5278
6867
|
})
|
|
5279
6868
|
});
|
|
@@ -5338,12 +6927,129 @@ function manualRunSource(loop, scheduledFor, now = new Date) {
|
|
|
5338
6927
|
return "retry_slot";
|
|
5339
6928
|
return "due_slot";
|
|
5340
6929
|
}
|
|
5341
|
-
|
|
5342
|
-
|
|
6930
|
+
var INLINE_RUNNER_ID_PATTERN = /^[^:\s]+:(\d+)$/;
|
|
6931
|
+
function inlineRunnerOwnerPid(claimedBy) {
|
|
6932
|
+
const match = claimedBy ? INLINE_RUNNER_ID_PATTERN.exec(claimedBy) : null;
|
|
6933
|
+
return match ? Number(match[1]) : undefined;
|
|
6934
|
+
}
|
|
6935
|
+
async function runLoopNow(deps) {
|
|
6936
|
+
const { store, runnerId } = deps;
|
|
6937
|
+
const loop = store.requireLoop(deps.idOrName);
|
|
6938
|
+
if (loop.archivedAt)
|
|
6939
|
+
throw new LoopArchivedError(loop.name || deps.idOrName);
|
|
6940
|
+
const now = deps.now?.() ?? new Date;
|
|
6941
|
+
if (deps.mode === "schedule") {
|
|
6942
|
+
const scheduledFor2 = now.toISOString();
|
|
6943
|
+
const updated = store.updateLoop(loop.id, { status: "active", nextRunAt: scheduledFor2 });
|
|
6944
|
+
return { mode: "schedule", loop: updated, scheduledFor: scheduledFor2 };
|
|
6945
|
+
}
|
|
6946
|
+
let scheduledFor = manualRunScheduledFor(loop, now);
|
|
6947
|
+
let source = manualRunSource(loop, scheduledFor, now);
|
|
6948
|
+
let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
|
|
6949
|
+
let claim = store.claimRun(loop, scheduledFor, runnerId, now);
|
|
6950
|
+
if (!claim && shouldAdvance) {
|
|
6951
|
+
const existing = store.getRunBySlot(loop.id, scheduledFor);
|
|
6952
|
+
if (existing && existing.status !== "running") {
|
|
6953
|
+
scheduledFor = now.toISOString();
|
|
6954
|
+
source = "ad_hoc";
|
|
6955
|
+
shouldAdvance = false;
|
|
6956
|
+
claim = store.claimRun(loop, scheduledFor, runnerId, now);
|
|
6957
|
+
}
|
|
6958
|
+
}
|
|
6959
|
+
if (!claim)
|
|
6960
|
+
throw new Error(`could not claim manual run for ${deps.idOrName}`);
|
|
6961
|
+
const run = await executeClaimedRun({
|
|
6962
|
+
store,
|
|
6963
|
+
runnerId,
|
|
6964
|
+
loop: claim.loop,
|
|
6965
|
+
run: claim.run,
|
|
6966
|
+
now: deps.now,
|
|
6967
|
+
execute: deps.execute
|
|
6968
|
+
});
|
|
6969
|
+
if (shouldAdvance) {
|
|
6970
|
+
advanceLoop(store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
|
|
6971
|
+
}
|
|
6972
|
+
return { mode: "inline", loop: claim.loop, run, source, advancedLoop: shouldAdvance };
|
|
6973
|
+
}
|
|
6974
|
+
var MAX_RETRY_DELAY_MS = 6 * 60 * 60 * 1000;
|
|
6975
|
+
var DEFAULT_CIRCUIT_BREAKER_THRESHOLD = 5;
|
|
6976
|
+
var CIRCUIT_BREAKER_REASON_PREFIX = "circuit breaker open";
|
|
6977
|
+
var MAX_SKIPS_PER_LOOP_PER_TICK = 10;
|
|
6978
|
+
var THROTTLED_RETRY_MULTIPLIER = 4;
|
|
6979
|
+
var MAX_RETRY_EXPONENT = 20;
|
|
6980
|
+
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
6981
|
+
const attempt = Math.max(1, run.attempt);
|
|
6982
|
+
const failure = classifyRunFailure(run);
|
|
6983
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
6984
|
+
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
6985
|
+
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
6986
|
+
const jitter = 0.5 + random();
|
|
6987
|
+
return Math.min(MAX_RETRY_DELAY_MS, Math.round(base * jitter));
|
|
6988
|
+
}
|
|
6989
|
+
function nextAfterRetry(loop, run, now, random) {
|
|
6990
|
+
return new Date(now.getTime() + retryBackoffDelayMs(loop, run, random)).toISOString();
|
|
5343
6991
|
}
|
|
5344
6992
|
function isDaemonLeaseLost(error) {
|
|
5345
6993
|
return error instanceof Error && error.message === "daemon lease lost";
|
|
5346
6994
|
}
|
|
6995
|
+
function resolveBreakerThreshold(loop, override) {
|
|
6996
|
+
const perLoop = loop.circuitBreakerThreshold;
|
|
6997
|
+
if (typeof perLoop === "number" && Number.isFinite(perLoop))
|
|
6998
|
+
return Math.floor(perLoop);
|
|
6999
|
+
const resolved = typeof override === "function" ? override(loop) : override;
|
|
7000
|
+
if (typeof resolved === "number" && Number.isFinite(resolved))
|
|
7001
|
+
return Math.floor(resolved);
|
|
7002
|
+
return DEFAULT_CIRCUIT_BREAKER_THRESHOLD;
|
|
7003
|
+
}
|
|
7004
|
+
function consecutiveFailureCount(store, loopId, maxAttempts = 1, scanLimit = 50) {
|
|
7005
|
+
const runs = store.listRuns({ loopId, limit: scanLimit });
|
|
7006
|
+
let watermark;
|
|
7007
|
+
for (const run of runs) {
|
|
7008
|
+
if (run.status !== "skipped" || !run.error?.startsWith(CIRCUIT_BREAKER_REASON_PREFIX))
|
|
7009
|
+
continue;
|
|
7010
|
+
const at = new Date(run.scheduledFor).getTime();
|
|
7011
|
+
if (watermark === undefined || at > watermark)
|
|
7012
|
+
watermark = at;
|
|
7013
|
+
}
|
|
7014
|
+
let count = 0;
|
|
7015
|
+
for (const run of runs) {
|
|
7016
|
+
if (run.status === "running" || run.status === "skipped")
|
|
7017
|
+
continue;
|
|
7018
|
+
if (watermark !== undefined && new Date(run.scheduledFor).getTime() <= watermark)
|
|
7019
|
+
continue;
|
|
7020
|
+
if (run.status === "succeeded")
|
|
7021
|
+
break;
|
|
7022
|
+
if (run.attempt < maxAttempts)
|
|
7023
|
+
continue;
|
|
7024
|
+
count += 1;
|
|
7025
|
+
}
|
|
7026
|
+
return count;
|
|
7027
|
+
}
|
|
7028
|
+
function awaitStrictlyNewerRunTimestamp(store, loopId) {
|
|
7029
|
+
const latest = store.listRuns({ loopId, limit: 1 })[0];
|
|
7030
|
+
if (!latest)
|
|
7031
|
+
return;
|
|
7032
|
+
const latestMs = new Date(latest.createdAt).getTime();
|
|
7033
|
+
for (let spin = 0;spin < 1e6 && Date.now() <= latestMs; spin += 1) {}
|
|
7034
|
+
}
|
|
7035
|
+
function tripCircuitBreaker(store, loop, run, finishedAt, failures, opts) {
|
|
7036
|
+
awaitStrictlyNewerRunTimestamp(store, loop.id);
|
|
7037
|
+
const reason = `${CIRCUIT_BREAKER_REASON_PREFIX}: ${failures} consecutive failed runs; loop auto-paused (resume with 'loops resume ${loop.name}')`;
|
|
7038
|
+
let markerAtMs = finishedAt.getTime();
|
|
7039
|
+
for (let probe = 0;probe < 1000 && store.getRunBySlot(loop.id, new Date(markerAtMs).toISOString()); probe += 1) {
|
|
7040
|
+
markerAtMs += 1;
|
|
7041
|
+
}
|
|
7042
|
+
const marker = store.createSkippedRun(loop, new Date(markerAtMs).toISOString(), reason, {
|
|
7043
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
7044
|
+
});
|
|
7045
|
+
const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
|
|
7046
|
+
store.updateLoop(loop.id, {
|
|
7047
|
+
status: "paused",
|
|
7048
|
+
nextRunAt,
|
|
7049
|
+
retryScheduledFor: undefined
|
|
7050
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
7051
|
+
opts.onRun?.(marker);
|
|
7052
|
+
}
|
|
5347
7053
|
function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
5348
7054
|
if (run.status === "running")
|
|
5349
7055
|
return;
|
|
@@ -5356,7 +7062,7 @@ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
|
5356
7062
|
if (shouldRetry) {
|
|
5357
7063
|
store.updateLoop(current.id, {
|
|
5358
7064
|
status: "active",
|
|
5359
|
-
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
7065
|
+
nextRunAt: nextAfterRetry(current, run, finishedAt, opts.random),
|
|
5360
7066
|
retryScheduledFor: run.scheduledFor
|
|
5361
7067
|
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
5362
7068
|
return;
|
|
@@ -5365,11 +7071,21 @@ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
|
5365
7071
|
if (deferredRetry) {
|
|
5366
7072
|
store.updateLoop(current.id, {
|
|
5367
7073
|
status: "active",
|
|
5368
|
-
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
7074
|
+
nextRunAt: nextAfterRetry(current, deferredRetry, finishedAt, opts.random),
|
|
5369
7075
|
retryScheduledFor: deferredRetry.scheduledFor
|
|
5370
7076
|
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
5371
7077
|
return;
|
|
5372
7078
|
}
|
|
7079
|
+
if (!succeeded) {
|
|
7080
|
+
const threshold = resolveBreakerThreshold(current, opts.circuitBreakerThreshold);
|
|
7081
|
+
if (threshold > 0) {
|
|
7082
|
+
const failures = consecutiveFailureCount(store, current.id, current.maxAttempts, Math.max(threshold * 4, 50));
|
|
7083
|
+
if (failures >= threshold) {
|
|
7084
|
+
tripCircuitBreaker(store, current, run, finishedAt, failures, opts);
|
|
7085
|
+
return;
|
|
7086
|
+
}
|
|
7087
|
+
}
|
|
7088
|
+
}
|
|
5373
7089
|
const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
|
|
5374
7090
|
store.updateLoop(current.id, {
|
|
5375
7091
|
status: nextRunAt ? "active" : "stopped",
|
|
@@ -5431,6 +7147,14 @@ async function executeClaimedRun(deps) {
|
|
|
5431
7147
|
clearInterval(heartbeat);
|
|
5432
7148
|
}
|
|
5433
7149
|
}
|
|
7150
|
+
function advanceOptions(deps) {
|
|
7151
|
+
return {
|
|
7152
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
7153
|
+
random: deps.random,
|
|
7154
|
+
circuitBreakerThreshold: deps.circuitBreakerThreshold,
|
|
7155
|
+
onRun: deps.onRun
|
|
7156
|
+
};
|
|
7157
|
+
}
|
|
5434
7158
|
async function runSlot(deps, loop, scheduledFor) {
|
|
5435
7159
|
const now = deps.now?.() ?? new Date;
|
|
5436
7160
|
deps.beforeRun?.(loop, scheduledFor);
|
|
@@ -5445,7 +7169,7 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
5445
7169
|
return;
|
|
5446
7170
|
throw error;
|
|
5447
7171
|
}
|
|
5448
|
-
advanceLoop(deps.store, loop, skipped, now, true,
|
|
7172
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
5449
7173
|
deps.onRun?.(skipped);
|
|
5450
7174
|
return skipped;
|
|
5451
7175
|
}
|
|
@@ -5472,7 +7196,7 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
5472
7196
|
daemonLeaseId: deps.daemonLeaseId,
|
|
5473
7197
|
onError: deps.onError
|
|
5474
7198
|
});
|
|
5475
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded",
|
|
7199
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
|
|
5476
7200
|
deps.onRun?.(finalRun);
|
|
5477
7201
|
return finalRun;
|
|
5478
7202
|
}
|
|
@@ -5492,7 +7216,7 @@ function claimSlot(deps, loop, scheduledFor) {
|
|
|
5492
7216
|
return;
|
|
5493
7217
|
throw error;
|
|
5494
7218
|
}
|
|
5495
|
-
advanceLoop(deps.store, loop, skipped, now, true,
|
|
7219
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
5496
7220
|
deps.onRun?.(skipped);
|
|
5497
7221
|
return skipped;
|
|
5498
7222
|
}
|
|
@@ -5510,8 +7234,7 @@ function claimSlot(deps, loop, scheduledFor) {
|
|
|
5510
7234
|
deps.onRun?.(claim.run);
|
|
5511
7235
|
return claim;
|
|
5512
7236
|
}
|
|
5513
|
-
function
|
|
5514
|
-
const now = deps.now?.() ?? new Date;
|
|
7237
|
+
function recoverAndExpire(deps, now) {
|
|
5515
7238
|
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
5516
7239
|
const recoveredByLoop = new Map;
|
|
5517
7240
|
for (const run of recovered) {
|
|
@@ -5523,21 +7246,22 @@ function claimDueRuns(deps) {
|
|
|
5523
7246
|
continue;
|
|
5524
7247
|
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
5525
7248
|
if (retryable) {
|
|
5526
|
-
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false,
|
|
5527
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5528
|
-
});
|
|
7249
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
|
|
5529
7250
|
continue;
|
|
5530
7251
|
}
|
|
5531
7252
|
for (const run of runs) {
|
|
5532
7253
|
const current = deps.store.getLoop(run.loopId);
|
|
5533
7254
|
if (current) {
|
|
5534
|
-
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false,
|
|
5535
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5536
|
-
});
|
|
7255
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
|
|
5537
7256
|
}
|
|
5538
7257
|
}
|
|
5539
7258
|
}
|
|
5540
7259
|
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
7260
|
+
return { recovered, expired };
|
|
7261
|
+
}
|
|
7262
|
+
function claimDueRuns(deps) {
|
|
7263
|
+
const now = deps.now?.() ?? new Date;
|
|
7264
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
5541
7265
|
const claims = [];
|
|
5542
7266
|
const claimed = [];
|
|
5543
7267
|
const skipped = [];
|
|
@@ -5545,11 +7269,14 @@ function claimDueRuns(deps) {
|
|
|
5545
7269
|
if (maxClaims === 0)
|
|
5546
7270
|
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
5547
7271
|
for (const loop of deps.store.dueLoops(now)) {
|
|
5548
|
-
if (claims.length
|
|
7272
|
+
if (claims.length >= maxClaims)
|
|
5549
7273
|
break;
|
|
5550
7274
|
const plan = dueSlots(loop, now);
|
|
7275
|
+
let loopSkips = 0;
|
|
5551
7276
|
for (const slot of plan.slots) {
|
|
5552
|
-
if (claims.length
|
|
7277
|
+
if (claims.length >= maxClaims)
|
|
7278
|
+
break;
|
|
7279
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
5553
7280
|
break;
|
|
5554
7281
|
const run = claimSlot(deps, loop, slot);
|
|
5555
7282
|
if (!run)
|
|
@@ -5559,6 +7286,7 @@ function claimDueRuns(deps) {
|
|
|
5559
7286
|
claimed.push(run.run);
|
|
5560
7287
|
} else if (run.status === "skipped") {
|
|
5561
7288
|
skipped.push(run);
|
|
7289
|
+
loopSkips += 1;
|
|
5562
7290
|
}
|
|
5563
7291
|
}
|
|
5564
7292
|
}
|
|
@@ -5566,46 +7294,25 @@ function claimDueRuns(deps) {
|
|
|
5566
7294
|
}
|
|
5567
7295
|
async function tick(deps) {
|
|
5568
7296
|
const now = deps.now?.() ?? new Date;
|
|
5569
|
-
const recovered = deps
|
|
5570
|
-
const recoveredByLoop = new Map;
|
|
5571
|
-
for (const run of recovered) {
|
|
5572
|
-
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
5573
|
-
}
|
|
5574
|
-
for (const runs of recoveredByLoop.values()) {
|
|
5575
|
-
const loop = deps.store.getLoop(runs[0].loopId);
|
|
5576
|
-
if (!loop)
|
|
5577
|
-
continue;
|
|
5578
|
-
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
5579
|
-
if (retryable) {
|
|
5580
|
-
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
5581
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5582
|
-
});
|
|
5583
|
-
continue;
|
|
5584
|
-
}
|
|
5585
|
-
for (const run of runs) {
|
|
5586
|
-
const current = deps.store.getLoop(run.loopId);
|
|
5587
|
-
if (current) {
|
|
5588
|
-
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
5589
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5590
|
-
});
|
|
5591
|
-
}
|
|
5592
|
-
}
|
|
5593
|
-
}
|
|
5594
|
-
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
7297
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
5595
7298
|
const claimed = [];
|
|
5596
7299
|
const completed = [];
|
|
5597
7300
|
const skipped = [];
|
|
5598
7301
|
for (const loop of deps.store.dueLoops(now)) {
|
|
5599
7302
|
const plan = dueSlots(loop, now);
|
|
7303
|
+
let loopSkips = 0;
|
|
5600
7304
|
for (const slot of plan.slots) {
|
|
7305
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
7306
|
+
break;
|
|
5601
7307
|
const run = await runSlot(deps, loop, slot);
|
|
5602
7308
|
if (!run)
|
|
5603
7309
|
continue;
|
|
5604
7310
|
if (run.status === "running")
|
|
5605
7311
|
claimed.push(run);
|
|
5606
|
-
else if (run.status === "skipped")
|
|
7312
|
+
else if (run.status === "skipped") {
|
|
5607
7313
|
skipped.push(run);
|
|
5608
|
-
|
|
7314
|
+
loopSkips += 1;
|
|
7315
|
+
} else
|
|
5609
7316
|
completed.push(run);
|
|
5610
7317
|
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
5611
7318
|
break;
|
|
@@ -5627,29 +7334,25 @@ class LoopsClient {
|
|
|
5627
7334
|
create(input) {
|
|
5628
7335
|
return this.store.createLoop(input);
|
|
5629
7336
|
}
|
|
5630
|
-
list() {
|
|
5631
|
-
return this.store.listLoops(
|
|
7337
|
+
list(filters = {}) {
|
|
7338
|
+
return this.store.listLoops({
|
|
7339
|
+
status: filters.status,
|
|
7340
|
+
limit: filters.limit,
|
|
7341
|
+
includeArchived: filters.includeArchived,
|
|
7342
|
+
archived: filters.archivedOnly
|
|
7343
|
+
});
|
|
5632
7344
|
}
|
|
5633
7345
|
get(idOrName) {
|
|
5634
7346
|
return this.store.requireLoop(idOrName);
|
|
5635
7347
|
}
|
|
5636
7348
|
pause(idOrName) {
|
|
5637
|
-
|
|
5638
|
-
if (loop.archivedAt)
|
|
5639
|
-
throw new Error(`loop is archived; unarchive it before pausing: ${idOrName}`);
|
|
5640
|
-
return this.store.updateLoop(loop.id, { status: "paused" });
|
|
7349
|
+
return this.store.updateLoop(this.get(idOrName).id, { status: "paused" });
|
|
5641
7350
|
}
|
|
5642
7351
|
resume(idOrName) {
|
|
5643
|
-
|
|
5644
|
-
if (loop.archivedAt)
|
|
5645
|
-
throw new Error(`loop is archived; unarchive it before resuming: ${idOrName}`);
|
|
5646
|
-
return this.store.updateLoop(loop.id, { status: "active" });
|
|
7352
|
+
return this.store.updateLoop(this.get(idOrName).id, { status: "active" });
|
|
5647
7353
|
}
|
|
5648
7354
|
stop(idOrName) {
|
|
5649
|
-
|
|
5650
|
-
if (loop.archivedAt)
|
|
5651
|
-
throw new Error(`loop is archived; unarchive it before stopping: ${idOrName}`);
|
|
5652
|
-
return this.store.updateLoop(loop.id, { status: "stopped", nextRunAt: undefined });
|
|
7355
|
+
return this.store.updateLoop(this.get(idOrName).id, { status: "stopped", nextRunAt: undefined });
|
|
5653
7356
|
}
|
|
5654
7357
|
archive(idOrName) {
|
|
5655
7358
|
return this.store.archiveLoop(idOrName);
|
|
@@ -5660,8 +7363,24 @@ class LoopsClient {
|
|
|
5660
7363
|
delete(idOrName) {
|
|
5661
7364
|
return this.store.deleteLoop(idOrName);
|
|
5662
7365
|
}
|
|
5663
|
-
runs(
|
|
5664
|
-
|
|
7366
|
+
runs(idOrName, filters = {}) {
|
|
7367
|
+
let loopId;
|
|
7368
|
+
if (idOrName) {
|
|
7369
|
+
try {
|
|
7370
|
+
loopId = this.get(idOrName).id;
|
|
7371
|
+
} catch (error) {
|
|
7372
|
+
if (error instanceof LoopNotFoundError)
|
|
7373
|
+
return [];
|
|
7374
|
+
throw error;
|
|
7375
|
+
}
|
|
7376
|
+
}
|
|
7377
|
+
return this.store.listRuns({ loopId, status: filters.status, limit: filters.limit });
|
|
7378
|
+
}
|
|
7379
|
+
doctor() {
|
|
7380
|
+
return runDoctor(this.store);
|
|
7381
|
+
}
|
|
7382
|
+
health(opts = {}) {
|
|
7383
|
+
return buildHealthReport(this.store, opts);
|
|
5665
7384
|
}
|
|
5666
7385
|
goal(idOrName) {
|
|
5667
7386
|
const goal = this.store.getGoal(idOrName) ?? this.store.findGoalByLoop(idOrName) ?? this.store.findGoalByRunId(idOrName);
|
|
@@ -5674,28 +7393,8 @@ class LoopsClient {
|
|
|
5674
7393
|
return tick({ store: this.store, runnerId: this.runnerId });
|
|
5675
7394
|
}
|
|
5676
7395
|
async runNow(idOrName) {
|
|
5677
|
-
const
|
|
5678
|
-
|
|
5679
|
-
throw new Error(`loop is archived; unarchive it before running: ${idOrName}`);
|
|
5680
|
-
const now = new Date;
|
|
5681
|
-
let scheduledFor = manualRunScheduledFor(loop, now);
|
|
5682
|
-
let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
|
|
5683
|
-
let claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
|
|
5684
|
-
if (!claim && shouldAdvance) {
|
|
5685
|
-
const existing = this.store.getRunBySlot(loop.id, scheduledFor);
|
|
5686
|
-
if (existing && existing.status !== "running") {
|
|
5687
|
-
scheduledFor = now.toISOString();
|
|
5688
|
-
shouldAdvance = false;
|
|
5689
|
-
claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
|
|
5690
|
-
}
|
|
5691
|
-
}
|
|
5692
|
-
if (!claim)
|
|
5693
|
-
throw new Error(`could not claim manual run for ${idOrName}`);
|
|
5694
|
-
const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });
|
|
5695
|
-
if (shouldAdvance) {
|
|
5696
|
-
advanceLoop(this.store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
|
|
5697
|
-
}
|
|
5698
|
-
return run;
|
|
7396
|
+
const result = await runLoopNow({ store: this.store, idOrName, runnerId: this.runnerId });
|
|
7397
|
+
return result.run;
|
|
5699
7398
|
}
|
|
5700
7399
|
close() {
|
|
5701
7400
|
if (this.ownStore)
|
|
@@ -5718,15 +7417,15 @@ function openAutomationsRuntimeBinding(overrides = {}) {
|
|
|
5718
7417
|
failCommand: "automations queue fail",
|
|
5719
7418
|
eventHandoff: {
|
|
5720
7419
|
envelopeCommand: "automations webhooks event",
|
|
5721
|
-
handlerCommand: "loops
|
|
5722
|
-
pipeExample: "automations --json webhooks event <route> --body-json '<json>' | loops --json
|
|
7420
|
+
handlerCommand: "loops routes create generic",
|
|
7421
|
+
pipeExample: "automations --json webhooks event <route> --body-json '<json>' | loops --json routes create generic",
|
|
5723
7422
|
boundary: "Use only for explicit event-envelope workflow handoff. OpenAutomations still owns deterministic automation materialization and queue state; OpenLoops owns workflow invocation."
|
|
5724
7423
|
},
|
|
5725
7424
|
requiredEnvironment: ["HASNA_AUTOMATIONS_DIR"],
|
|
5726
7425
|
guarantees: [
|
|
5727
7426
|
"OpenAutomations owns automation specs, run materialization, queue state, DLQ, replay, idempotency, and approvals.",
|
|
5728
7427
|
"OpenLoops may execute claimed actions through explicit command or SDK handoff only.",
|
|
5729
|
-
"OpenLoops may consume exported event envelopes only through explicit
|
|
7428
|
+
"OpenLoops may consume exported event envelopes only through explicit routes create commands.",
|
|
5730
7429
|
"Workers must complete or fail actions by action id and runner id so OpenAutomations can enforce queue leases."
|
|
5731
7430
|
],
|
|
5732
7431
|
nonGoals: [
|