@hasna/loops 0.3.59 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +247 -0
- package/LICENSE +197 -13
- package/README.md +28 -60
- package/dist/cli/index.js +7269 -5776
- package/dist/daemon/control.d.ts +21 -1
- package/dist/daemon/daemon.d.ts +5 -0
- package/dist/daemon/index.js +2985 -1072
- package/dist/daemon/install.d.ts +1 -1
- package/dist/index.d.ts +21 -8
- package/dist/index.js +5906 -4580
- package/dist/lib/accounts.d.ts +6 -1
- package/dist/lib/agent-adapter.d.ts +74 -0
- package/dist/lib/backup.d.ts +25 -0
- package/dist/lib/errors.d.ts +21 -0
- package/dist/lib/executor.d.ts +10 -1
- package/dist/lib/goal/metadata.d.ts +16 -0
- package/dist/lib/goal/model-factory.d.ts +1 -0
- package/dist/lib/goal/prompts.d.ts +3 -1
- package/dist/lib/goal/types.d.ts +3 -0
- package/dist/lib/health.d.ts +1 -1
- package/dist/lib/ids.d.ts +8 -1
- package/dist/lib/machines.d.ts +6 -0
- package/dist/lib/process-identity.d.ts +16 -0
- package/dist/lib/{schedule.d.ts → recurrence.d.ts} +1 -0
- package/dist/lib/redact.d.ts +21 -0
- package/dist/lib/route/cursors.d.ts +18 -0
- package/dist/lib/route/drain.d.ts +6 -0
- package/dist/lib/route/fields.d.ts +27 -0
- package/dist/lib/route/gates.d.ts +26 -0
- package/dist/lib/route/index.d.ts +18 -0
- package/dist/lib/route/options.d.ts +31 -0
- package/dist/lib/route/parse.d.ts +8 -0
- package/dist/lib/route/pr-review.d.ts +11 -0
- package/dist/lib/route/provider.d.ts +41 -0
- package/dist/lib/route/route-event.d.ts +23 -0
- package/dist/lib/route/route-tasks.d.ts +75 -0
- package/dist/lib/route/throttle.d.ts +47 -0
- package/dist/lib/route/todos-cli.d.ts +21 -0
- package/dist/lib/route/types.d.ts +88 -0
- package/dist/lib/run-artifacts.d.ts +17 -2
- package/dist/lib/run-envelope.d.ts +41 -0
- package/dist/lib/scheduler.d.ts +78 -2
- package/dist/lib/store.d.ts +63 -0
- package/dist/lib/store.js +1011 -266
- package/dist/lib/template-kit.d.ts +70 -0
- package/dist/lib/templates-custom.d.ts +34 -0
- package/dist/lib/templates.d.ts +13 -81
- package/dist/lib/workflow-runner.d.ts +2 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.js +3313 -1165
- package/dist/sdk/index.d.ts +29 -3
- package/dist/sdk/index.js +2910 -897
- package/dist/test-helpers.d.ts +37 -0
- package/dist/types.d.ts +2 -0
- package/docs/USAGE.md +40 -18
- package/package.json +6 -3
package/dist/sdk/index.js
CHANGED
|
@@ -1,14 +1,50 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/lib/store.ts
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync } from "fs";
|
|
4
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
|
|
5
5
|
import { tmpdir } from "os";
|
|
6
|
-
import { dirname, join as join3 } from "path";
|
|
6
|
+
import { basename as basename2, dirname as dirname2, join as join3 } from "path";
|
|
7
|
+
|
|
8
|
+
// src/lib/errors.ts
|
|
9
|
+
class CodedError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
constructor(code, message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = new.target.name;
|
|
14
|
+
this.code = code;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
class LoopNotFoundError extends CodedError {
|
|
19
|
+
constructor(idOrName) {
|
|
20
|
+
super("LOOP_NOT_FOUND", `loop not found: ${idOrName}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class LoopArchivedError extends CodedError {
|
|
25
|
+
constructor(idOrName) {
|
|
26
|
+
super("LOOP_ARCHIVED", `loop is archived: ${idOrName}; unarchive it before modifying`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
class AmbiguousNameError extends CodedError {
|
|
31
|
+
constructor(name) {
|
|
32
|
+
super("AMBIGUOUS_NAME", `ambiguous loop name: ${name}; use a loop id`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class ValidationError extends CodedError {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super("VALIDATION_ERROR", message);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
7
41
|
|
|
8
42
|
// src/lib/ids.ts
|
|
9
43
|
import { randomBytes } from "crypto";
|
|
10
|
-
|
|
11
|
-
|
|
44
|
+
var TIME_HEX_LENGTH = 12;
|
|
45
|
+
function genId() {
|
|
46
|
+
const time = Date.now().toString(16).padStart(TIME_HEX_LENGTH, "0");
|
|
47
|
+
return `${time}${randomBytes(10).toString("hex")}`;
|
|
12
48
|
}
|
|
13
49
|
function nowIso() {
|
|
14
50
|
return new Date().toISOString();
|
|
@@ -42,7 +78,166 @@ function launchdPlistPath() {
|
|
|
42
78
|
return join(homedir(), "Library", "LaunchAgents", "com.hasna.loops.daemon.plist");
|
|
43
79
|
}
|
|
44
80
|
|
|
45
|
-
// src/lib/
|
|
81
|
+
// src/lib/process-identity.ts
|
|
82
|
+
import { readFileSync } from "fs";
|
|
83
|
+
import { spawnSync } from "child_process";
|
|
84
|
+
var START_TIME_TOLERANCE_MS = 5000;
|
|
85
|
+
var clockTicksCache;
|
|
86
|
+
function clockTicksPerSecond() {
|
|
87
|
+
if (clockTicksCache !== undefined)
|
|
88
|
+
return clockTicksCache;
|
|
89
|
+
try {
|
|
90
|
+
const run = spawnSync("getconf", ["CLK_TCK"], { encoding: "utf8" });
|
|
91
|
+
const value = Number(run.stdout.trim());
|
|
92
|
+
clockTicksCache = run.status === 0 && Number.isFinite(value) && value > 0 ? value : 100;
|
|
93
|
+
} catch {
|
|
94
|
+
clockTicksCache = 100;
|
|
95
|
+
}
|
|
96
|
+
return clockTicksCache;
|
|
97
|
+
}
|
|
98
|
+
var bootTimeCache;
|
|
99
|
+
var bootTimeResolved = false;
|
|
100
|
+
function bootTimeMs() {
|
|
101
|
+
if (bootTimeResolved)
|
|
102
|
+
return bootTimeCache;
|
|
103
|
+
bootTimeResolved = true;
|
|
104
|
+
try {
|
|
105
|
+
const match = readFileSync("/proc/stat", "utf8").match(/^btime (\d+)$/m);
|
|
106
|
+
bootTimeCache = match ? Number(match[1]) * 1000 : undefined;
|
|
107
|
+
} catch {
|
|
108
|
+
bootTimeCache = undefined;
|
|
109
|
+
}
|
|
110
|
+
return bootTimeCache;
|
|
111
|
+
}
|
|
112
|
+
function procStatFields(path) {
|
|
113
|
+
try {
|
|
114
|
+
const stat = readFileSync(path, "utf8");
|
|
115
|
+
const closeParen = stat.lastIndexOf(")");
|
|
116
|
+
if (closeParen < 0)
|
|
117
|
+
return;
|
|
118
|
+
return stat.slice(closeParen + 2).split(" ");
|
|
119
|
+
} catch {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function processStartTimeMs(pid) {
|
|
124
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
125
|
+
return;
|
|
126
|
+
if (process.platform === "linux") {
|
|
127
|
+
const fields = procStatFields(`/proc/${pid}/stat`);
|
|
128
|
+
const startTicks = fields ? Number(fields[19]) : Number.NaN;
|
|
129
|
+
const boot = bootTimeMs();
|
|
130
|
+
if (Number.isFinite(startTicks) && boot !== undefined) {
|
|
131
|
+
return boot + Math.round(startTicks / clockTicksPerSecond() * 1000);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
const run = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" });
|
|
136
|
+
if (run.status === 0) {
|
|
137
|
+
const parsed = Date.parse(run.stdout.trim());
|
|
138
|
+
if (Number.isFinite(parsed))
|
|
139
|
+
return parsed;
|
|
140
|
+
}
|
|
141
|
+
} catch {}
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
function sameProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
145
|
+
if (recorded === undefined || actualMs === undefined)
|
|
146
|
+
return true;
|
|
147
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
148
|
+
if (!Number.isFinite(recordedMs))
|
|
149
|
+
return true;
|
|
150
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
151
|
+
}
|
|
152
|
+
function verifiedProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
153
|
+
if (recorded === undefined || actualMs === undefined)
|
|
154
|
+
return false;
|
|
155
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
156
|
+
if (!Number.isFinite(recordedMs))
|
|
157
|
+
return false;
|
|
158
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/lib/redact.ts
|
|
162
|
+
var SCRUBBED = "[SCRUBBED]";
|
|
163
|
+
var TOKEN_PATTERNS = [
|
|
164
|
+
/\bsk-ant-[A-Za-z0-9_-]{8,}/g,
|
|
165
|
+
/\bsk-proj-[A-Za-z0-9_-]{8,}/g,
|
|
166
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
167
|
+
/\bghp_[A-Za-z0-9]{16,}\b/g,
|
|
168
|
+
/\bgithub_pat_[A-Za-z0-9_]{16,}\b/g,
|
|
169
|
+
/\bxox[a-z]-[A-Za-z0-9-]{8,}/g,
|
|
170
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{10,}\b/g
|
|
171
|
+
];
|
|
172
|
+
var AUTHORIZATION_PATTERN = /(\bAuthorization(?:\\?["'])?\s*[:=]\s*(?:\\?["'])?(?:Bearer|Basic)\s+)[A-Za-z0-9._~+/=-]{16,}/gi;
|
|
173
|
+
var PEM_PATTERN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|$)/g;
|
|
174
|
+
var ASSIGNMENT_PATTERN = /([A-Za-z0-9_.-]{0,64}(?:key|token|secret|passwd|password|passphrase|credentials?)(?:\\?["'])?\s*[:=]\s*)(\\"([^"\\\r\n]{12,})\\"|"((?:[^"\\\r\n]|\\[^\r\n]){12,})"|'([^'\\\r\n]{12,})'|([A-Za-z0-9+/_.=-]{16,}))/gi;
|
|
175
|
+
var BENIGN_KEY_NAME_PATTERN = /(?:^|[._-])(?:idempotency|dedupe|route)[_-]?key$/i;
|
|
176
|
+
var ASSIGNMENT_HINTS = ["key", "token", "secret", "pass", "credential"];
|
|
177
|
+
function mayContainAssignment(text) {
|
|
178
|
+
const lowered = text.toLowerCase();
|
|
179
|
+
return ASSIGNMENT_HINTS.some((hint) => lowered.includes(hint));
|
|
180
|
+
}
|
|
181
|
+
function shannonEntropy(value) {
|
|
182
|
+
const counts = new Map;
|
|
183
|
+
for (const char of value)
|
|
184
|
+
counts.set(char, (counts.get(char) ?? 0) + 1);
|
|
185
|
+
let entropy = 0;
|
|
186
|
+
for (const count of counts.values()) {
|
|
187
|
+
const probability = count / value.length;
|
|
188
|
+
entropy -= probability * Math.log2(probability);
|
|
189
|
+
}
|
|
190
|
+
return entropy;
|
|
191
|
+
}
|
|
192
|
+
function isLikelySecretValue(value) {
|
|
193
|
+
if (value.length < 12)
|
|
194
|
+
return false;
|
|
195
|
+
if (value.includes(SCRUBBED))
|
|
196
|
+
return false;
|
|
197
|
+
if (value.startsWith("/") || value.startsWith("~") || value.startsWith("./"))
|
|
198
|
+
return false;
|
|
199
|
+
return shannonEntropy(value) >= 3.2;
|
|
200
|
+
}
|
|
201
|
+
function scrubSecrets(text) {
|
|
202
|
+
if (!text)
|
|
203
|
+
return text;
|
|
204
|
+
let scrubbed = text;
|
|
205
|
+
for (const pattern of TOKEN_PATTERNS)
|
|
206
|
+
scrubbed = scrubbed.replace(pattern, SCRUBBED);
|
|
207
|
+
scrubbed = scrubbed.replace(AUTHORIZATION_PATTERN, `$1${SCRUBBED}`);
|
|
208
|
+
scrubbed = scrubbed.replace(PEM_PATTERN, SCRUBBED);
|
|
209
|
+
if (!mayContainAssignment(scrubbed))
|
|
210
|
+
return scrubbed;
|
|
211
|
+
scrubbed = scrubbed.replace(ASSIGNMENT_PATTERN, (match, prefix, quotedValue, escapedQuoted, doubleQuoted, singleQuoted, bare) => {
|
|
212
|
+
const value = escapedQuoted ?? doubleQuoted ?? singleQuoted ?? bare ?? "";
|
|
213
|
+
const keyName = /^[A-Za-z0-9_.-]*/.exec(prefix)?.[0] ?? "";
|
|
214
|
+
if (BENIGN_KEY_NAME_PATTERN.test(keyName))
|
|
215
|
+
return match;
|
|
216
|
+
if (!isLikelySecretValue(value))
|
|
217
|
+
return match;
|
|
218
|
+
const quote = quotedValue.startsWith("\\\"") ? "\\\"" : quotedValue.startsWith('"') ? '"' : quotedValue.startsWith("'") ? "'" : "";
|
|
219
|
+
return `${prefix}${quote}${SCRUBBED}${quote}`;
|
|
220
|
+
});
|
|
221
|
+
return scrubbed;
|
|
222
|
+
}
|
|
223
|
+
function scrubSecretsDeep(value) {
|
|
224
|
+
if (typeof value === "string")
|
|
225
|
+
return scrubSecrets(value);
|
|
226
|
+
if (Array.isArray(value))
|
|
227
|
+
return value.map((entry) => scrubSecretsDeep(entry));
|
|
228
|
+
if (value !== null && typeof value === "object") {
|
|
229
|
+
const toJSON = value.toJSON;
|
|
230
|
+
if (typeof toJSON === "function")
|
|
231
|
+
return scrubSecretsDeep(toJSON.call(value));
|
|
232
|
+
const scrubbed = {};
|
|
233
|
+
for (const [key, entry] of Object.entries(value))
|
|
234
|
+
scrubbed[key] = scrubSecretsDeep(entry);
|
|
235
|
+
return scrubbed;
|
|
236
|
+
}
|
|
237
|
+
return value;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/lib/recurrence.ts
|
|
46
241
|
function assertDate(value, label) {
|
|
47
242
|
const date = new Date(value);
|
|
48
243
|
if (Number.isNaN(date.getTime()))
|
|
@@ -79,7 +274,20 @@ function parseField(expr, min, max) {
|
|
|
79
274
|
}
|
|
80
275
|
return out;
|
|
81
276
|
}
|
|
277
|
+
var MINUTE_MS = 60000;
|
|
278
|
+
var PARSED_CRON_CACHE_LIMIT = 256;
|
|
279
|
+
var parsedCronCache = new Map;
|
|
280
|
+
var emittedScheduleWarnings = new Set;
|
|
281
|
+
function warnOnce(key, message) {
|
|
282
|
+
if (emittedScheduleWarnings.has(key))
|
|
283
|
+
return;
|
|
284
|
+
emittedScheduleWarnings.add(key);
|
|
285
|
+
console.warn(`[open-loops] WARN ${message}`);
|
|
286
|
+
}
|
|
82
287
|
function parseCron(expr) {
|
|
288
|
+
const cached = parsedCronCache.get(expr);
|
|
289
|
+
if (cached)
|
|
290
|
+
return cached;
|
|
83
291
|
const fields = expr.trim().split(/\s+/);
|
|
84
292
|
if (fields.length !== 5)
|
|
85
293
|
throw new Error(`cron must have 5 fields, got ${fields.length}: "${expr}"`);
|
|
@@ -89,7 +297,7 @@ function parseCron(expr) {
|
|
|
89
297
|
dowSet.delete(7);
|
|
90
298
|
dowSet.add(0);
|
|
91
299
|
}
|
|
92
|
-
|
|
300
|
+
const parsed = {
|
|
93
301
|
minute: parseField(minute, 0, 59),
|
|
94
302
|
hour: parseField(hour, 0, 23),
|
|
95
303
|
dom: parseField(dom, 1, 31),
|
|
@@ -98,6 +306,13 @@ function parseCron(expr) {
|
|
|
98
306
|
domRestricted: dom !== "*",
|
|
99
307
|
dowRestricted: dow !== "*"
|
|
100
308
|
};
|
|
309
|
+
if (parsedCronCache.size >= PARSED_CRON_CACHE_LIMIT) {
|
|
310
|
+
const oldest = parsedCronCache.keys().next().value;
|
|
311
|
+
if (oldest !== undefined)
|
|
312
|
+
parsedCronCache.delete(oldest);
|
|
313
|
+
}
|
|
314
|
+
parsedCronCache.set(expr, parsed);
|
|
315
|
+
return parsed;
|
|
101
316
|
}
|
|
102
317
|
function cronMatches(cron, date) {
|
|
103
318
|
if (!cron.minute.has(date.getMinutes()))
|
|
@@ -171,14 +386,59 @@ function latestIntervalSlot(first, now, everyMs) {
|
|
|
171
386
|
const steps = Math.floor((now.getTime() - first.getTime()) / everyMs);
|
|
172
387
|
return new Date(first.getTime() + steps * everyMs);
|
|
173
388
|
}
|
|
389
|
+
function floorToMinute(date) {
|
|
390
|
+
const floored = new Date(date.getTime());
|
|
391
|
+
floored.setSeconds(0, 0);
|
|
392
|
+
return floored;
|
|
393
|
+
}
|
|
394
|
+
var LATEST_CRON_SCAN_LIMIT_MINUTES = 366 * 24 * 60;
|
|
395
|
+
function latestDailyCronSlot(cron, now) {
|
|
396
|
+
const hoursDesc = [...cron.hour].sort((a, b) => b - a);
|
|
397
|
+
const minutesDesc = [...cron.minute].sort((a, b) => b - a);
|
|
398
|
+
const candidate = floorToMinute(now);
|
|
399
|
+
for (let dayOffset = 0;dayOffset < 2; dayOffset += 1) {
|
|
400
|
+
const isToday = dayOffset === 0;
|
|
401
|
+
for (const hour of hoursDesc) {
|
|
402
|
+
if (isToday && hour > now.getHours())
|
|
403
|
+
continue;
|
|
404
|
+
const minuteCap = isToday && hour === now.getHours() ? now.getMinutes() : 59;
|
|
405
|
+
const minute = minutesDesc.find((value) => value <= minuteCap);
|
|
406
|
+
if (minute === undefined)
|
|
407
|
+
continue;
|
|
408
|
+
candidate.setHours(hour, minute, 0, 0);
|
|
409
|
+
return candidate;
|
|
410
|
+
}
|
|
411
|
+
candidate.setDate(candidate.getDate() - 1);
|
|
412
|
+
}
|
|
413
|
+
throw new Error("unreachable: daily cron pattern must match within two days");
|
|
414
|
+
}
|
|
174
415
|
function latestCronSlot(first, now, expression) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
416
|
+
if (first.getTime() > now.getTime())
|
|
417
|
+
return first;
|
|
418
|
+
const cron = parseCron(expression);
|
|
419
|
+
if (!cron.domRestricted && !cron.dowRestricted && cron.month.size === 12) {
|
|
420
|
+
const latest = latestDailyCronSlot(cron, now);
|
|
421
|
+
return latest.getTime() >= first.getTime() ? latest : first;
|
|
422
|
+
}
|
|
423
|
+
const scanFloorMs = Math.max(first.getTime(), now.getTime() - LATEST_CRON_SCAN_LIMIT_MINUTES * MINUTE_MS);
|
|
424
|
+
const cursor = floorToMinute(now);
|
|
425
|
+
while (cursor.getTime() >= scanFloorMs) {
|
|
426
|
+
if (cronMatches(cron, cursor))
|
|
427
|
+
return cursor;
|
|
428
|
+
cursor.setMinutes(cursor.getMinutes() - 1);
|
|
181
429
|
}
|
|
430
|
+
if (first.getTime() < scanFloorMs) {
|
|
431
|
+
warnOnce(`latest-cron:${expression}`, `latestCronSlot: no match for "${expression}" within the ${LATEST_CRON_SCAN_LIMIT_MINUTES}-minute scan window; using the stored next run as the latest slot`);
|
|
432
|
+
}
|
|
433
|
+
return first;
|
|
434
|
+
}
|
|
435
|
+
var MAX_CATCH_UP_SLOTS = 1000;
|
|
436
|
+
function catchUpSlotLimit(loop) {
|
|
437
|
+
if (loop.catchUpLimit > MAX_CATCH_UP_SLOTS) {
|
|
438
|
+
warnOnce(`catch-up-limit:${loop.id}`, `dueSlots: loop ${loop.id} catchUpLimit ${loop.catchUpLimit} exceeds the per-plan cap; using ${MAX_CATCH_UP_SLOTS}`);
|
|
439
|
+
return MAX_CATCH_UP_SLOTS;
|
|
440
|
+
}
|
|
441
|
+
return loop.catchUpLimit;
|
|
182
442
|
}
|
|
183
443
|
function dueSlots(loop, now) {
|
|
184
444
|
if (!loop.nextRunAt || loop.status !== "active")
|
|
@@ -197,9 +457,10 @@ function dueSlots(loop, now) {
|
|
|
197
457
|
return { slots: [next.toISOString()] };
|
|
198
458
|
case "interval": {
|
|
199
459
|
if (catchUp === "all") {
|
|
460
|
+
const limit = catchUpSlotLimit(loop);
|
|
200
461
|
const slots = [];
|
|
201
462
|
let cursor = next;
|
|
202
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
463
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
203
464
|
slots.push(cursor.toISOString());
|
|
204
465
|
cursor = new Date(cursor.getTime() + loop.schedule.everyMs);
|
|
205
466
|
}
|
|
@@ -211,9 +472,10 @@ function dueSlots(loop, now) {
|
|
|
211
472
|
}
|
|
212
473
|
case "cron": {
|
|
213
474
|
if (catchUp === "all") {
|
|
475
|
+
const limit = catchUpSlotLimit(loop);
|
|
214
476
|
const slots = [];
|
|
215
477
|
let cursor = next;
|
|
216
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
478
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
217
479
|
slots.push(cursor.toISOString());
|
|
218
480
|
cursor = nextCronRun(loop.schedule.expression, cursor);
|
|
219
481
|
}
|
|
@@ -312,8 +574,311 @@ function updateReadyFlags(nodes, planStatus) {
|
|
|
312
574
|
}
|
|
313
575
|
|
|
314
576
|
// src/lib/workflow-spec.ts
|
|
315
|
-
import { readFileSync } from "fs";
|
|
577
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
316
578
|
import { isAbsolute, resolve } from "path";
|
|
579
|
+
|
|
580
|
+
// src/lib/agent-adapter.ts
|
|
581
|
+
import { spawn } from "child_process";
|
|
582
|
+
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS = new Set([
|
|
583
|
+
"e",
|
|
584
|
+
"exec",
|
|
585
|
+
"agent",
|
|
586
|
+
"start",
|
|
587
|
+
"--ephemeral",
|
|
588
|
+
"--ignore-rules",
|
|
589
|
+
"--skip-git-repo-check",
|
|
590
|
+
"--json",
|
|
591
|
+
"--output-last-message",
|
|
592
|
+
"-o",
|
|
593
|
+
"--output-schema",
|
|
594
|
+
"--dangerously-bypass-approvals-and-sandbox"
|
|
595
|
+
]);
|
|
596
|
+
var CODEX_LIKE_SANDBOXES = ["read-only", "workspace-write", "danger-full-access"];
|
|
597
|
+
var CURSOR_SANDBOXES = ["enabled", "disabled"];
|
|
598
|
+
var PERMISSION_MODES = ["default", "plan", "auto", "bypass"];
|
|
599
|
+
function assertOptionalNonEmptyString(value, label) {
|
|
600
|
+
if (value === undefined)
|
|
601
|
+
return;
|
|
602
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
603
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
604
|
+
}
|
|
605
|
+
function validateAgentOptions(target, label, capabilities) {
|
|
606
|
+
const provider = target.provider;
|
|
607
|
+
if (typeof target.prompt !== "string" || target.prompt.trim() === "") {
|
|
608
|
+
throw new Error(`${label}.prompt must be a non-empty string`);
|
|
609
|
+
}
|
|
610
|
+
assertOptionalNonEmptyString(target.model, `${label}.model`);
|
|
611
|
+
assertOptionalNonEmptyString(target.variant, `${label}.variant`);
|
|
612
|
+
assertOptionalNonEmptyString(target.agent, `${label}.agent`);
|
|
613
|
+
assertOptionalNonEmptyString(target.authProfile, `${label}.authProfile`);
|
|
614
|
+
assertOptionalNonEmptyString(target.configIsolation, `${label}.configIsolation`);
|
|
615
|
+
if (target.configIsolation !== undefined && target.configIsolation !== "safe" && target.configIsolation !== "none") {
|
|
616
|
+
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
617
|
+
}
|
|
618
|
+
if (target.authProfile !== undefined && provider !== "codewith") {
|
|
619
|
+
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
620
|
+
}
|
|
621
|
+
if (provider === "opencode" && (typeof target.model !== "string" || target.model.trim() === "")) {
|
|
622
|
+
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
623
|
+
}
|
|
624
|
+
if (provider === "cursor" && target.variant !== undefined) {
|
|
625
|
+
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
626
|
+
}
|
|
627
|
+
if (provider === "codex" && target.agent !== undefined) {
|
|
628
|
+
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
629
|
+
}
|
|
630
|
+
if (provider === "codewith" && target.agent !== undefined) {
|
|
631
|
+
throw new Error(`${label}.agent is not supported for provider codewith durable background-agent execution`);
|
|
632
|
+
}
|
|
633
|
+
if (provider === "codewith") {
|
|
634
|
+
const unsafe = target.extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS.has(arg));
|
|
635
|
+
if (unsafe) {
|
|
636
|
+
throw new Error(`${label}.extraArgs cannot include ${unsafe}; codewith agent steps use durable agent start, not exec/ephemeral flags`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (target.addDirs?.length && !["codewith", "codex"].includes(provider)) {
|
|
640
|
+
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
641
|
+
}
|
|
642
|
+
if (target.permissionMode !== undefined) {
|
|
643
|
+
if (!PERMISSION_MODES.includes(target.permissionMode)) {
|
|
644
|
+
throw new Error(`${label}.permissionMode must be one of ${PERMISSION_MODES.join(", ")}`);
|
|
645
|
+
}
|
|
646
|
+
if (target.permissionMode === "plan" && !["claude", "cursor"].includes(provider)) {
|
|
647
|
+
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
648
|
+
}
|
|
649
|
+
if (target.permissionMode === "auto" && provider !== "claude") {
|
|
650
|
+
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (target.sandbox !== undefined) {
|
|
654
|
+
if (!capabilities.sandbox.length) {
|
|
655
|
+
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
656
|
+
}
|
|
657
|
+
if (!capabilities.sandbox.includes(target.sandbox)) {
|
|
658
|
+
throw new Error(`${label}.sandbox must be one of ${capabilities.sandbox.join(", ")}`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function codewithLikeSandbox(target) {
|
|
663
|
+
return target.sandbox ?? (target.permissionMode === "bypass" ? "danger-full-access" : "workspace-write");
|
|
664
|
+
}
|
|
665
|
+
function configStringValue(value) {
|
|
666
|
+
return JSON.stringify(value);
|
|
667
|
+
}
|
|
668
|
+
function buildAgentInvocation(target) {
|
|
669
|
+
const isolation = target.configIsolation ?? "safe";
|
|
670
|
+
const permissionMode = target.permissionMode ?? "default";
|
|
671
|
+
const args = [];
|
|
672
|
+
switch (target.provider) {
|
|
673
|
+
case "claude": {
|
|
674
|
+
if (isolation === "safe")
|
|
675
|
+
args.push("--safe-mode", "--setting-sources", "local", "--no-session-persistence");
|
|
676
|
+
if (permissionMode !== "default") {
|
|
677
|
+
const mode = permissionMode === "bypass" ? "bypassPermissions" : permissionMode;
|
|
678
|
+
args.push("--permission-mode", mode);
|
|
679
|
+
}
|
|
680
|
+
args.push("-p", "--output-format", "json");
|
|
681
|
+
if (target.model)
|
|
682
|
+
args.push("--model", target.model);
|
|
683
|
+
if (target.variant)
|
|
684
|
+
args.push("--effort", target.variant);
|
|
685
|
+
if (target.agent)
|
|
686
|
+
args.push("--agent", target.agent);
|
|
687
|
+
args.push(...target.extraArgs ?? []);
|
|
688
|
+
return { command: "claude", args, stdin: target.prompt };
|
|
689
|
+
}
|
|
690
|
+
case "cursor": {
|
|
691
|
+
args.push("-c", [
|
|
692
|
+
"set -eu",
|
|
693
|
+
"if command -v agent >/dev/null 2>&1; then",
|
|
694
|
+
' exec agent "$@"',
|
|
695
|
+
"else",
|
|
696
|
+
" echo 'Executable not found in PATH: agent' >&2",
|
|
697
|
+
" exit 127",
|
|
698
|
+
"fi"
|
|
699
|
+
].join(`
|
|
700
|
+
`), "openloops-cursor", "-p");
|
|
701
|
+
if (permissionMode === "plan")
|
|
702
|
+
args.push("--mode", "plan");
|
|
703
|
+
if (permissionMode === "bypass")
|
|
704
|
+
args.push("--force");
|
|
705
|
+
const cursorSandbox = target.sandbox ?? (isolation === "safe" ? "enabled" : undefined);
|
|
706
|
+
if (cursorSandbox)
|
|
707
|
+
args.push("--sandbox", cursorSandbox);
|
|
708
|
+
if (target.model)
|
|
709
|
+
args.push("--model", target.model);
|
|
710
|
+
if (target.agent)
|
|
711
|
+
args.push("--agent", target.agent);
|
|
712
|
+
args.push(...target.extraArgs ?? []);
|
|
713
|
+
return { command: "sh", args, stdin: target.prompt, preflightAnyOf: ["agent"] };
|
|
714
|
+
}
|
|
715
|
+
case "codewith": {
|
|
716
|
+
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : []);
|
|
717
|
+
if (target.variant)
|
|
718
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
719
|
+
args.push("--ask-for-approval", "never", "--sandbox", codewithLikeSandbox(target));
|
|
720
|
+
if (target.cwd)
|
|
721
|
+
args.push("--cd", target.cwd);
|
|
722
|
+
for (const dir of target.addDirs ?? [])
|
|
723
|
+
args.push("--add-dir", dir);
|
|
724
|
+
if (target.model)
|
|
725
|
+
args.push("--model", target.model);
|
|
726
|
+
args.push(...target.extraArgs ?? []);
|
|
727
|
+
args.push("agent", "start");
|
|
728
|
+
if (target.cwd)
|
|
729
|
+
args.push("--cwd", target.cwd);
|
|
730
|
+
args.push(target.prompt);
|
|
731
|
+
return { command: "codewith", args };
|
|
732
|
+
}
|
|
733
|
+
case "codex": {
|
|
734
|
+
if (target.variant)
|
|
735
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
736
|
+
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
737
|
+
if (isolation === "safe")
|
|
738
|
+
args.push("--ignore-rules");
|
|
739
|
+
if (target.cwd)
|
|
740
|
+
args.push("--cd", target.cwd);
|
|
741
|
+
for (const dir of target.addDirs ?? [])
|
|
742
|
+
args.push("--add-dir", dir);
|
|
743
|
+
if (target.model)
|
|
744
|
+
args.push("--model", target.model);
|
|
745
|
+
args.push(...target.extraArgs ?? []);
|
|
746
|
+
return { command: "codex", args, stdin: target.prompt };
|
|
747
|
+
}
|
|
748
|
+
case "aicopilot":
|
|
749
|
+
case "opencode": {
|
|
750
|
+
args.push("run", "--format", "json");
|
|
751
|
+
if (isolation === "safe")
|
|
752
|
+
args.push("--pure");
|
|
753
|
+
if (permissionMode === "bypass")
|
|
754
|
+
args.push("--dangerously-skip-permissions");
|
|
755
|
+
if (target.cwd)
|
|
756
|
+
args.push("--dir", target.cwd);
|
|
757
|
+
if (target.model)
|
|
758
|
+
args.push("--model", target.model);
|
|
759
|
+
if (target.variant)
|
|
760
|
+
args.push("--variant", target.variant);
|
|
761
|
+
if (target.agent)
|
|
762
|
+
args.push("--agent", target.agent);
|
|
763
|
+
args.push(...target.extraArgs ?? []);
|
|
764
|
+
return { command: target.provider, args, stdin: target.prompt };
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
function adapterFor(provider, capabilities) {
|
|
769
|
+
return {
|
|
770
|
+
provider,
|
|
771
|
+
capabilities,
|
|
772
|
+
validate(target, label = provider) {
|
|
773
|
+
validateAgentOptions(target, label, capabilities);
|
|
774
|
+
},
|
|
775
|
+
buildInvocation(target) {
|
|
776
|
+
validateAgentOptions(target, provider, capabilities);
|
|
777
|
+
return buildAgentInvocation(target);
|
|
778
|
+
}
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
var PROVIDER_ADAPTERS = {
|
|
782
|
+
claude: adapterFor("claude", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
783
|
+
cursor: adapterFor("cursor", { sandbox: CURSOR_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
784
|
+
codewith: adapterFor("codewith", { sandbox: CODEX_LIKE_SANDBOXES, durable: true, remote: false, promptChannel: "argv" }),
|
|
785
|
+
codex: adapterFor("codex", { sandbox: CODEX_LIKE_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
786
|
+
aicopilot: adapterFor("aicopilot", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
787
|
+
opencode: adapterFor("opencode", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" })
|
|
788
|
+
};
|
|
789
|
+
var AGENT_PROVIDERS = Object.keys(PROVIDER_ADAPTERS);
|
|
790
|
+
function providerAdapter(provider) {
|
|
791
|
+
const adapter = PROVIDER_ADAPTERS[provider];
|
|
792
|
+
if (!adapter)
|
|
793
|
+
throw new Error(`unsupported agent provider: ${String(provider)}`);
|
|
794
|
+
return adapter;
|
|
795
|
+
}
|
|
796
|
+
var DEFAULT_CAPTURE_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
797
|
+
function killProcessGroup(pgid) {
|
|
798
|
+
try {
|
|
799
|
+
process.kill(-pgid, "SIGTERM");
|
|
800
|
+
} catch {
|
|
801
|
+
try {
|
|
802
|
+
process.kill(pgid, "SIGTERM");
|
|
803
|
+
} catch {}
|
|
804
|
+
}
|
|
805
|
+
setTimeout(() => {
|
|
806
|
+
try {
|
|
807
|
+
process.kill(-pgid, "SIGKILL");
|
|
808
|
+
} catch {
|
|
809
|
+
try {
|
|
810
|
+
process.kill(pgid, "SIGKILL");
|
|
811
|
+
} catch {}
|
|
812
|
+
}
|
|
813
|
+
}, 2000).unref();
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
class BoundedOutputBuffer {
|
|
817
|
+
maxBytes;
|
|
818
|
+
text = "";
|
|
819
|
+
truncatedBytes = 0;
|
|
820
|
+
constructor(maxBytes) {
|
|
821
|
+
this.maxBytes = maxBytes;
|
|
822
|
+
}
|
|
823
|
+
append(chunk) {
|
|
824
|
+
if (!chunk)
|
|
825
|
+
return;
|
|
826
|
+
this.text += chunk;
|
|
827
|
+
if (Buffer.byteLength(this.text, "utf8") <= this.maxBytes)
|
|
828
|
+
return;
|
|
829
|
+
this.text = scrubSecrets(this.text);
|
|
830
|
+
const encoded = Buffer.from(this.text, "utf8");
|
|
831
|
+
if (encoded.length <= this.maxBytes)
|
|
832
|
+
return;
|
|
833
|
+
let start = encoded.length - this.maxBytes;
|
|
834
|
+
while (start < encoded.length && (encoded[start] & 192) === 128)
|
|
835
|
+
start++;
|
|
836
|
+
this.truncatedBytes += start;
|
|
837
|
+
this.text = encoded.subarray(start).toString("utf8");
|
|
838
|
+
}
|
|
839
|
+
value() {
|
|
840
|
+
if (this.truncatedBytes === 0)
|
|
841
|
+
return this.text;
|
|
842
|
+
return `[truncated ${this.truncatedBytes} bytes]
|
|
843
|
+
${this.text}`;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
async function spawnCapture(command, args, opts) {
|
|
847
|
+
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_CAPTURE_MAX_OUTPUT_BYTES;
|
|
848
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
849
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
850
|
+
let timedOut = false;
|
|
851
|
+
let error;
|
|
852
|
+
const child = spawn(command, args, {
|
|
853
|
+
cwd: opts.cwd,
|
|
854
|
+
env: opts.env,
|
|
855
|
+
detached: true,
|
|
856
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
857
|
+
});
|
|
858
|
+
const timer = setTimeout(() => {
|
|
859
|
+
timedOut = true;
|
|
860
|
+
if (child.pid)
|
|
861
|
+
killProcessGroup(child.pid);
|
|
862
|
+
}, opts.timeoutMs);
|
|
863
|
+
timer.unref();
|
|
864
|
+
child.stdout?.setEncoding("utf8");
|
|
865
|
+
child.stderr?.setEncoding("utf8");
|
|
866
|
+
child.stdout?.on("data", (chunk) => stdout.append(chunk));
|
|
867
|
+
child.stderr?.on("data", (chunk) => stderr.append(chunk));
|
|
868
|
+
const [status, signal] = await new Promise((resolve) => {
|
|
869
|
+
child.once("error", (err) => {
|
|
870
|
+
error = err.message;
|
|
871
|
+
resolve([null, null]);
|
|
872
|
+
});
|
|
873
|
+
child.once("close", (code, sig) => resolve([code, sig]));
|
|
874
|
+
});
|
|
875
|
+
clearTimeout(timer);
|
|
876
|
+
if (timedOut && !error)
|
|
877
|
+
error = `timed out after ${opts.timeoutMs}ms`;
|
|
878
|
+
return { status, signal, stdout: stdout.value(), stderr: stderr.value(), error, timedOut };
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// src/lib/workflow-spec.ts
|
|
317
882
|
function assertObject(value, label) {
|
|
318
883
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
319
884
|
throw new Error(`${label} must be an object`);
|
|
@@ -376,7 +941,7 @@ function readPromptFile(rawPath, label, opts) {
|
|
|
376
941
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
377
942
|
let prompt;
|
|
378
943
|
try {
|
|
379
|
-
prompt =
|
|
944
|
+
prompt = readFileSync2(path, "utf8");
|
|
380
945
|
} catch (error) {
|
|
381
946
|
const code = error && typeof error === "object" && "code" in error ? String(error.code) : undefined;
|
|
382
947
|
throw new Error(`${label}.promptFile could not be read${code ? `: ${code}` : ""}`);
|
|
@@ -395,7 +960,7 @@ function matchingPromptSource(value, prompt, opts) {
|
|
|
395
960
|
return;
|
|
396
961
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
397
962
|
try {
|
|
398
|
-
return
|
|
963
|
+
return readFileSync2(path, "utf8") === prompt ? { type: "file", path } : undefined;
|
|
399
964
|
} catch {
|
|
400
965
|
return;
|
|
401
966
|
}
|
|
@@ -442,65 +1007,14 @@ function validateTarget(value, label, opts) {
|
|
|
442
1007
|
if (!hasPrompt && !hasPromptFile)
|
|
443
1008
|
throw new Error(`${label}.prompt must be a non-empty string or ${label}.promptFile must be set`);
|
|
444
1009
|
const promptFields = hasPrompt ? { prompt: value.prompt, promptSource: matchingPromptSource(value.promptSource, value.prompt, opts) } : readPromptFile(value.promptFile, label, opts);
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
1010
|
+
if (!AGENT_PROVIDERS.includes(value.provider)) {
|
|
1011
|
+
throw new Error(`${label}.provider must be one of ${AGENT_PROVIDERS.join(", ")}`);
|
|
1012
|
+
}
|
|
448
1013
|
optionalTimeoutMs(value.timeoutMs, `${label}.timeoutMs`);
|
|
449
1014
|
optionalPositiveInteger(value.idleTimeoutMs, `${label}.idleTimeoutMs`);
|
|
450
|
-
|
|
451
|
-
assertString(value.authProfile, `${label}.authProfile`);
|
|
452
|
-
if (value.provider !== "codewith")
|
|
453
|
-
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
454
|
-
}
|
|
455
|
-
if (value.configIsolation !== undefined) {
|
|
456
|
-
assertString(value.configIsolation, `${label}.configIsolation`);
|
|
457
|
-
if (!["safe", "none"].includes(value.configIsolation))
|
|
458
|
-
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
459
|
-
}
|
|
460
|
-
if (value.model !== undefined)
|
|
461
|
-
assertString(value.model, `${label}.model`);
|
|
462
|
-
if (value.provider === "opencode" && (typeof value.model !== "string" || value.model.trim() === "")) {
|
|
463
|
-
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
464
|
-
}
|
|
465
|
-
if (value.variant !== undefined)
|
|
466
|
-
assertString(value.variant, `${label}.variant`);
|
|
467
|
-
if (value.agent !== undefined)
|
|
468
|
-
assertString(value.agent, `${label}.agent`);
|
|
469
|
-
if (value.provider === "cursor" && value.variant !== undefined)
|
|
470
|
-
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
471
|
-
if (value.provider === "codex" && value.agent !== undefined)
|
|
472
|
-
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
1015
|
+
const extraArgs = optionalStringArray(value.extraArgs, `${label}.extraArgs`);
|
|
473
1016
|
optionalStringArray(value.addDirs, `${label}.addDirs`);
|
|
474
|
-
|
|
475
|
-
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
476
|
-
}
|
|
477
|
-
if (value.permissionMode !== undefined) {
|
|
478
|
-
assertString(value.permissionMode, `${label}.permissionMode`);
|
|
479
|
-
const permissionModes = ["default", "plan", "auto", "bypass"];
|
|
480
|
-
if (!permissionModes.includes(value.permissionMode)) {
|
|
481
|
-
throw new Error(`${label}.permissionMode must be one of ${permissionModes.join(", ")}`);
|
|
482
|
-
}
|
|
483
|
-
if (value.permissionMode === "plan" && !["claude", "cursor"].includes(value.provider)) {
|
|
484
|
-
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
485
|
-
}
|
|
486
|
-
if (value.permissionMode === "auto" && value.provider !== "claude") {
|
|
487
|
-
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
if (value.sandbox !== undefined) {
|
|
491
|
-
assertString(value.sandbox, `${label}.sandbox`);
|
|
492
|
-
const codexLike = ["read-only", "workspace-write", "danger-full-access"];
|
|
493
|
-
const cursorLike = ["enabled", "disabled"];
|
|
494
|
-
if (["codewith", "codex"].includes(value.provider)) {
|
|
495
|
-
if (!codexLike.includes(value.sandbox))
|
|
496
|
-
throw new Error(`${label}.sandbox must be one of ${codexLike.join(", ")}`);
|
|
497
|
-
} else if (value.provider === "cursor") {
|
|
498
|
-
if (!cursorLike.includes(value.sandbox))
|
|
499
|
-
throw new Error(`${label}.sandbox must be one of ${cursorLike.join(", ")}`);
|
|
500
|
-
} else {
|
|
501
|
-
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
502
|
-
}
|
|
503
|
-
}
|
|
1017
|
+
providerAdapter(value.provider).validate({ ...value, extraArgs, ...promptFields }, label);
|
|
504
1018
|
if (value.allowlist !== undefined) {
|
|
505
1019
|
assertObject(value.allowlist, `${label}.allowlist`);
|
|
506
1020
|
optionalStringArray(value.allowlist.tools, `${label}.allowlist.tools`);
|
|
@@ -545,7 +1059,9 @@ function validateTarget(value, label, opts) {
|
|
|
545
1059
|
if (value.routing.eventSource !== undefined)
|
|
546
1060
|
assertString(value.routing.eventSource, `${label}.routing.eventSource`);
|
|
547
1061
|
}
|
|
548
|
-
const target = { ...value };
|
|
1062
|
+
const target = { ...value, extraArgs };
|
|
1063
|
+
if (!extraArgs)
|
|
1064
|
+
delete target.extraArgs;
|
|
549
1065
|
delete target.promptFile;
|
|
550
1066
|
delete target.promptSource;
|
|
551
1067
|
return { ...target, ...promptFields };
|
|
@@ -628,8 +1144,8 @@ function workflowBodyFromJson(value, fallbackName, opts = {}) {
|
|
|
628
1144
|
|
|
629
1145
|
// src/lib/run-artifacts.ts
|
|
630
1146
|
import { createHash } from "crypto";
|
|
631
|
-
import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
|
|
632
|
-
import { basename, join as join2 } from "path";
|
|
1147
|
+
import { mkdirSync as mkdirSync2, renameSync, rmdirSync, rmSync, writeFileSync } from "fs";
|
|
1148
|
+
import { basename, dirname, join as join2 } from "path";
|
|
633
1149
|
function shortHash(value) {
|
|
634
1150
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
635
1151
|
}
|
|
@@ -650,13 +1166,14 @@ function workflowRunProjectSlug(projectKey) {
|
|
|
650
1166
|
return "global";
|
|
651
1167
|
return safeRunPathSlug(projectKey.startsWith("/") ? basename(projectKey) : projectKey, "project");
|
|
652
1168
|
}
|
|
653
|
-
function
|
|
1169
|
+
function stageWorkflowRunManifest(args) {
|
|
654
1170
|
const projectSlug = workflowRunProjectSlug(args.projectKey);
|
|
655
1171
|
const subjectKey = workflowRunSubjectKey(args.subjectKind, args.rawSubjectRef);
|
|
656
1172
|
const dir = join2(args.loopsDataDir, "runs", projectSlug, subjectKey, args.workflowRunId);
|
|
657
1173
|
mkdirSync2(dir, { recursive: true, mode: 448 });
|
|
658
1174
|
const manifestPath = join2(dir, "manifest.json");
|
|
659
|
-
|
|
1175
|
+
const tmpPath = `${manifestPath}.tmp`;
|
|
1176
|
+
writeFileSync(tmpPath, JSON.stringify({
|
|
660
1177
|
version: 1,
|
|
661
1178
|
workflowRunId: args.workflowRunId,
|
|
662
1179
|
workflowId: args.workflowId,
|
|
@@ -669,13 +1186,26 @@ function writeWorkflowRunManifest(args) {
|
|
|
669
1186
|
createdAt: new Date().toISOString(),
|
|
670
1187
|
...args.payload
|
|
671
1188
|
}, null, 2), { mode: 384 });
|
|
672
|
-
return manifestPath;
|
|
1189
|
+
return { manifestPath, tmpPath };
|
|
1190
|
+
}
|
|
1191
|
+
function commitWorkflowRunManifest(staged) {
|
|
1192
|
+
renameSync(staged.tmpPath, staged.manifestPath);
|
|
1193
|
+
return staged.manifestPath;
|
|
1194
|
+
}
|
|
1195
|
+
function discardWorkflowRunManifest(staged) {
|
|
1196
|
+
rmSync(staged.tmpPath, { force: true });
|
|
1197
|
+
try {
|
|
1198
|
+
rmdirSync(dirname(staged.manifestPath));
|
|
1199
|
+
} catch {}
|
|
673
1200
|
}
|
|
674
1201
|
|
|
675
1202
|
// src/lib/store.ts
|
|
676
1203
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
677
1204
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
678
1205
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1206
|
+
var SCHEMA_USER_VERSION = 6;
|
|
1207
|
+
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1208
|
+
var PRUNE_BATCH_SIZE = 400;
|
|
679
1209
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
680
1210
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
681
1211
|
function rowToLoop(row) {
|
|
@@ -716,6 +1246,8 @@ function rowToRun(row) {
|
|
|
716
1246
|
claimedBy: row.claimed_by ?? undefined,
|
|
717
1247
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
718
1248
|
pid: row.pid ?? undefined,
|
|
1249
|
+
pgid: row.pgid ?? undefined,
|
|
1250
|
+
processStartedAt: row.process_started_at ?? undefined,
|
|
719
1251
|
exitCode: row.exit_code ?? undefined,
|
|
720
1252
|
durationMs: row.duration_ms ?? undefined,
|
|
721
1253
|
stdout: row.stdout ?? undefined,
|
|
@@ -901,6 +1433,24 @@ function isProcessAlive(pid) {
|
|
|
901
1433
|
return false;
|
|
902
1434
|
}
|
|
903
1435
|
}
|
|
1436
|
+
function isRecordedProcessAlive(pid, processStartedAt) {
|
|
1437
|
+
if (!pid || !isProcessAlive(pid))
|
|
1438
|
+
return false;
|
|
1439
|
+
return sameProcessStart(processStartedAt ?? undefined, processStartTimeMs(pid));
|
|
1440
|
+
}
|
|
1441
|
+
function isoProcessStart(pid) {
|
|
1442
|
+
const startedMs = processStartTimeMs(pid);
|
|
1443
|
+
return startedMs === undefined ? undefined : new Date(startedMs).toISOString();
|
|
1444
|
+
}
|
|
1445
|
+
function isLiveStepProcess(pid, stepStartedAt) {
|
|
1446
|
+
if (!isProcessAlive(pid))
|
|
1447
|
+
return false;
|
|
1448
|
+
const actualMs = processStartTimeMs(pid);
|
|
1449
|
+
const stepStartMs = stepStartedAt ? Date.parse(stepStartedAt) : Number.NaN;
|
|
1450
|
+
if (actualMs === undefined || !Number.isFinite(stepStartMs))
|
|
1451
|
+
return true;
|
|
1452
|
+
return actualMs >= stepStartMs - START_TIME_TOLERANCE_MS;
|
|
1453
|
+
}
|
|
904
1454
|
function rowToLease(row) {
|
|
905
1455
|
return {
|
|
906
1456
|
id: row.id,
|
|
@@ -920,6 +1470,9 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
920
1470
|
}
|
|
921
1471
|
return;
|
|
922
1472
|
}
|
|
1473
|
+
function scrubbedOrNull(value) {
|
|
1474
|
+
return value == null ? null : scrubSecrets(value);
|
|
1475
|
+
}
|
|
923
1476
|
function chmodIfExists(path, mode) {
|
|
924
1477
|
try {
|
|
925
1478
|
if (existsSync(path))
|
|
@@ -927,7 +1480,7 @@ function chmodIfExists(path, mode) {
|
|
|
927
1480
|
} catch {}
|
|
928
1481
|
}
|
|
929
1482
|
function ensurePrivateStorePath(file) {
|
|
930
|
-
const dir =
|
|
1483
|
+
const dir = dirname2(file);
|
|
931
1484
|
mkdirSync3(dir, { recursive: true, mode: 448 });
|
|
932
1485
|
chmodIfExists(dir, 448);
|
|
933
1486
|
chmodIfExists(file, 384);
|
|
@@ -942,14 +1495,19 @@ class Store {
|
|
|
942
1495
|
const file = path ?? dbPath();
|
|
943
1496
|
if (file !== ":memory:")
|
|
944
1497
|
ensurePrivateStorePath(file);
|
|
945
|
-
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) :
|
|
1498
|
+
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
|
|
946
1499
|
this.db = new Database(file);
|
|
947
1500
|
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
948
1501
|
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
949
1502
|
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
950
1503
|
if (file !== ":memory:")
|
|
951
1504
|
ensurePrivateStorePath(file);
|
|
952
|
-
|
|
1505
|
+
try {
|
|
1506
|
+
this.migrate();
|
|
1507
|
+
} catch (error) {
|
|
1508
|
+
this.db.close();
|
|
1509
|
+
throw error;
|
|
1510
|
+
}
|
|
953
1511
|
}
|
|
954
1512
|
migrate() {
|
|
955
1513
|
this.db.exec(`
|
|
@@ -957,16 +1515,78 @@ class Store {
|
|
|
957
1515
|
id TEXT PRIMARY KEY,
|
|
958
1516
|
applied_at TEXT NOT NULL
|
|
959
1517
|
);
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1518
|
+
`);
|
|
1519
|
+
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1520
|
+
const userVersion = versionRow?.user_version ?? 0;
|
|
1521
|
+
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1522
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1523
|
+
}
|
|
1524
|
+
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1525
|
+
for (const migration of this.migrations()) {
|
|
1526
|
+
if (!migration.baseline && applied.has(migration.id))
|
|
1527
|
+
continue;
|
|
1528
|
+
migration.apply();
|
|
1529
|
+
if (!applied.has(migration.id)) {
|
|
1530
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
if (userVersion !== SCHEMA_USER_VERSION)
|
|
1534
|
+
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
1535
|
+
}
|
|
1536
|
+
migrations() {
|
|
1537
|
+
return [
|
|
1538
|
+
{ id: "0001_initial_and_workflows", baseline: true, apply: () => this.createBaseSchema() },
|
|
1539
|
+
{ id: "0002_loop_machines", baseline: true, apply: () => this.addColumnIfMissing("loops", "machine_json", "TEXT") },
|
|
1540
|
+
{
|
|
1541
|
+
id: "0003_goals",
|
|
1542
|
+
baseline: true,
|
|
1543
|
+
apply: () => {
|
|
1544
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1545
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1546
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1547
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1548
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1549
|
+
}
|
|
1550
|
+
},
|
|
1551
|
+
{
|
|
1552
|
+
id: "0004_loop_archive_metadata",
|
|
1553
|
+
baseline: true,
|
|
1554
|
+
apply: () => {
|
|
1555
|
+
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1556
|
+
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1557
|
+
}
|
|
1558
|
+
},
|
|
1559
|
+
{
|
|
1560
|
+
id: "0005_workflow_invocations_and_admission",
|
|
1561
|
+
baseline: true,
|
|
1562
|
+
apply: () => {
|
|
1563
|
+
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1564
|
+
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1565
|
+
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1566
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1567
|
+
this.createWorkflowRunBackfillIndexes();
|
|
1568
|
+
}
|
|
1569
|
+
},
|
|
1570
|
+
{
|
|
1571
|
+
id: "0006_run_process_tracking",
|
|
1572
|
+
apply: () => {
|
|
1573
|
+
this.addColumnIfMissing("loop_runs", "pgid", "INTEGER");
|
|
1574
|
+
this.addColumnIfMissing("loop_runs", "process_started_at", "TEXT");
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
];
|
|
1578
|
+
}
|
|
1579
|
+
createBaseSchema() {
|
|
1580
|
+
this.db.exec(`
|
|
1581
|
+
CREATE TABLE IF NOT EXISTS loops (
|
|
1582
|
+
id TEXT PRIMARY KEY,
|
|
1583
|
+
name TEXT NOT NULL,
|
|
1584
|
+
description TEXT,
|
|
1585
|
+
status TEXT NOT NULL,
|
|
1586
|
+
archived_at TEXT,
|
|
1587
|
+
archived_from_status TEXT,
|
|
1588
|
+
schedule_json TEXT NOT NULL,
|
|
1589
|
+
target_json TEXT NOT NULL,
|
|
970
1590
|
goal_json TEXT,
|
|
971
1591
|
machine_json TEXT,
|
|
972
1592
|
next_run_at TEXT,
|
|
@@ -996,6 +1616,8 @@ class Store {
|
|
|
996
1616
|
claimed_by TEXT,
|
|
997
1617
|
lease_expires_at TEXT,
|
|
998
1618
|
pid INTEGER,
|
|
1619
|
+
pgid INTEGER,
|
|
1620
|
+
process_started_at TEXT,
|
|
999
1621
|
exit_code INTEGER,
|
|
1000
1622
|
duration_ms INTEGER,
|
|
1001
1623
|
stdout TEXT,
|
|
@@ -1220,24 +1842,6 @@ class Store {
|
|
|
1220
1842
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
1221
1843
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
1222
1844
|
`);
|
|
1223
|
-
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
1224
|
-
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1225
|
-
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1226
|
-
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1227
|
-
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1228
|
-
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1229
|
-
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1230
|
-
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1231
|
-
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1232
|
-
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1233
|
-
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1234
|
-
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1235
|
-
this.createWorkflowRunBackfillIndexes();
|
|
1236
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
1237
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
1238
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
1239
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0004_loop_archive_metadata", nowIso());
|
|
1240
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0005_workflow_invocations_and_admission", nowIso());
|
|
1241
1845
|
}
|
|
1242
1846
|
addColumnIfMissing(table, column, definition) {
|
|
1243
1847
|
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
@@ -1251,6 +1855,9 @@ class Store {
|
|
|
1251
1855
|
CREATE INDEX IF NOT EXISTS idx_workflow_runs_work_item ON workflow_runs(work_item_id);
|
|
1252
1856
|
`);
|
|
1253
1857
|
}
|
|
1858
|
+
transact(fn) {
|
|
1859
|
+
return this.db.inTransaction ? fn() : this.writeTransaction(fn);
|
|
1860
|
+
}
|
|
1254
1861
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
1255
1862
|
if (!opts.daemonLeaseId)
|
|
1256
1863
|
return;
|
|
@@ -1332,14 +1939,14 @@ class Store {
|
|
|
1332
1939
|
return byId;
|
|
1333
1940
|
const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1334
1941
|
if (rows.length === 0)
|
|
1335
|
-
throw new
|
|
1942
|
+
throw new LoopNotFoundError(idOrName);
|
|
1336
1943
|
if (rows.length > 1)
|
|
1337
|
-
throw new
|
|
1944
|
+
throw new AmbiguousNameError(idOrName);
|
|
1338
1945
|
return rowToLoop(rows[0]);
|
|
1339
1946
|
}
|
|
1340
1947
|
requireLoop(idOrName) {
|
|
1341
1948
|
return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
|
|
1342
|
-
throw new
|
|
1949
|
+
throw new LoopNotFoundError(idOrName);
|
|
1343
1950
|
})();
|
|
1344
1951
|
}
|
|
1345
1952
|
listLoops(opts = {}) {
|
|
@@ -1376,7 +1983,9 @@ class Store {
|
|
|
1376
1983
|
try {
|
|
1377
1984
|
const current = this.getLoop(id);
|
|
1378
1985
|
if (!current)
|
|
1379
|
-
throw new
|
|
1986
|
+
throw new LoopNotFoundError(id);
|
|
1987
|
+
if (current.archivedAt)
|
|
1988
|
+
throw new LoopArchivedError(current.name || id);
|
|
1380
1989
|
const merged = { ...current, ...patch, updatedAt: updated };
|
|
1381
1990
|
const res = this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
1382
1991
|
expires_at=$expiresAt, updated_at=$updated
|
|
@@ -1608,10 +2217,10 @@ class Store {
|
|
|
1608
2217
|
renameLoop(id, name, opts = {}) {
|
|
1609
2218
|
const current = this.getLoop(id);
|
|
1610
2219
|
if (!current)
|
|
1611
|
-
throw new
|
|
2220
|
+
throw new LoopNotFoundError(id);
|
|
1612
2221
|
const trimmed = name.trim();
|
|
1613
2222
|
if (!trimmed)
|
|
1614
|
-
throw new
|
|
2223
|
+
throw new ValidationError("loop name must not be empty");
|
|
1615
2224
|
const updated = (opts.now ?? new Date).toISOString();
|
|
1616
2225
|
this.db.query(`UPDATE loops SET name=$name, updated_at=$updated
|
|
1617
2226
|
WHERE id=$id
|
|
@@ -1630,25 +2239,27 @@ class Store {
|
|
|
1630
2239
|
return after;
|
|
1631
2240
|
}
|
|
1632
2241
|
archiveLoop(idOrName) {
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
2242
|
+
return this.transact(() => {
|
|
2243
|
+
const loop = this.requireLoop(idOrName);
|
|
2244
|
+
if (loop.archivedAt)
|
|
2245
|
+
return loop;
|
|
2246
|
+
const updated = nowIso();
|
|
2247
|
+
const archivedStatus = loop.status === "active" ? "paused" : loop.status;
|
|
2248
|
+
this.db.query(`UPDATE loops
|
|
2249
|
+
SET status=$status, archived_at=$archivedAt, archived_from_status=$archivedFromStatus, updated_at=$updated
|
|
2250
|
+
WHERE id=$id`).run({
|
|
2251
|
+
$id: loop.id,
|
|
2252
|
+
$status: archivedStatus,
|
|
2253
|
+
$archivedAt: updated,
|
|
2254
|
+
$archivedFromStatus: loop.status,
|
|
2255
|
+
$updated: updated
|
|
2256
|
+
});
|
|
2257
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
2258
|
+
const archived = this.getLoop(loop.id);
|
|
2259
|
+
if (!archived)
|
|
2260
|
+
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
2261
|
+
return archived;
|
|
1646
2262
|
});
|
|
1647
|
-
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
1648
|
-
const archived = this.getLoop(loop.id);
|
|
1649
|
-
if (!archived)
|
|
1650
|
-
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
1651
|
-
return archived;
|
|
1652
2263
|
}
|
|
1653
2264
|
unarchiveLoop(idOrName) {
|
|
1654
2265
|
const loop = this.requireLoop(idOrName);
|
|
@@ -1669,10 +2280,12 @@ class Store {
|
|
|
1669
2280
|
return unarchived;
|
|
1670
2281
|
}
|
|
1671
2282
|
deleteLoop(idOrName) {
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
2283
|
+
return this.transact(() => {
|
|
2284
|
+
const loop = this.requireLoop(idOrName);
|
|
2285
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
|
|
2286
|
+
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
2287
|
+
return res.changes > 0;
|
|
2288
|
+
});
|
|
1676
2289
|
}
|
|
1677
2290
|
createWorkflow(input) {
|
|
1678
2291
|
const normalized = normalizeCreateWorkflowInput(input);
|
|
@@ -2189,26 +2802,7 @@ class Store {
|
|
|
2189
2802
|
try {
|
|
2190
2803
|
this.assertDaemonLeaseFence(opts, now);
|
|
2191
2804
|
for (const node of withReady) {
|
|
2192
|
-
this.
|
|
2193
|
-
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2194
|
-
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2195
|
-
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2196
|
-
$id: node.nodeId,
|
|
2197
|
-
$goalId: goal.goalId,
|
|
2198
|
-
$planId: goal.planId,
|
|
2199
|
-
$key: node.key,
|
|
2200
|
-
$sequence: node.sequence,
|
|
2201
|
-
$priority: node.priority,
|
|
2202
|
-
$objective: node.objective,
|
|
2203
|
-
$status: node.status,
|
|
2204
|
-
$ready: node.ready ? 1 : 0,
|
|
2205
|
-
$tokenBudget: node.tokenBudget ?? null,
|
|
2206
|
-
$tokensUsed: node.tokensUsed,
|
|
2207
|
-
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2208
|
-
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2209
|
-
$created: node.createdAt,
|
|
2210
|
-
$updated: node.updatedAt
|
|
2211
|
-
});
|
|
2805
|
+
this.insertGoalPlanNode(goal, node);
|
|
2212
2806
|
}
|
|
2213
2807
|
this.db.exec("COMMIT");
|
|
2214
2808
|
} catch (error) {
|
|
@@ -2219,6 +2813,38 @@ class Store {
|
|
|
2219
2813
|
}
|
|
2220
2814
|
return this.listGoalPlanNodes(goalId);
|
|
2221
2815
|
}
|
|
2816
|
+
insertGoalPlanNode(goal, node) {
|
|
2817
|
+
let id = node.nodeId;
|
|
2818
|
+
for (let attempt = 0;attempt < 3; attempt += 1) {
|
|
2819
|
+
const res = this.db.query(`INSERT OR IGNORE INTO goal_plan_nodes (id, goal_id, plan_id, key, sequence, priority, objective, status, ready,
|
|
2820
|
+
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2821
|
+
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2822
|
+
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2823
|
+
$id: id,
|
|
2824
|
+
$goalId: goal.goalId,
|
|
2825
|
+
$planId: goal.planId,
|
|
2826
|
+
$key: node.key,
|
|
2827
|
+
$sequence: node.sequence,
|
|
2828
|
+
$priority: node.priority,
|
|
2829
|
+
$objective: node.objective,
|
|
2830
|
+
$status: node.status,
|
|
2831
|
+
$ready: node.ready ? 1 : 0,
|
|
2832
|
+
$tokenBudget: node.tokenBudget ?? null,
|
|
2833
|
+
$tokensUsed: node.tokensUsed,
|
|
2834
|
+
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2835
|
+
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2836
|
+
$created: node.createdAt,
|
|
2837
|
+
$updated: node.updatedAt
|
|
2838
|
+
});
|
|
2839
|
+
if (res.changes === 1)
|
|
2840
|
+
return;
|
|
2841
|
+
const existingByKey = this.db.query("SELECT id FROM goal_plan_nodes WHERE plan_id = ? AND key = ? LIMIT 1").get(goal.planId, node.key);
|
|
2842
|
+
if (existingByKey)
|
|
2843
|
+
return;
|
|
2844
|
+
id = genId();
|
|
2845
|
+
}
|
|
2846
|
+
throw new Error(`goal plan node was not inserted after id retries: ${goal.planId}/${node.key}`);
|
|
2847
|
+
}
|
|
2222
2848
|
listGoalPlanNodes(goalIdOrPlanId) {
|
|
2223
2849
|
const rows = this.db.query("SELECT * FROM goal_plan_nodes WHERE goal_id = ? OR plan_id = ? ORDER BY sequence ASC").all(goalIdOrPlanId, goalIdOrPlanId);
|
|
2224
2850
|
return rows.map(rowToGoalPlanNode);
|
|
@@ -2310,8 +2936,8 @@ class Store {
|
|
|
2310
2936
|
$status: input.status,
|
|
2311
2937
|
$nodeKey: input.nodeKey ?? null,
|
|
2312
2938
|
$tokensUsed: input.tokensUsed ?? 0,
|
|
2313
|
-
$evidence: input.evidence ? JSON.stringify(input.evidence) : null,
|
|
2314
|
-
$rawResponse: input.rawResponse === undefined ? null : JSON.stringify(input.rawResponse),
|
|
2939
|
+
$evidence: input.evidence ? scrubSecrets(JSON.stringify(scrubSecretsDeep(input.evidence))) : null,
|
|
2940
|
+
$rawResponse: input.rawResponse === undefined ? null : scrubSecrets(JSON.stringify(scrubSecretsDeep(input.rawResponse))),
|
|
2315
2941
|
$created: now,
|
|
2316
2942
|
$updated: now
|
|
2317
2943
|
});
|
|
@@ -2349,7 +2975,6 @@ class Store {
|
|
|
2349
2975
|
const targetInput = input.loop?.target.type === "workflow" ? input.loop.target.input : undefined;
|
|
2350
2976
|
const invocationId = input.invocationId ?? targetInput?.workflowInvocationId ?? targetInput?.invocationId;
|
|
2351
2977
|
const workItemId = input.workItemId ?? targetInput?.workflowWorkItemId ?? targetInput?.workItemId;
|
|
2352
|
-
let manifestPath;
|
|
2353
2978
|
if (input.idempotencyKey) {
|
|
2354
2979
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2355
2980
|
if (existing) {
|
|
@@ -2357,6 +2982,28 @@ class Store {
|
|
|
2357
2982
|
return rowToWorkflowRun(existing);
|
|
2358
2983
|
}
|
|
2359
2984
|
}
|
|
2985
|
+
const runId = genId();
|
|
2986
|
+
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2987
|
+
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2988
|
+
const staged = stageWorkflowRunManifest({
|
|
2989
|
+
loopsDataDir: this.rootDir,
|
|
2990
|
+
workflowRunId: runId,
|
|
2991
|
+
workflowId: input.workflow.id,
|
|
2992
|
+
workflowName: input.workflow.name,
|
|
2993
|
+
invocationId,
|
|
2994
|
+
workItemId,
|
|
2995
|
+
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2996
|
+
subjectKind: invocation?.subjectRef.kind ?? (input.loop ? "loop" : "workflow"),
|
|
2997
|
+
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url ?? input.loop?.name ?? input.workflow.name,
|
|
2998
|
+
payload: {
|
|
2999
|
+
workflowInvocation: invocation,
|
|
3000
|
+
workflowWorkItem: workItem,
|
|
3001
|
+
loopId: input.loop?.id,
|
|
3002
|
+
loopRunId: input.loopRun?.id,
|
|
3003
|
+
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
3004
|
+
}
|
|
3005
|
+
});
|
|
3006
|
+
const manifestPath = staged.manifestPath;
|
|
2360
3007
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2361
3008
|
try {
|
|
2362
3009
|
this.assertDaemonLeaseFence(input, now);
|
|
@@ -2364,30 +3011,10 @@ class Store {
|
|
|
2364
3011
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2365
3012
|
if (existing) {
|
|
2366
3013
|
this.db.exec("COMMIT");
|
|
3014
|
+
discardWorkflowRunManifest(staged);
|
|
2367
3015
|
return rowToWorkflowRun(existing);
|
|
2368
3016
|
}
|
|
2369
3017
|
}
|
|
2370
|
-
const runId = genId();
|
|
2371
|
-
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2372
|
-
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2373
|
-
manifestPath = invocation || workItem ? writeWorkflowRunManifest({
|
|
2374
|
-
loopsDataDir: this.rootDir,
|
|
2375
|
-
workflowRunId: runId,
|
|
2376
|
-
workflowId: input.workflow.id,
|
|
2377
|
-
workflowName: input.workflow.name,
|
|
2378
|
-
invocationId,
|
|
2379
|
-
workItemId,
|
|
2380
|
-
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2381
|
-
subjectKind: invocation?.subjectRef.kind,
|
|
2382
|
-
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url,
|
|
2383
|
-
payload: {
|
|
2384
|
-
workflowInvocation: invocation,
|
|
2385
|
-
workflowWorkItem: workItem,
|
|
2386
|
-
loopId: input.loop?.id,
|
|
2387
|
-
loopRunId: input.loopRun?.id,
|
|
2388
|
-
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
2389
|
-
}
|
|
2390
|
-
}) : undefined;
|
|
2391
3018
|
this.db.query(`INSERT INTO workflow_runs (id, workflow_id, workflow_name, loop_id, loop_run_id, invocation_id, work_item_id,
|
|
2392
3019
|
scheduled_for, idempotency_key, manifest_path, status, started_at, finished_at, duration_ms, error,
|
|
2393
3020
|
created_at, updated_at)
|
|
@@ -2454,6 +3081,7 @@ class Store {
|
|
|
2454
3081
|
$created: now
|
|
2455
3082
|
});
|
|
2456
3083
|
this.db.exec("COMMIT");
|
|
3084
|
+
commitWorkflowRunManifest(staged);
|
|
2457
3085
|
const run = this.getWorkflowRun(runId);
|
|
2458
3086
|
if (!run)
|
|
2459
3087
|
throw new Error(`workflow run not found after create: ${runId}`);
|
|
@@ -2462,8 +3090,7 @@ class Store {
|
|
|
2462
3090
|
try {
|
|
2463
3091
|
this.db.exec("ROLLBACK");
|
|
2464
3092
|
} catch {}
|
|
2465
|
-
|
|
2466
|
-
rmSync(manifestPath, { force: true });
|
|
3093
|
+
discardWorkflowRunManifest(staged);
|
|
2467
3094
|
throw error;
|
|
2468
3095
|
}
|
|
2469
3096
|
}
|
|
@@ -2561,29 +3188,32 @@ class Store {
|
|
|
2561
3188
|
return run;
|
|
2562
3189
|
}
|
|
2563
3190
|
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
3191
|
+
return this.transact(() => {
|
|
3192
|
+
const now = nowIso();
|
|
3193
|
+
const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
|
|
3194
|
+
const live = before.filter((step) => step.pid !== undefined && isLiveStepProcess(step.pid, step.startedAt));
|
|
3195
|
+
if (live.length > 0) {
|
|
3196
|
+
throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
|
|
3197
|
+
}
|
|
3198
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
3199
|
+
SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
|
|
3200
|
+
stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
|
|
3201
|
+
WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
|
|
3202
|
+
if (before.length > 0) {
|
|
3203
|
+
this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
|
|
3204
|
+
reason,
|
|
3205
|
+
recoveredSteps: before.map((step) => step.stepId)
|
|
3206
|
+
});
|
|
3207
|
+
}
|
|
3208
|
+
return {
|
|
3209
|
+
run: this.requireWorkflowRun(workflowRunId),
|
|
3210
|
+
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
3211
|
+
};
|
|
3212
|
+
});
|
|
2584
3213
|
}
|
|
2585
3214
|
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
2586
3215
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3216
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2587
3217
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2588
3218
|
try {
|
|
2589
3219
|
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
@@ -2598,9 +3228,9 @@ class Store {
|
|
|
2598
3228
|
$finished: finishedAt,
|
|
2599
3229
|
$exitCode: patch.exitCode ?? null,
|
|
2600
3230
|
$durationMs: patch.durationMs ?? null,
|
|
2601
|
-
$stdout: patch.stdout
|
|
2602
|
-
$stderr: patch.stderr
|
|
2603
|
-
$error:
|
|
3231
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3232
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3233
|
+
$error: error ?? null,
|
|
2604
3234
|
$updated: finishedAt,
|
|
2605
3235
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2606
3236
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
@@ -2608,15 +3238,15 @@ class Store {
|
|
|
2608
3238
|
if (res.changes === 1) {
|
|
2609
3239
|
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
2610
3240
|
exitCode: patch.exitCode,
|
|
2611
|
-
error
|
|
3241
|
+
error
|
|
2612
3242
|
});
|
|
2613
3243
|
}
|
|
2614
3244
|
this.db.exec("COMMIT");
|
|
2615
|
-
} catch (
|
|
3245
|
+
} catch (error2) {
|
|
2616
3246
|
try {
|
|
2617
3247
|
this.db.exec("ROLLBACK");
|
|
2618
3248
|
} catch {}
|
|
2619
|
-
throw
|
|
3249
|
+
throw error2;
|
|
2620
3250
|
}
|
|
2621
3251
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
2622
3252
|
if (!run)
|
|
@@ -2656,6 +3286,7 @@ class Store {
|
|
|
2656
3286
|
}
|
|
2657
3287
|
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
2658
3288
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3289
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2659
3290
|
let changed = false;
|
|
2660
3291
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2661
3292
|
try {
|
|
@@ -2668,25 +3299,25 @@ class Store {
|
|
|
2668
3299
|
$status: status,
|
|
2669
3300
|
$finished: finishedAt,
|
|
2670
3301
|
$durationMs: patch.durationMs ?? null,
|
|
2671
|
-
$error:
|
|
3302
|
+
$error: error ?? null,
|
|
2672
3303
|
$updated: finishedAt,
|
|
2673
3304
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2674
3305
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
2675
3306
|
});
|
|
2676
3307
|
changed = res.changes === 1;
|
|
2677
3308
|
if (changed)
|
|
2678
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error
|
|
3309
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error });
|
|
2679
3310
|
if (changed) {
|
|
2680
3311
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
2681
|
-
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus,
|
|
3312
|
+
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
2682
3313
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
2683
3314
|
}
|
|
2684
3315
|
this.db.exec("COMMIT");
|
|
2685
|
-
} catch (
|
|
3316
|
+
} catch (error2) {
|
|
2686
3317
|
try {
|
|
2687
3318
|
this.db.exec("ROLLBACK");
|
|
2688
3319
|
} catch {}
|
|
2689
|
-
throw
|
|
3320
|
+
throw error2;
|
|
2690
3321
|
}
|
|
2691
3322
|
const run = this.getWorkflowRun(workflowRunId);
|
|
2692
3323
|
if (!run)
|
|
@@ -2719,24 +3350,26 @@ class Store {
|
|
|
2719
3350
|
}
|
|
2720
3351
|
}
|
|
2721
3352
|
appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
3353
|
+
return this.transact(() => {
|
|
3354
|
+
const now = nowIso();
|
|
3355
|
+
const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
|
|
3356
|
+
const sequence = (current?.sequence ?? 0) + 1;
|
|
3357
|
+
const id = genId();
|
|
3358
|
+
this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
|
|
3359
|
+
VALUES ($id, $workflowRunId, $sequence, $eventType, $stepId, $payload, $created)`).run({
|
|
3360
|
+
$id: id,
|
|
3361
|
+
$workflowRunId: workflowRunId,
|
|
3362
|
+
$sequence: sequence,
|
|
3363
|
+
$eventType: eventType,
|
|
3364
|
+
$stepId: stepId ?? null,
|
|
3365
|
+
$payload: payload ? JSON.stringify(payload) : null,
|
|
3366
|
+
$created: now
|
|
3367
|
+
});
|
|
3368
|
+
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
3369
|
+
if (!event)
|
|
3370
|
+
throw new Error(`workflow event not found after append: ${id}`);
|
|
3371
|
+
return rowToWorkflowEvent(event);
|
|
2735
3372
|
});
|
|
2736
|
-
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
2737
|
-
if (!event)
|
|
2738
|
-
throw new Error(`workflow event not found after append: ${id}`);
|
|
2739
|
-
return rowToWorkflowEvent(event);
|
|
2740
3373
|
}
|
|
2741
3374
|
listWorkflowEvents(workflowRunId, limit = 200) {
|
|
2742
3375
|
const rows = this.db.query("SELECT * FROM workflow_events WHERE workflow_run_id = ? ORDER BY sequence ASC LIMIT ?").all(workflowRunId, limit);
|
|
@@ -2752,35 +3385,64 @@ class Store {
|
|
|
2752
3385
|
}
|
|
2753
3386
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
2754
3387
|
const now = (opts.now ?? new Date).toISOString();
|
|
2755
|
-
const
|
|
3388
|
+
const startedMs = processStartTimeMs(pid);
|
|
3389
|
+
const processStartedAt = startedMs === undefined ? null : new Date(startedMs).toISOString();
|
|
3390
|
+
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2756
3391
|
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
2757
3392
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2758
3393
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2759
3394
|
))`).run({
|
|
2760
3395
|
$id: id,
|
|
2761
3396
|
$pid: pid,
|
|
3397
|
+
$processStartedAt: processStartedAt,
|
|
2762
3398
|
$updated: now,
|
|
2763
3399
|
$claimedBy: claimedBy,
|
|
2764
3400
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2765
3401
|
$now: now
|
|
2766
|
-
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
3402
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2767
3403
|
WHERE id=$id AND status='running'
|
|
2768
3404
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2769
3405
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2770
|
-
))`).run({
|
|
3406
|
+
))`).run({
|
|
3407
|
+
$id: id,
|
|
3408
|
+
$pid: pid,
|
|
3409
|
+
$processStartedAt: processStartedAt,
|
|
3410
|
+
$updated: now,
|
|
3411
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3412
|
+
$now: now
|
|
3413
|
+
});
|
|
2771
3414
|
if (res.changes !== 1)
|
|
2772
3415
|
return;
|
|
2773
3416
|
return this.getRun(id);
|
|
2774
3417
|
}
|
|
3418
|
+
recordRunProcess(runId, info, opts = {}) {
|
|
3419
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
3420
|
+
const res = this.db.query(`UPDATE loop_runs SET pid=$pid, pgid=$pgid, process_started_at=$processStartedAt, updated_at=$updated
|
|
3421
|
+
WHERE id=$id AND status='running'
|
|
3422
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3423
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3424
|
+
))`).run({
|
|
3425
|
+
$id: runId,
|
|
3426
|
+
$pid: info.pid,
|
|
3427
|
+
$pgid: info.pgid ?? null,
|
|
3428
|
+
$processStartedAt: info.processStartedAt ?? isoProcessStart(info.pid) ?? now,
|
|
3429
|
+
$updated: now,
|
|
3430
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3431
|
+
$now: now
|
|
3432
|
+
});
|
|
3433
|
+
if (res.changes !== 1)
|
|
3434
|
+
return;
|
|
3435
|
+
return this.getRun(runId);
|
|
3436
|
+
}
|
|
2775
3437
|
hasLiveWorkflowStepProcesses(loopRunId) {
|
|
2776
|
-
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
|
|
3438
|
+
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid, wsr.started_at AS started_at
|
|
2777
3439
|
FROM workflow_runs wr
|
|
2778
3440
|
JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
|
|
2779
3441
|
WHERE wr.loop_run_id = ?
|
|
2780
3442
|
AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
2781
3443
|
AND wsr.status = 'running'
|
|
2782
3444
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
2783
|
-
return liveWorkflowSteps.some((step) =>
|
|
3445
|
+
return liveWorkflowSteps.some((step) => isLiveStepProcess(step.pid, step.started_at));
|
|
2784
3446
|
}
|
|
2785
3447
|
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
2786
3448
|
const now = nowIso();
|
|
@@ -2854,7 +3516,7 @@ class Store {
|
|
|
2854
3516
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
2855
3517
|
if (existing) {
|
|
2856
3518
|
if (existing.status === "running") {
|
|
2857
|
-
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid
|
|
3519
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
2858
3520
|
this.db.exec("COMMIT");
|
|
2859
3521
|
return;
|
|
2860
3522
|
}
|
|
@@ -2863,7 +3525,7 @@ class Store {
|
|
|
2863
3525
|
return;
|
|
2864
3526
|
}
|
|
2865
3527
|
const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
|
|
2866
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3528
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2867
3529
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2868
3530
|
WHERE id=$id AND status='running' AND lease_expires_at <= $now`).run({
|
|
2869
3531
|
$id: existing.id,
|
|
@@ -2885,7 +3547,7 @@ class Store {
|
|
|
2885
3547
|
}
|
|
2886
3548
|
const attempt = existing.attempt + 1;
|
|
2887
3549
|
const res2 = this.db.query(`UPDATE loop_runs SET attempt=$attempt, status='running', started_at=$started, finished_at=NULL,
|
|
2888
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3550
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2889
3551
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2890
3552
|
WHERE id=$id
|
|
2891
3553
|
AND status IN ('failed', 'timed_out', 'abandoned')
|
|
@@ -2933,6 +3595,7 @@ class Store {
|
|
|
2933
3595
|
}
|
|
2934
3596
|
finalizeRun(id, patch, opts = {}) {
|
|
2935
3597
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3598
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2936
3599
|
const params = {
|
|
2937
3600
|
$id: id,
|
|
2938
3601
|
$status: patch.status,
|
|
@@ -2940,41 +3603,43 @@ class Store {
|
|
|
2940
3603
|
$pid: patch.pid ?? null,
|
|
2941
3604
|
$exitCode: patch.exitCode ?? null,
|
|
2942
3605
|
$durationMs: patch.durationMs ?? null,
|
|
2943
|
-
$stdout: patch.stdout
|
|
2944
|
-
$stderr: patch.stderr
|
|
2945
|
-
$error:
|
|
3606
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3607
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3608
|
+
$error: error ?? null,
|
|
2946
3609
|
$updated: finishedAt,
|
|
2947
3610
|
$claimedBy: opts.claimedBy ?? null,
|
|
2948
3611
|
$now: (opts.now ?? new Date).toISOString(),
|
|
2949
3612
|
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
2950
3613
|
};
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
AND
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
3614
|
+
return this.transact(() => {
|
|
3615
|
+
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3616
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3617
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
3618
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3619
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3620
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3621
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
3622
|
+
const run = this.getRun(id);
|
|
3623
|
+
if (!run)
|
|
3624
|
+
throw new Error(`run not found after finalize: ${id}`);
|
|
3625
|
+
if (opts.claimedBy && res.changes !== 1)
|
|
3626
|
+
return run;
|
|
3627
|
+
if (res.changes === 1) {
|
|
3628
|
+
this.setWorkflowWorkItemsForLoopRun(run, error, finishedAt);
|
|
3629
|
+
const loop = this.getLoop(run.loopId);
|
|
3630
|
+
const itemStatus = workItemStatusForLoopRun(run.status, run.attempt, loop?.maxAttempts);
|
|
3631
|
+
if (loop?.target.type === "workflow" && itemStatus && itemStatus !== "admitted") {
|
|
3632
|
+
const workItemId = loop.target.input?.workflowWorkItemId ?? loop.target.input?.workItemId;
|
|
3633
|
+
this.maybeArchiveGeneratedRouteWorkflow({
|
|
3634
|
+
workflowId: loop.target.workflowId,
|
|
3635
|
+
loopId: loop.id,
|
|
3636
|
+
workItemId,
|
|
3637
|
+
updated: finishedAt
|
|
3638
|
+
});
|
|
3639
|
+
}
|
|
2975
3640
|
}
|
|
2976
|
-
|
|
2977
|
-
|
|
3641
|
+
return run;
|
|
3642
|
+
});
|
|
2978
3643
|
}
|
|
2979
3644
|
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
2980
3645
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
@@ -3024,6 +3689,9 @@ class Store {
|
|
|
3024
3689
|
});
|
|
3025
3690
|
}
|
|
3026
3691
|
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
3692
|
+
return this.recoverExpiredRunLeasesDetailed(now, opts).abandoned;
|
|
3693
|
+
}
|
|
3694
|
+
recoverExpiredRunLeasesDetailed(now = new Date, opts = {}) {
|
|
3027
3695
|
const limit = Math.max(1, Math.min(1000, Math.floor(opts.limit ?? DEFAULT_RECOVERY_BATCH_LIMIT)));
|
|
3028
3696
|
const scanLimit = Math.max(limit, Math.min(5000, Math.floor(opts.scanLimit ?? limit * DEFAULT_RECOVERY_SCAN_MULTIPLIER)));
|
|
3029
3697
|
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
@@ -3031,15 +3699,15 @@ class Store {
|
|
|
3031
3699
|
ORDER BY lease_expires_at ASC
|
|
3032
3700
|
LIMIT ?`).all(now.toISOString(), scanLimit);
|
|
3033
3701
|
const recovered = [];
|
|
3702
|
+
const deferred = [];
|
|
3034
3703
|
for (const row of rows) {
|
|
3035
3704
|
if (recovered.length >= limit)
|
|
3036
3705
|
break;
|
|
3037
|
-
if (row.pid
|
|
3038
|
-
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3039
|
-
continue;
|
|
3040
|
-
}
|
|
3041
|
-
if (this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3706
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at) || this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3042
3707
|
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3708
|
+
const deferredRun = this.getRun(row.id);
|
|
3709
|
+
if (deferredRun)
|
|
3710
|
+
deferred.push(deferredRun);
|
|
3043
3711
|
continue;
|
|
3044
3712
|
}
|
|
3045
3713
|
const finished = now.toISOString();
|
|
@@ -3123,7 +3791,7 @@ class Store {
|
|
|
3123
3791
|
if (run)
|
|
3124
3792
|
recovered.push(run);
|
|
3125
3793
|
}
|
|
3126
|
-
return recovered;
|
|
3794
|
+
return { abandoned: recovered, deferred };
|
|
3127
3795
|
}
|
|
3128
3796
|
expireLoops(now = new Date, opts = {}) {
|
|
3129
3797
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND archived_at IS NULL AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
@@ -3156,6 +3824,83 @@ class Store {
|
|
|
3156
3824
|
const row = status ? this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE status = ?").get(status) : this.db.query("SELECT COUNT(*) AS count FROM loop_runs").get();
|
|
3157
3825
|
return row?.count ?? 0;
|
|
3158
3826
|
}
|
|
3827
|
+
pruneHistory(opts) {
|
|
3828
|
+
const { maxAgeDays, keepPerLoop } = opts;
|
|
3829
|
+
if (maxAgeDays === undefined && keepPerLoop === undefined) {
|
|
3830
|
+
throw new ValidationError("pruneHistory requires maxAgeDays and/or keepPerLoop");
|
|
3831
|
+
}
|
|
3832
|
+
if (maxAgeDays !== undefined && (!Number.isFinite(maxAgeDays) || maxAgeDays < 0)) {
|
|
3833
|
+
throw new ValidationError(`pruneHistory maxAgeDays must be a non-negative number: ${maxAgeDays}`);
|
|
3834
|
+
}
|
|
3835
|
+
if (keepPerLoop !== undefined && (!Number.isInteger(keepPerLoop) || keepPerLoop < 0)) {
|
|
3836
|
+
throw new ValidationError(`pruneHistory keepPerLoop must be a non-negative integer: ${keepPerLoop}`);
|
|
3837
|
+
}
|
|
3838
|
+
const now = opts.now ?? new Date;
|
|
3839
|
+
const dryRun = opts.dryRun ?? false;
|
|
3840
|
+
const cutoff = maxAgeDays === undefined ? undefined : new Date(now.getTime() - maxAgeDays * 86400000).toISOString();
|
|
3841
|
+
const terminal = TERMINAL_RUN_STATUSES.map((status) => `'${status}'`).join(",");
|
|
3842
|
+
const candidateIds = this.db.query(`WITH ranked AS (
|
|
3843
|
+
SELECT id, status, created_at,
|
|
3844
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS recency
|
|
3845
|
+
FROM loop_runs
|
|
3846
|
+
)
|
|
3847
|
+
SELECT id FROM ranked
|
|
3848
|
+
WHERE status IN (${terminal})
|
|
3849
|
+
AND ($cutoff IS NULL OR created_at < $cutoff)
|
|
3850
|
+
AND ($keep IS NULL OR recency > $keep)`).all({ $cutoff: cutoff ?? null, $keep: keepPerLoop ?? null }).map((row) => row.id);
|
|
3851
|
+
const summary = {
|
|
3852
|
+
dryRun,
|
|
3853
|
+
cutoff,
|
|
3854
|
+
keepPerLoop,
|
|
3855
|
+
loopRuns: dryRun ? candidateIds.length : 0,
|
|
3856
|
+
workflowRuns: 0,
|
|
3857
|
+
goalRuns: 0
|
|
3858
|
+
};
|
|
3859
|
+
const manifestPaths = [];
|
|
3860
|
+
for (let offset = 0;offset < candidateIds.length; offset += PRUNE_BATCH_SIZE) {
|
|
3861
|
+
const batch = candidateIds.slice(offset, offset + PRUNE_BATCH_SIZE);
|
|
3862
|
+
const batchPlaceholders = batch.map(() => "?").join(",");
|
|
3863
|
+
if (dryRun) {
|
|
3864
|
+
const workflowRunIds = this.db.query(`SELECT id FROM workflow_runs WHERE loop_run_id IN (${batchPlaceholders})`).all(...batch).map((row) => row.id);
|
|
3865
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3866
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3867
|
+
summary.goalRuns += this.db.query(`SELECT COUNT(*) AS count FROM goal_runs
|
|
3868
|
+
WHERE loop_run_id IN (${batchPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).get(...batch, ...workflowRunIds)?.count ?? 0;
|
|
3869
|
+
continue;
|
|
3870
|
+
}
|
|
3871
|
+
this.transact(() => {
|
|
3872
|
+
const confirmed = this.db.query(`SELECT id FROM loop_runs WHERE id IN (${batchPlaceholders}) AND status IN (${terminal})`).all(...batch).map((row) => row.id);
|
|
3873
|
+
if (confirmed.length === 0)
|
|
3874
|
+
return;
|
|
3875
|
+
const runPlaceholders = confirmed.map(() => "?").join(",");
|
|
3876
|
+
const workflowRuns = this.db.query(`SELECT id, manifest_path FROM workflow_runs WHERE loop_run_id IN (${runPlaceholders})`).all(...confirmed);
|
|
3877
|
+
const workflowRunIds = workflowRuns.map((row) => row.id);
|
|
3878
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3879
|
+
summary.loopRuns += confirmed.length;
|
|
3880
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3881
|
+
summary.goalRuns += this.db.query(`DELETE FROM goal_runs WHERE loop_run_id IN (${runPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).run(...confirmed, ...workflowRunIds).changes;
|
|
3882
|
+
if (workflowRunIds.length > 0) {
|
|
3883
|
+
this.db.query(`DELETE FROM workflow_runs WHERE id IN (${workflowPlaceholders})`).run(...workflowRunIds);
|
|
3884
|
+
}
|
|
3885
|
+
this.db.query(`DELETE FROM loop_runs WHERE id IN (${runPlaceholders}) AND status IN (${terminal})`).run(...confirmed);
|
|
3886
|
+
for (const row of workflowRuns) {
|
|
3887
|
+
if (row.manifest_path)
|
|
3888
|
+
manifestPaths.push(row.manifest_path);
|
|
3889
|
+
}
|
|
3890
|
+
});
|
|
3891
|
+
}
|
|
3892
|
+
for (const manifestPath of manifestPaths) {
|
|
3893
|
+
const runDir = dirname2(manifestPath);
|
|
3894
|
+
try {
|
|
3895
|
+
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
3896
|
+
rmSync2(runDir, { recursive: true, force: true });
|
|
3897
|
+
} else {
|
|
3898
|
+
rmSync2(manifestPath, { force: true });
|
|
3899
|
+
}
|
|
3900
|
+
} catch {}
|
|
3901
|
+
}
|
|
3902
|
+
return summary;
|
|
3903
|
+
}
|
|
3159
3904
|
acquireDaemonLease(input) {
|
|
3160
3905
|
const now = input.now ?? new Date;
|
|
3161
3906
|
const expiresAt = new Date(now.getTime() + input.ttlMs).toISOString();
|
|
@@ -3218,16 +3963,245 @@ class Store {
|
|
|
3218
3963
|
}
|
|
3219
3964
|
}
|
|
3220
3965
|
|
|
3966
|
+
// src/lib/doctor.ts
|
|
3967
|
+
import { spawnSync as spawnSync5 } from "child_process";
|
|
3968
|
+
import { accessSync as accessSync2, constants as constants2 } from "fs";
|
|
3969
|
+
|
|
3970
|
+
// src/daemon/control.ts
|
|
3971
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync4, readFileSync as readFileSync3, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
3972
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
3973
|
+
import { hostname } from "os";
|
|
3974
|
+
import { dirname as dirname3, join as join4 } from "path";
|
|
3975
|
+
|
|
3976
|
+
// src/daemon/loop.ts
|
|
3977
|
+
function realSleep(ms) {
|
|
3978
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
3979
|
+
}
|
|
3980
|
+
async function runLoop(opts) {
|
|
3981
|
+
const sleep = opts.sleep ?? realSleep;
|
|
3982
|
+
const sliceMs = opts.sliceMs ?? 200;
|
|
3983
|
+
while (!opts.shouldStop()) {
|
|
3984
|
+
try {
|
|
3985
|
+
await opts.tickFn();
|
|
3986
|
+
} catch (err) {
|
|
3987
|
+
opts.onTickError?.(err);
|
|
3988
|
+
}
|
|
3989
|
+
let waited = 0;
|
|
3990
|
+
while (waited < opts.intervalMs && !opts.shouldStop()) {
|
|
3991
|
+
const chunk = Math.min(sliceMs, opts.intervalMs - waited);
|
|
3992
|
+
await sleep(chunk);
|
|
3993
|
+
waited += chunk;
|
|
3994
|
+
}
|
|
3995
|
+
}
|
|
3996
|
+
}
|
|
3997
|
+
|
|
3998
|
+
// src/daemon/control.ts
|
|
3999
|
+
function readPidRecord(path = pidFilePath()) {
|
|
4000
|
+
if (!existsSync2(path))
|
|
4001
|
+
return;
|
|
4002
|
+
let raw;
|
|
4003
|
+
try {
|
|
4004
|
+
raw = readFileSync3(path, "utf8").trim();
|
|
4005
|
+
} catch {
|
|
4006
|
+
return;
|
|
4007
|
+
}
|
|
4008
|
+
if (!raw)
|
|
4009
|
+
return;
|
|
4010
|
+
if (raw.startsWith("{")) {
|
|
4011
|
+
try {
|
|
4012
|
+
const parsed = JSON.parse(raw);
|
|
4013
|
+
const pid2 = Number(parsed.pid);
|
|
4014
|
+
if (!Number.isInteger(pid2) || pid2 <= 0)
|
|
4015
|
+
return;
|
|
4016
|
+
const startedAt = Number(parsed.startedAt);
|
|
4017
|
+
return { pid: pid2, startedAt: Number.isFinite(startedAt) ? startedAt : undefined };
|
|
4018
|
+
} catch {
|
|
4019
|
+
return;
|
|
4020
|
+
}
|
|
4021
|
+
}
|
|
4022
|
+
const pid = Number(raw);
|
|
4023
|
+
return Number.isInteger(pid) && pid > 0 ? { pid } : undefined;
|
|
4024
|
+
}
|
|
4025
|
+
function writePid(pid = process.pid, path = pidFilePath()) {
|
|
4026
|
+
mkdirSync4(dirname3(path), { recursive: true, mode: 448 });
|
|
4027
|
+
writeFileSync2(path, JSON.stringify({ pid, startedAt: processStartTimeMs(pid) }));
|
|
4028
|
+
}
|
|
4029
|
+
function removePid(path = pidFilePath()) {
|
|
4030
|
+
rmSync3(path, { force: true });
|
|
4031
|
+
}
|
|
4032
|
+
function isAlive(pid, startedAt) {
|
|
4033
|
+
try {
|
|
4034
|
+
process.kill(pid, 0);
|
|
4035
|
+
} catch (err) {
|
|
4036
|
+
if (err.code !== "EPERM")
|
|
4037
|
+
return false;
|
|
4038
|
+
}
|
|
4039
|
+
return sameProcessStart(startedAt, processStartTimeMs(pid));
|
|
4040
|
+
}
|
|
4041
|
+
function isDaemonRunning(path = pidFilePath()) {
|
|
4042
|
+
const record = readPidRecord(path);
|
|
4043
|
+
if (!record)
|
|
4044
|
+
return { running: false, stale: false };
|
|
4045
|
+
if (isAlive(record.pid, record.startedAt))
|
|
4046
|
+
return { running: true, stale: false, pid: record.pid };
|
|
4047
|
+
return { running: false, stale: true, pid: record.pid };
|
|
4048
|
+
}
|
|
4049
|
+
function toReapableProcess(run) {
|
|
4050
|
+
return { pid: run.pid, pgid: run.pgid, processStartedAt: run.processStartedAt };
|
|
4051
|
+
}
|
|
4052
|
+
function ownProcessGroupId() {
|
|
4053
|
+
if (process.platform === "linux") {
|
|
4054
|
+
const fields = procStatFields("/proc/self/stat");
|
|
4055
|
+
const pgrp = fields ? Number(fields[2]) : Number.NaN;
|
|
4056
|
+
if (Number.isInteger(pgrp) && pgrp > 0)
|
|
4057
|
+
return pgrp;
|
|
4058
|
+
}
|
|
4059
|
+
try {
|
|
4060
|
+
const run = spawnSync2("ps", ["-o", "pgid=", "-p", String(process.pid)], { encoding: "utf8" });
|
|
4061
|
+
const pgid = Number(run.stdout.trim());
|
|
4062
|
+
if (run.status === 0 && Number.isInteger(pgid) && pgid > 0)
|
|
4063
|
+
return pgid;
|
|
4064
|
+
} catch {}
|
|
4065
|
+
return;
|
|
4066
|
+
}
|
|
4067
|
+
function signalReapTarget(target, signal) {
|
|
4068
|
+
try {
|
|
4069
|
+
process.kill(target.group ? -target.id : target.id, signal);
|
|
4070
|
+
return true;
|
|
4071
|
+
} catch (err) {
|
|
4072
|
+
return err.code === "EPERM";
|
|
4073
|
+
}
|
|
4074
|
+
}
|
|
4075
|
+
function processExists(pid) {
|
|
4076
|
+
try {
|
|
4077
|
+
process.kill(pid, 0);
|
|
4078
|
+
return true;
|
|
4079
|
+
} catch (err) {
|
|
4080
|
+
return err.code === "EPERM";
|
|
4081
|
+
}
|
|
4082
|
+
}
|
|
4083
|
+
async function reapProcessGroups(entries, opts = {}) {
|
|
4084
|
+
const sleep = opts.sleep ?? realSleep;
|
|
4085
|
+
const graceMs = Math.max(100, opts.graceMs ?? 2000);
|
|
4086
|
+
const ownPgid = ownProcessGroupId();
|
|
4087
|
+
const targets = new Map;
|
|
4088
|
+
for (const entry of entries) {
|
|
4089
|
+
const leaderPid = entry.pid ?? entry.pgid;
|
|
4090
|
+
if (leaderPid !== undefined && processExists(leaderPid)) {
|
|
4091
|
+
if (entry.processStartedAt === undefined) {
|
|
4092
|
+
opts.log?.(`skipping reap of pid ${leaderPid}: no recorded start-time fingerprint to verify identity`);
|
|
4093
|
+
continue;
|
|
4094
|
+
}
|
|
4095
|
+
if (!verifiedProcessStart(entry.processStartedAt, processStartTimeMs(leaderPid))) {
|
|
4096
|
+
opts.log?.(`skipping reap of pid ${leaderPid}: start-time fingerprint mismatch or unverifiable (pid recycled)`);
|
|
4097
|
+
continue;
|
|
4098
|
+
}
|
|
4099
|
+
}
|
|
4100
|
+
const usableGroup = entry.pgid !== undefined && Number.isInteger(entry.pgid) && entry.pgid > 1 && entry.pgid !== process.pid && entry.pgid !== ownPgid;
|
|
4101
|
+
const usablePid = entry.pid !== undefined && Number.isInteger(entry.pid) && entry.pid > 1 && entry.pid !== process.pid;
|
|
4102
|
+
const target = usableGroup ? { id: entry.pgid, group: true } : usablePid ? { id: entry.pid, group: false } : undefined;
|
|
4103
|
+
if (!target)
|
|
4104
|
+
continue;
|
|
4105
|
+
targets.set(`${target.group ? "g" : "p"}:${target.id}`, target);
|
|
4106
|
+
}
|
|
4107
|
+
const live = [...targets.values()].filter((target) => signalReapTarget(target, "SIGTERM"));
|
|
4108
|
+
if (live.length === 0)
|
|
4109
|
+
return [];
|
|
4110
|
+
const steps = Math.max(1, Math.ceil(graceMs / 100));
|
|
4111
|
+
let remaining = live;
|
|
4112
|
+
for (let i = 0;i < steps && remaining.length > 0; i++) {
|
|
4113
|
+
await sleep(100);
|
|
4114
|
+
remaining = remaining.filter((target) => signalReapTarget(target, 0));
|
|
4115
|
+
}
|
|
4116
|
+
for (const target of remaining) {
|
|
4117
|
+
signalReapTarget(target, "SIGKILL");
|
|
4118
|
+
opts.log?.(`escalated to SIGKILL for ${target.group ? "pgid" : "pid"} ${target.id}`);
|
|
4119
|
+
}
|
|
4120
|
+
return live.map((target) => target.id);
|
|
4121
|
+
}
|
|
4122
|
+
function daemonStatus(store, path = pidFilePath()) {
|
|
4123
|
+
return {
|
|
4124
|
+
...isDaemonRunning(path),
|
|
4125
|
+
lease: store.getDaemonLease(),
|
|
4126
|
+
host: hostname(),
|
|
4127
|
+
loops: {
|
|
4128
|
+
total: store.countLoops(),
|
|
4129
|
+
active: store.countLoops("active"),
|
|
4130
|
+
paused: store.countLoops("paused"),
|
|
4131
|
+
stopped: store.countLoops("stopped"),
|
|
4132
|
+
expired: store.countLoops("expired"),
|
|
4133
|
+
archived: store.countLoops(undefined, { archived: true })
|
|
4134
|
+
},
|
|
4135
|
+
runs: {
|
|
4136
|
+
total: store.countRuns(),
|
|
4137
|
+
running: store.countRuns("running"),
|
|
4138
|
+
failed: store.countRuns("failed"),
|
|
4139
|
+
succeeded: store.countRuns("succeeded"),
|
|
4140
|
+
abandoned: store.countRuns("abandoned")
|
|
4141
|
+
},
|
|
4142
|
+
logPath: daemonLogPath()
|
|
4143
|
+
};
|
|
4144
|
+
}
|
|
4145
|
+
async function stopDaemon(opts = {}) {
|
|
4146
|
+
const path = opts.path ?? pidFilePath();
|
|
4147
|
+
const record = readPidRecord(path);
|
|
4148
|
+
const state = isDaemonRunning(path);
|
|
4149
|
+
if (state.stale) {
|
|
4150
|
+
removePid(path);
|
|
4151
|
+
return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
|
|
4152
|
+
}
|
|
4153
|
+
if (!state.running || !state.pid)
|
|
4154
|
+
return { wasRunning: false, stopped: false, forced: false };
|
|
4155
|
+
const sleep = opts.sleep ?? realSleep;
|
|
4156
|
+
const store = new Store(join4(dirname3(path), "loops.db"));
|
|
4157
|
+
try {
|
|
4158
|
+
const lease = store.getDaemonLease();
|
|
4159
|
+
if (!lease || lease.pid !== state.pid || new Date(lease.expiresAt).getTime() <= Date.now()) {
|
|
4160
|
+
removePid(path);
|
|
4161
|
+
return { wasRunning: false, stopped: false, forced: false, pid: state.pid };
|
|
4162
|
+
}
|
|
4163
|
+
try {
|
|
4164
|
+
process.kill(state.pid, "SIGTERM");
|
|
4165
|
+
} catch {
|
|
4166
|
+
removePid(path);
|
|
4167
|
+
return { wasRunning: true, stopped: true, forced: false, pid: state.pid };
|
|
4168
|
+
}
|
|
4169
|
+
const steps = Math.max(1, Math.ceil((opts.timeoutMs ?? 6000) / 100));
|
|
4170
|
+
for (let i = 0;i < steps; i++) {
|
|
4171
|
+
await sleep(100);
|
|
4172
|
+
if (!isAlive(state.pid, record?.startedAt)) {
|
|
4173
|
+
removePid(path);
|
|
4174
|
+
return { wasRunning: true, stopped: true, forced: false, pid: state.pid };
|
|
4175
|
+
}
|
|
4176
|
+
}
|
|
4177
|
+
try {
|
|
4178
|
+
process.kill(state.pid, "SIGKILL");
|
|
4179
|
+
} catch {}
|
|
4180
|
+
await sleep(150);
|
|
4181
|
+
removePid(path);
|
|
4182
|
+
const ownedRuns = store.listRuns({ status: "running", limit: 1000 }).filter((run) => run.claimedBy !== undefined && run.claimedBy.endsWith(`:${lease.id}`));
|
|
4183
|
+
const reapedPgids = await reapProcessGroups(ownedRuns.map(toReapableProcess), {
|
|
4184
|
+
sleep,
|
|
4185
|
+
graceMs: opts.reapGraceMs
|
|
4186
|
+
});
|
|
4187
|
+
return { wasRunning: true, stopped: !isAlive(state.pid, record?.startedAt), forced: true, pid: state.pid, reapedPgids };
|
|
4188
|
+
} finally {
|
|
4189
|
+
store.close();
|
|
4190
|
+
}
|
|
4191
|
+
}
|
|
4192
|
+
|
|
3221
4193
|
// src/lib/executor.ts
|
|
3222
|
-
import { spawn, spawnSync as
|
|
4194
|
+
import { spawn as spawn2, spawnSync as spawnSync4 } from "child_process";
|
|
3223
4195
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
3224
4196
|
import { once } from "events";
|
|
3225
|
-
import {
|
|
4197
|
+
import { lstatSync, mkdirSync as mkdirSync5, realpathSync } from "fs";
|
|
4198
|
+
import { dirname as dirname4, resolve as resolve2 } from "path";
|
|
3226
4199
|
|
|
3227
4200
|
// src/lib/accounts.ts
|
|
3228
|
-
import { spawnSync } from "child_process";
|
|
3229
|
-
import { existsSync as
|
|
4201
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
4202
|
+
import { existsSync as existsSync3 } from "fs";
|
|
3230
4203
|
var EXPORT_RE = /^export\s+([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
4204
|
+
var ACCOUNTS_ENV_TIMEOUT_MS = 30000;
|
|
3231
4205
|
function accountToolForProvider(provider) {
|
|
3232
4206
|
switch (provider) {
|
|
3233
4207
|
case "claude":
|
|
@@ -3289,21 +4263,17 @@ function accountDirEnvVar(tool) {
|
|
|
3289
4263
|
return;
|
|
3290
4264
|
}
|
|
3291
4265
|
}
|
|
3292
|
-
function
|
|
3293
|
-
if (!account)
|
|
3294
|
-
return {};
|
|
4266
|
+
function requiredAccountTool(account, toolHint) {
|
|
3295
4267
|
const tool = account.tool ?? toolHint;
|
|
3296
4268
|
if (!tool)
|
|
3297
4269
|
throw new Error("account.tool is required when no provider tool can be inferred");
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
stdio: ["ignore", "pipe", "pipe"]
|
|
3302
|
-
});
|
|
4270
|
+
return tool;
|
|
4271
|
+
}
|
|
4272
|
+
function accountEnvFromResult(account, tool, result) {
|
|
3303
4273
|
if (result.error) {
|
|
3304
|
-
throw new Error(`failed to run accounts env for ${account.profile}/${tool}: ${result.error
|
|
4274
|
+
throw new Error(`failed to run accounts env for ${account.profile}/${tool}: ${result.error}`);
|
|
3305
4275
|
}
|
|
3306
|
-
if ((result.status ??
|
|
4276
|
+
if ((result.status ?? 1) !== 0) {
|
|
3307
4277
|
const stderr = result.stderr.trim();
|
|
3308
4278
|
throw new Error(`accounts env failed for ${account.profile}/${tool}${stderr ? `: ${stderr}` : ""}`);
|
|
3309
4279
|
}
|
|
@@ -3311,7 +4281,7 @@ function resolveAccountEnv(account, toolHint, env) {
|
|
|
3311
4281
|
const profileDir = (accountDirEnvVar(tool) ? accountEnv[accountDirEnvVar(tool)] : undefined) ?? primaryAccountDir(result.stdout);
|
|
3312
4282
|
if (!profileDir)
|
|
3313
4283
|
throw new Error(`accounts env returned no profile directory for ${account.profile}/${tool}`);
|
|
3314
|
-
if (!
|
|
4284
|
+
if (!existsSync3(profileDir))
|
|
3315
4285
|
throw new Error(`account profile directory does not exist for ${account.profile}/${tool}: ${profileDir}`);
|
|
3316
4286
|
return {
|
|
3317
4287
|
...accountEnv,
|
|
@@ -3319,11 +4289,38 @@ function resolveAccountEnv(account, toolHint, env) {
|
|
|
3319
4289
|
LOOPS_ACCOUNT_TOOL: tool
|
|
3320
4290
|
};
|
|
3321
4291
|
}
|
|
4292
|
+
async function resolveAccountEnv(account, toolHint, env) {
|
|
4293
|
+
if (!account)
|
|
4294
|
+
return {};
|
|
4295
|
+
const tool = requiredAccountTool(account, toolHint);
|
|
4296
|
+
const result = await spawnCapture("accounts", ["env", account.profile, "--tool", tool], {
|
|
4297
|
+
env,
|
|
4298
|
+
timeoutMs: ACCOUNTS_ENV_TIMEOUT_MS
|
|
4299
|
+
});
|
|
4300
|
+
return accountEnvFromResult(account, tool, result);
|
|
4301
|
+
}
|
|
4302
|
+
function resolveAccountEnvSync(account, toolHint, env) {
|
|
4303
|
+
if (!account)
|
|
4304
|
+
return {};
|
|
4305
|
+
const tool = requiredAccountTool(account, toolHint);
|
|
4306
|
+
const result = spawnSync3("accounts", ["env", account.profile, "--tool", tool], {
|
|
4307
|
+
encoding: "utf8",
|
|
4308
|
+
env,
|
|
4309
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
4310
|
+
timeout: ACCOUNTS_ENV_TIMEOUT_MS
|
|
4311
|
+
});
|
|
4312
|
+
return accountEnvFromResult(account, tool, {
|
|
4313
|
+
status: result.status,
|
|
4314
|
+
stdout: result.stdout ?? "",
|
|
4315
|
+
stderr: result.stderr ?? "",
|
|
4316
|
+
error: result.error?.message
|
|
4317
|
+
});
|
|
4318
|
+
}
|
|
3322
4319
|
|
|
3323
4320
|
// src/lib/env.ts
|
|
3324
4321
|
import { accessSync, constants } from "fs";
|
|
3325
4322
|
import { homedir as homedir2 } from "os";
|
|
3326
|
-
import { delimiter, join as
|
|
4323
|
+
import { delimiter, join as join5 } from "path";
|
|
3327
4324
|
function compactPathParts(parts) {
|
|
3328
4325
|
const seen = new Set;
|
|
3329
4326
|
const result = [];
|
|
@@ -3339,14 +4336,14 @@ function compactPathParts(parts) {
|
|
|
3339
4336
|
function commonExecutableDirs(env = process.env) {
|
|
3340
4337
|
const home = env.HOME || homedir2();
|
|
3341
4338
|
return compactPathParts([
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
env.BUN_INSTALL ?
|
|
4339
|
+
join5(home, ".local", "bin"),
|
|
4340
|
+
join5(home, ".bun", "bin"),
|
|
4341
|
+
join5(home, ".cargo", "bin"),
|
|
4342
|
+
join5(home, ".npm-global", "bin"),
|
|
4343
|
+
join5(home, "bin"),
|
|
4344
|
+
env.BUN_INSTALL ? join5(env.BUN_INSTALL, "bin") : undefined,
|
|
3348
4345
|
env.PNPM_HOME,
|
|
3349
|
-
env.NPM_CONFIG_PREFIX ?
|
|
4346
|
+
env.NPM_CONFIG_PREFIX ? join5(env.NPM_CONFIG_PREFIX, "bin") : undefined,
|
|
3350
4347
|
"/opt/homebrew/bin",
|
|
3351
4348
|
"/usr/local/bin",
|
|
3352
4349
|
"/usr/bin",
|
|
@@ -3370,7 +4367,7 @@ function executableExists(command, env = process.env) {
|
|
|
3370
4367
|
if (command.includes("/"))
|
|
3371
4368
|
return isExecutable(command);
|
|
3372
4369
|
for (const dir of (env.PATH ?? "").split(delimiter)) {
|
|
3373
|
-
if (dir && isExecutable(
|
|
4370
|
+
if (dir && isExecutable(join5(dir, command)))
|
|
3374
4371
|
return true;
|
|
3375
4372
|
}
|
|
3376
4373
|
return false;
|
|
@@ -3380,10 +4377,20 @@ function commandNotFoundMessage(command, env = process.env) {
|
|
|
3380
4377
|
}
|
|
3381
4378
|
|
|
3382
4379
|
// src/lib/machines.ts
|
|
3383
|
-
import {
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
4380
|
+
import { createRequire } from "module";
|
|
4381
|
+
var consumerModule;
|
|
4382
|
+
function machinesConsumer() {
|
|
4383
|
+
if (!consumerModule) {
|
|
4384
|
+
try {
|
|
4385
|
+
const resolved = Bun.resolveSync("@hasna/machines/consumer", import.meta.dir);
|
|
4386
|
+
consumerModule = createRequire(import.meta.url)(resolved);
|
|
4387
|
+
} catch (error) {
|
|
4388
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
4389
|
+
throw new Error(`@hasna/machines is not available; install the optional dependency to use machine-assigned loops: ${detail}`);
|
|
4390
|
+
}
|
|
4391
|
+
}
|
|
4392
|
+
return consumerModule;
|
|
4393
|
+
}
|
|
3387
4394
|
function compact(value) {
|
|
3388
4395
|
const text = value?.trim();
|
|
3389
4396
|
return text ? text : undefined;
|
|
@@ -3420,14 +4427,18 @@ function machineFromRoute(route, topology) {
|
|
|
3420
4427
|
};
|
|
3421
4428
|
}
|
|
3422
4429
|
function listOpenMachines() {
|
|
3423
|
-
const topology = discoverMachineTopology();
|
|
4430
|
+
const topology = machinesConsumer().discoverMachineTopology();
|
|
3424
4431
|
return topology.machines.map((entry) => entryToSummary(entry, topology));
|
|
3425
4432
|
}
|
|
3426
4433
|
function resolveLoopMachine(machineId) {
|
|
3427
|
-
const
|
|
3428
|
-
const
|
|
4434
|
+
const consumer = machinesConsumer();
|
|
4435
|
+
const topology = consumer.discoverMachineTopology();
|
|
4436
|
+
const route = consumer.resolveMachineRoute(machineId, { topology });
|
|
3429
4437
|
return machineFromRoute(route, topology);
|
|
3430
4438
|
}
|
|
4439
|
+
function resolveMachineCommand(machineId, command) {
|
|
4440
|
+
return machinesConsumer().resolveMachineCommand(machineId, command);
|
|
4441
|
+
}
|
|
3431
4442
|
function refreshLoopMachine(machine) {
|
|
3432
4443
|
return resolveLoopMachine(machine.id);
|
|
3433
4444
|
}
|
|
@@ -3435,6 +4446,10 @@ function refreshLoopMachine(machine) {
|
|
|
3435
4446
|
// src/lib/executor.ts
|
|
3436
4447
|
var DEFAULT_TIMEOUT_MS = 30 * 60000;
|
|
3437
4448
|
var DEFAULT_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
4449
|
+
var DEFAULT_AGENT_IDLE_TIMEOUT_MS = 30 * 60000;
|
|
4450
|
+
var BUFFERED_OUTPUT_PROVIDERS = new Set(["claude", "codewith", "opencode", "aicopilot"]);
|
|
4451
|
+
var DEFAULT_BUFFERED_AGENT_IDLE_TIMEOUT_MS = 4 * 60 * 60000;
|
|
4452
|
+
var WORKTREE_GIT_TIMEOUT_MS = 5 * 60000;
|
|
3438
4453
|
var AUTH_ENV_KEYS = [
|
|
3439
4454
|
"CLAUDE_CONFIG_DIR",
|
|
3440
4455
|
"CODEWITH_HOME",
|
|
@@ -3470,39 +4485,47 @@ var TRANSPORT_ENV_KEYS = new Set([
|
|
|
3470
4485
|
"USER",
|
|
3471
4486
|
"XDG_RUNTIME_DIR"
|
|
3472
4487
|
]);
|
|
3473
|
-
function
|
|
3474
|
-
const
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
const overflow = Buffer.byteLength(next, "utf8") - maxBytes;
|
|
3478
|
-
return `[truncated ${overflow} bytes]
|
|
3479
|
-
${next.slice(-maxBytes)}`;
|
|
3480
|
-
}
|
|
3481
|
-
function killProcessGroup(pid) {
|
|
3482
|
-
try {
|
|
3483
|
-
process.kill(-pid, "SIGTERM");
|
|
3484
|
-
} catch {
|
|
3485
|
-
try {
|
|
3486
|
-
process.kill(pid, "SIGTERM");
|
|
3487
|
-
} catch {}
|
|
3488
|
-
}
|
|
3489
|
-
setTimeout(() => {
|
|
3490
|
-
try {
|
|
3491
|
-
process.kill(-pid, "SIGKILL");
|
|
3492
|
-
} catch {
|
|
3493
|
-
try {
|
|
3494
|
-
process.kill(pid, "SIGKILL");
|
|
3495
|
-
} catch {}
|
|
3496
|
-
}
|
|
3497
|
-
}, 2000).unref();
|
|
3498
|
-
}
|
|
3499
|
-
function shellQuote(value) {
|
|
3500
|
-
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
4488
|
+
function boundedText(text, maxBytes) {
|
|
4489
|
+
const buffer = new BoundedOutputBuffer(maxBytes);
|
|
4490
|
+
buffer.append(text);
|
|
4491
|
+
return buffer.value();
|
|
3501
4492
|
}
|
|
3502
|
-
function
|
|
3503
|
-
const
|
|
3504
|
-
|
|
3505
|
-
|
|
4493
|
+
function buildResult(status, startedAt, fields = {}) {
|
|
4494
|
+
const finishedAt = fields.finishedAt ?? nowIso();
|
|
4495
|
+
return {
|
|
4496
|
+
status,
|
|
4497
|
+
exitCode: fields.exitCode,
|
|
4498
|
+
stdout: fields.stdout ?? "",
|
|
4499
|
+
stderr: fields.stderr ?? "",
|
|
4500
|
+
error: fields.error,
|
|
4501
|
+
pid: fields.pid,
|
|
4502
|
+
startedAt,
|
|
4503
|
+
finishedAt,
|
|
4504
|
+
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
4505
|
+
};
|
|
4506
|
+
}
|
|
4507
|
+
function failureResult(startedAt, error, fields = {}) {
|
|
4508
|
+
return buildResult("failed", startedAt, { ...fields, error });
|
|
4509
|
+
}
|
|
4510
|
+
function timeoutResult(startedAt, error, fields = {}) {
|
|
4511
|
+
return buildResult("timed_out", startedAt, { ...fields, error });
|
|
4512
|
+
}
|
|
4513
|
+
function successResult(startedAt, fields = {}) {
|
|
4514
|
+
return buildResult("succeeded", startedAt, fields);
|
|
4515
|
+
}
|
|
4516
|
+
function notifySpawn(pid, opts) {
|
|
4517
|
+
if (!pid)
|
|
4518
|
+
return;
|
|
4519
|
+
opts.onSpawn?.(pid);
|
|
4520
|
+
opts.onSpawnProcess?.({ pid, pgid: pid, processStartedAt: nowIso() });
|
|
4521
|
+
}
|
|
4522
|
+
function shellQuote(value) {
|
|
4523
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
4524
|
+
}
|
|
4525
|
+
function metadataEnv(metadata) {
|
|
4526
|
+
const env = {};
|
|
4527
|
+
if (metadata.loopId)
|
|
4528
|
+
env.LOOPS_LOOP_ID = metadata.loopId;
|
|
3506
4529
|
if (metadata.loopName)
|
|
3507
4530
|
env.LOOPS_LOOP_NAME = metadata.loopName;
|
|
3508
4531
|
if (metadata.runId)
|
|
@@ -3525,6 +4548,21 @@ function metadataEnv(metadata) {
|
|
|
3525
4548
|
env.LOOPS_GOAL_NODE_KEY = metadata.goalNodeKey;
|
|
3526
4549
|
return env;
|
|
3527
4550
|
}
|
|
4551
|
+
function codewithAgentIdempotencyKey(metadata) {
|
|
4552
|
+
const parts = [
|
|
4553
|
+
"openloops",
|
|
4554
|
+
metadata.workflowRunId ? `workflow-run:${metadata.workflowRunId}` : undefined,
|
|
4555
|
+
metadata.workflowStepId ? `step:${metadata.workflowStepId}` : undefined,
|
|
4556
|
+
metadata.runId ? `loop-run:${metadata.runId}` : undefined,
|
|
4557
|
+
metadata.loopId ? `loop:${metadata.loopId}` : undefined,
|
|
4558
|
+
metadata.scheduledFor ? `scheduled:${metadata.scheduledFor}` : undefined,
|
|
4559
|
+
metadata.goalId ? `goal:${metadata.goalId}` : undefined,
|
|
4560
|
+
metadata.goalNodeKey ? `node:${metadata.goalNodeKey}` : undefined
|
|
4561
|
+
].filter(Boolean);
|
|
4562
|
+
if (parts.length > 1)
|
|
4563
|
+
return parts.join(":");
|
|
4564
|
+
return `openloops:adhoc:${randomBytes2(16).toString("hex")}`;
|
|
4565
|
+
}
|
|
3528
4566
|
function allowlistEnv(allowlist) {
|
|
3529
4567
|
const env = {};
|
|
3530
4568
|
if (allowlist?.tools?.length)
|
|
@@ -3535,207 +4573,21 @@ function allowlistEnv(allowlist) {
|
|
|
3535
4573
|
env.LOOPS_AGENT_ALLOWLIST_ENFORCEMENT = "metadata_only";
|
|
3536
4574
|
return env;
|
|
3537
4575
|
}
|
|
3538
|
-
function
|
|
3539
|
-
|
|
3540
|
-
case "claude":
|
|
3541
|
-
return "claude";
|
|
3542
|
-
case "cursor":
|
|
3543
|
-
return "sh";
|
|
3544
|
-
case "codewith":
|
|
3545
|
-
return "codewith";
|
|
3546
|
-
case "aicopilot":
|
|
3547
|
-
return "aicopilot";
|
|
3548
|
-
case "opencode":
|
|
3549
|
-
return "opencode";
|
|
3550
|
-
case "codex":
|
|
3551
|
-
return "codex";
|
|
3552
|
-
}
|
|
3553
|
-
}
|
|
3554
|
-
function codewithLikeSandbox(target) {
|
|
3555
|
-
const sandbox = target.sandbox ?? (target.permissionMode === "bypass" ? "danger-full-access" : "workspace-write");
|
|
3556
|
-
if (sandbox !== "read-only" && sandbox !== "workspace-write" && sandbox !== "danger-full-access") {
|
|
3557
|
-
throw new Error(`${target.provider} sandbox must be read-only, workspace-write, or danger-full-access`);
|
|
3558
|
-
}
|
|
3559
|
-
return sandbox;
|
|
3560
|
-
}
|
|
3561
|
-
function configStringValue(value) {
|
|
3562
|
-
return JSON.stringify(value);
|
|
3563
|
-
}
|
|
3564
|
-
function assertStringOption(value, label) {
|
|
3565
|
-
if (value !== undefined && typeof value !== "string")
|
|
3566
|
-
throw new Error(`${label} must be a string`);
|
|
3567
|
-
}
|
|
3568
|
-
function assertSupportedAgentOptions(target) {
|
|
3569
|
-
assertStringOption(target.variant, `${target.provider}.variant`);
|
|
3570
|
-
assertStringOption(target.model, `${target.provider}.model`);
|
|
3571
|
-
assertStringOption(target.agent, `${target.provider}.agent`);
|
|
3572
|
-
assertStringOption(target.authProfile, `${target.provider}.authProfile`);
|
|
3573
|
-
assertStringOption(target.configIsolation, `${target.provider}.configIsolation`);
|
|
3574
|
-
if (target.provider === "opencode" && (target.model === undefined || target.model.trim() === "")) {
|
|
3575
|
-
throw new Error("opencode.model is required; pass a provider/model id such as openrouter/google/gemini-2.5-flash");
|
|
3576
|
-
}
|
|
3577
|
-
if (target.configIsolation !== undefined && target.configIsolation !== "safe" && target.configIsolation !== "none") {
|
|
3578
|
-
throw new Error(`${target.provider}.configIsolation must be safe or none`);
|
|
3579
|
-
}
|
|
3580
|
-
if (target.authProfile !== undefined && target.provider !== "codewith") {
|
|
3581
|
-
throw new Error(`${target.provider}.authProfile is supported only for codewith`);
|
|
3582
|
-
}
|
|
3583
|
-
if (target.addDirs?.length && !["codewith", "codex"].includes(target.provider)) {
|
|
3584
|
-
throw new Error(`${target.provider}.addDirs is currently supported only for codewith or codex`);
|
|
3585
|
-
}
|
|
3586
|
-
if (target.permissionMode && !["default", "plan", "auto", "bypass"].includes(target.permissionMode)) {
|
|
3587
|
-
throw new Error(`${target.provider}.permissionMode must be default, plan, auto, or bypass`);
|
|
3588
|
-
}
|
|
3589
|
-
if (target.sandbox && !["read-only", "workspace-write", "danger-full-access", "enabled", "disabled"].includes(target.sandbox)) {
|
|
3590
|
-
throw new Error(`${target.provider}.sandbox is not supported: ${target.sandbox}`);
|
|
3591
|
-
}
|
|
3592
|
-
if (target.provider === "codex" && target.agent !== undefined)
|
|
3593
|
-
throw new Error("codex.agent is not supported");
|
|
3594
|
-
if (["codewith", "codex"].includes(target.provider)) {
|
|
3595
|
-
if (target.permissionMode && !["default", "bypass"].includes(target.permissionMode)) {
|
|
3596
|
-
throw new Error(`${target.provider}.permissionMode supports only default or bypass`);
|
|
3597
|
-
}
|
|
3598
|
-
if (target.sandbox)
|
|
3599
|
-
codewithLikeSandbox(target);
|
|
3600
|
-
return;
|
|
3601
|
-
}
|
|
3602
|
-
if (target.provider === "claude") {
|
|
3603
|
-
if (target.sandbox !== undefined)
|
|
3604
|
-
throw new Error("claude.sandbox is not supported");
|
|
4576
|
+
function defaultAgentIdleTimeoutMs(target, opts) {
|
|
4577
|
+
if (target.timeoutMs !== undefined || target.idleTimeoutMs !== undefined)
|
|
3605
4578
|
return;
|
|
4579
|
+
const raw = opts.env?.LOOPS_AGENT_IDLE_TIMEOUT_MS ?? process.env.LOOPS_AGENT_IDLE_TIMEOUT_MS;
|
|
4580
|
+
if (raw !== undefined && raw !== "") {
|
|
4581
|
+
const normalized = raw.trim().toLowerCase();
|
|
4582
|
+
if (normalized === "0" || normalized === "none" || normalized === "off")
|
|
4583
|
+
return;
|
|
4584
|
+
const parsed = Number(normalized);
|
|
4585
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
4586
|
+
return parsed;
|
|
3606
4587
|
}
|
|
3607
|
-
|
|
3608
|
-
if (target.variant !== undefined)
|
|
3609
|
-
throw new Error("cursor.variant is not supported");
|
|
3610
|
-
if (target.permissionMode === "auto")
|
|
3611
|
-
throw new Error("cursor.permissionMode auto is not supported; use provider-specific extraArgs for Cursor auto-review");
|
|
3612
|
-
if (target.sandbox !== undefined && target.sandbox !== "enabled" && target.sandbox !== "disabled") {
|
|
3613
|
-
throw new Error("cursor.sandbox must be enabled or disabled");
|
|
3614
|
-
}
|
|
3615
|
-
return;
|
|
3616
|
-
}
|
|
3617
|
-
if (target.permissionMode && !["default", "bypass"].includes(target.permissionMode)) {
|
|
3618
|
-
throw new Error(`${target.provider}.permissionMode supports only default or bypass`);
|
|
3619
|
-
}
|
|
3620
|
-
if (target.sandbox !== undefined)
|
|
3621
|
-
throw new Error(`${target.provider}.sandbox is not supported`);
|
|
3622
|
-
}
|
|
3623
|
-
function agentArgs(target) {
|
|
3624
|
-
assertSupportedAgentOptions(target);
|
|
3625
|
-
const isolation = target.configIsolation ?? "safe";
|
|
3626
|
-
const permissionMode = target.permissionMode ?? "default";
|
|
3627
|
-
const args = [];
|
|
3628
|
-
switch (target.provider) {
|
|
3629
|
-
case "claude":
|
|
3630
|
-
if (isolation === "safe")
|
|
3631
|
-
args.push("--safe-mode", "--setting-sources", "local", "--no-session-persistence");
|
|
3632
|
-
if (permissionMode !== "default") {
|
|
3633
|
-
const mode = permissionMode === "bypass" ? "bypassPermissions" : permissionMode === "plan" || permissionMode === "auto" ? permissionMode : undefined;
|
|
3634
|
-
if (mode)
|
|
3635
|
-
args.push("--permission-mode", mode);
|
|
3636
|
-
}
|
|
3637
|
-
args.push("-p", "--output-format", "json");
|
|
3638
|
-
if (target.model)
|
|
3639
|
-
args.push("--model", target.model);
|
|
3640
|
-
if (target.variant)
|
|
3641
|
-
args.push("--effort", target.variant);
|
|
3642
|
-
if (target.agent)
|
|
3643
|
-
args.push("--agent", target.agent);
|
|
3644
|
-
args.push(...target.extraArgs ?? []);
|
|
3645
|
-
return args;
|
|
3646
|
-
case "cursor":
|
|
3647
|
-
args.push("-c", [
|
|
3648
|
-
"set -eu",
|
|
3649
|
-
"if command -v agent >/dev/null 2>&1; then",
|
|
3650
|
-
' exec agent "$@"',
|
|
3651
|
-
"else",
|
|
3652
|
-
" echo 'Executable not found in PATH: agent' >&2",
|
|
3653
|
-
" exit 127",
|
|
3654
|
-
"fi"
|
|
3655
|
-
].join(`
|
|
3656
|
-
`), "openloops-cursor", "-p");
|
|
3657
|
-
if (permissionMode === "plan")
|
|
3658
|
-
args.push("--mode", "plan");
|
|
3659
|
-
if (permissionMode === "bypass")
|
|
3660
|
-
args.push("--force");
|
|
3661
|
-
const cursorSandbox = target.sandbox ?? (isolation === "safe" ? "enabled" : undefined);
|
|
3662
|
-
if (cursorSandbox) {
|
|
3663
|
-
if (cursorSandbox !== "enabled" && cursorSandbox !== "disabled")
|
|
3664
|
-
throw new Error("cursor sandbox must be enabled or disabled");
|
|
3665
|
-
args.push("--sandbox", cursorSandbox);
|
|
3666
|
-
}
|
|
3667
|
-
if (target.model)
|
|
3668
|
-
args.push("--model", target.model);
|
|
3669
|
-
if (target.agent)
|
|
3670
|
-
args.push("--agent", target.agent);
|
|
3671
|
-
args.push(...target.extraArgs ?? []);
|
|
3672
|
-
return args;
|
|
3673
|
-
case "codewith":
|
|
3674
|
-
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : []);
|
|
3675
|
-
if (target.variant)
|
|
3676
|
-
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
3677
|
-
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
3678
|
-
if (isolation === "safe")
|
|
3679
|
-
args.push("--ignore-rules");
|
|
3680
|
-
if (target.cwd)
|
|
3681
|
-
args.push("--cd", target.cwd);
|
|
3682
|
-
for (const dir of target.addDirs ?? [])
|
|
3683
|
-
args.push("--add-dir", dir);
|
|
3684
|
-
if (target.model)
|
|
3685
|
-
args.push("--model", target.model);
|
|
3686
|
-
if (target.agent)
|
|
3687
|
-
args.push("--agent", target.agent);
|
|
3688
|
-
args.push(...target.extraArgs ?? []);
|
|
3689
|
-
return args;
|
|
3690
|
-
case "codex":
|
|
3691
|
-
if (target.variant)
|
|
3692
|
-
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
3693
|
-
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
3694
|
-
if (isolation === "safe")
|
|
3695
|
-
args.push("--ignore-rules");
|
|
3696
|
-
if (target.cwd)
|
|
3697
|
-
args.push("--cd", target.cwd);
|
|
3698
|
-
for (const dir of target.addDirs ?? [])
|
|
3699
|
-
args.push("--add-dir", dir);
|
|
3700
|
-
if (target.model)
|
|
3701
|
-
args.push("--model", target.model);
|
|
3702
|
-
args.push(...target.extraArgs ?? []);
|
|
3703
|
-
return args;
|
|
3704
|
-
case "aicopilot":
|
|
3705
|
-
args.push("run", "--format", "json");
|
|
3706
|
-
if (isolation === "safe")
|
|
3707
|
-
args.push("--pure");
|
|
3708
|
-
if (permissionMode === "bypass")
|
|
3709
|
-
args.push("--dangerously-skip-permissions");
|
|
3710
|
-
if (target.cwd)
|
|
3711
|
-
args.push("--dir", target.cwd);
|
|
3712
|
-
if (target.model)
|
|
3713
|
-
args.push("--model", target.model);
|
|
3714
|
-
if (target.variant)
|
|
3715
|
-
args.push("--variant", target.variant);
|
|
3716
|
-
if (target.agent)
|
|
3717
|
-
args.push("--agent", target.agent);
|
|
3718
|
-
args.push(...target.extraArgs ?? []);
|
|
3719
|
-
return args;
|
|
3720
|
-
case "opencode":
|
|
3721
|
-
args.push("run", "--format", "json");
|
|
3722
|
-
if (isolation === "safe")
|
|
3723
|
-
args.push("--pure");
|
|
3724
|
-
if (permissionMode === "bypass")
|
|
3725
|
-
args.push("--dangerously-skip-permissions");
|
|
3726
|
-
if (target.cwd)
|
|
3727
|
-
args.push("--dir", target.cwd);
|
|
3728
|
-
if (target.model)
|
|
3729
|
-
args.push("--model", target.model);
|
|
3730
|
-
if (target.variant)
|
|
3731
|
-
args.push("--variant", target.variant);
|
|
3732
|
-
if (target.agent)
|
|
3733
|
-
args.push("--agent", target.agent);
|
|
3734
|
-
args.push(...target.extraArgs ?? []);
|
|
3735
|
-
return args;
|
|
3736
|
-
}
|
|
4588
|
+
return BUFFERED_OUTPUT_PROVIDERS.has(target.provider) ? DEFAULT_BUFFERED_AGENT_IDLE_TIMEOUT_MS : DEFAULT_AGENT_IDLE_TIMEOUT_MS;
|
|
3737
4589
|
}
|
|
3738
|
-
function commandSpec(target) {
|
|
4590
|
+
function commandSpec(target, opts) {
|
|
3739
4591
|
if (target.type === "command") {
|
|
3740
4592
|
const commandTarget = target;
|
|
3741
4593
|
return {
|
|
@@ -3751,24 +4603,27 @@ function commandSpec(target) {
|
|
|
3751
4603
|
};
|
|
3752
4604
|
}
|
|
3753
4605
|
const agentTarget = target;
|
|
4606
|
+
const adapter = providerAdapter(agentTarget.provider);
|
|
4607
|
+
const invocation = adapter.buildInvocation(agentTarget);
|
|
3754
4608
|
return {
|
|
3755
|
-
command:
|
|
3756
|
-
args:
|
|
4609
|
+
command: invocation.command,
|
|
4610
|
+
args: invocation.args,
|
|
3757
4611
|
cwd: agentTarget.cwd,
|
|
3758
4612
|
timeoutMs: agentTarget.timeoutMs ?? null,
|
|
3759
|
-
idleTimeoutMs: agentTarget.idleTimeoutMs,
|
|
4613
|
+
idleTimeoutMs: agentTarget.idleTimeoutMs ?? defaultAgentIdleTimeoutMs(agentTarget, opts),
|
|
3760
4614
|
account: agentTarget.account,
|
|
3761
4615
|
accountTool: agentTarget.account?.tool ?? accountToolForProvider(agentTarget.provider),
|
|
3762
4616
|
nativeAuthProfile: agentTarget.authProfile ? { provider: agentTarget.provider, profile: agentTarget.authProfile } : undefined,
|
|
3763
|
-
preflightAnyOf:
|
|
3764
|
-
stdin:
|
|
3765
|
-
allowlist: agentTarget.allowlist
|
|
4617
|
+
preflightAnyOf: invocation.preflightAnyOf,
|
|
4618
|
+
stdin: invocation.stdin,
|
|
4619
|
+
allowlist: agentTarget.allowlist,
|
|
4620
|
+
worktree: agentTarget.worktree,
|
|
4621
|
+
codewithDurableAgent: adapter.capabilities.durable ? { target: agentTarget } : undefined
|
|
3766
4622
|
};
|
|
3767
4623
|
}
|
|
3768
|
-
function
|
|
4624
|
+
function composeExecutionEnv(spec, metadata, opts, accountEnv) {
|
|
3769
4625
|
const env = { ...opts.env ?? process.env };
|
|
3770
|
-
if (
|
|
3771
|
-
const accountEnv = resolveAccountEnv(spec.account, spec.accountTool, env);
|
|
4626
|
+
if (accountEnv) {
|
|
3772
4627
|
for (const key of AUTH_ENV_KEYS)
|
|
3773
4628
|
delete env[key];
|
|
3774
4629
|
Object.assign(env, accountEnv);
|
|
@@ -3779,6 +4634,14 @@ function executionEnv(spec, metadata, opts) {
|
|
|
3779
4634
|
Object.assign(env, metadataEnv(metadata));
|
|
3780
4635
|
return env;
|
|
3781
4636
|
}
|
|
4637
|
+
async function executionEnv(spec, metadata, opts) {
|
|
4638
|
+
const accountEnv = spec.account ? await resolveAccountEnv(spec.account, spec.accountTool, { ...opts.env ?? process.env }) : undefined;
|
|
4639
|
+
return composeExecutionEnv(spec, metadata, opts, accountEnv);
|
|
4640
|
+
}
|
|
4641
|
+
function executionEnvSync(spec, metadata, opts) {
|
|
4642
|
+
const accountEnv = spec.account ? resolveAccountEnvSync(spec.account, spec.accountTool, { ...opts.env ?? process.env }) : undefined;
|
|
4643
|
+
return composeExecutionEnv(spec, metadata, opts, accountEnv);
|
|
4644
|
+
}
|
|
3782
4645
|
function resolvedMachine(opts) {
|
|
3783
4646
|
if (!opts.machine)
|
|
3784
4647
|
return;
|
|
@@ -3796,13 +4659,21 @@ function hereDoc(value) {
|
|
|
3796
4659
|
}
|
|
3797
4660
|
return [`cat > "$__OPENLOOPS_STDIN" <<'${delimiter2}'`, value, delimiter2];
|
|
3798
4661
|
}
|
|
3799
|
-
function remoteBootstrapLines(spec, metadata) {
|
|
4662
|
+
function remoteBootstrapLines(spec, metadata, opts = {}) {
|
|
3800
4663
|
const lines = [
|
|
3801
4664
|
"set -e",
|
|
3802
4665
|
'export PATH="$HOME/.local/bin:$HOME/.bun/bin:$HOME/.cargo/bin:$HOME/.npm-global/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${PATH:+:$PATH}"'
|
|
3803
4666
|
];
|
|
3804
|
-
|
|
4667
|
+
const worktree = spec.worktree;
|
|
4668
|
+
const worktreeManaged = worktree?.enabled && (worktree.mode === "auto" || worktree.mode === "required");
|
|
4669
|
+
if (opts.worktree && worktree && worktreeManaged) {
|
|
4670
|
+
lines.push(...remoteWorktreePrepareLines(worktree));
|
|
4671
|
+
lines.push(...remoteWorktreeEnterLines(worktree, spec.cwd));
|
|
4672
|
+
} else if (worktree && worktreeManaged) {
|
|
4673
|
+
lines.push(`cd ${shellQuote(worktree.originalCwd)}`);
|
|
4674
|
+
} else if (spec.cwd) {
|
|
3805
4675
|
lines.push(`cd ${shellQuote(spec.cwd)}`);
|
|
4676
|
+
}
|
|
3806
4677
|
if (spec.account) {
|
|
3807
4678
|
if (!spec.accountTool)
|
|
3808
4679
|
throw new Error("account.tool is required when no provider tool can be inferred");
|
|
@@ -3818,16 +4689,79 @@ function remoteBootstrapLines(spec, metadata) {
|
|
|
3818
4689
|
}
|
|
3819
4690
|
return lines;
|
|
3820
4691
|
}
|
|
3821
|
-
function
|
|
3822
|
-
const
|
|
4692
|
+
function remoteWorktreePrepareLines(worktree) {
|
|
4693
|
+
const { repoRoot, path, branch } = worktree;
|
|
4694
|
+
if (!repoRoot || !path || !branch) {
|
|
4695
|
+
return [
|
|
4696
|
+
"__openloops_prepare_worktree() {",
|
|
4697
|
+
` echo ${shellQuote("worktree preparation requires repoRoot, path, and branch metadata")} >&2`,
|
|
4698
|
+
" return 1",
|
|
4699
|
+
"}"
|
|
4700
|
+
];
|
|
4701
|
+
}
|
|
4702
|
+
return [
|
|
4703
|
+
"__openloops_prepare_worktree() {",
|
|
4704
|
+
` local repo=${shellQuote(repoRoot)} path=${shellQuote(path)} branch=${shellQuote(branch)}`,
|
|
4705
|
+
" local top expected_common actual_common current",
|
|
4706
|
+
' if [ -L "$path" ]; then echo "refusing symlinked worktree path $path" >&2; return 1; fi',
|
|
4707
|
+
' if [ -e "$path" ]; then',
|
|
4708
|
+
' top="$(git -C "$path" rev-parse --show-toplevel 2>/dev/null)" || { echo "refusing to reuse non-worktree path: $path" >&2; return 1; }',
|
|
4709
|
+
' if [ "$(cd "$top" 2>/dev/null && pwd -P)" != "$(cd "$path" 2>/dev/null && pwd -P)" ]; then echo "existing worktree top-level mismatch for $path: $top" >&2; return 1; fi',
|
|
4710
|
+
' expected_common="$(cd "$repo" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
|
|
4711
|
+
' actual_common="$(cd "$path" 2>/dev/null && cd "$(git rev-parse --git-common-dir 2>/dev/null)" 2>/dev/null && pwd -P)"',
|
|
4712
|
+
' if [ -z "$expected_common" ] || [ "$expected_common" != "$actual_common" ]; then echo "existing worktree $path belongs to a different git common dir" >&2; return 1; fi',
|
|
4713
|
+
' current="$(git -C "$path" branch --show-current 2>/dev/null)"',
|
|
4714
|
+
' if [ "$current" != "$branch" ]; then echo "existing worktree $path is on branch ${current:-unknown}, expected $branch" >&2; return 1; fi',
|
|
4715
|
+
" return 0",
|
|
4716
|
+
" fi",
|
|
4717
|
+
' git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || { echo "worktree repoRoot is not a git repository: $repo" >&2; return 1; }',
|
|
4718
|
+
' mkdir -p "$(dirname "$path")" || return 1',
|
|
4719
|
+
" # Preparation chatter goes to stderr so run stdout stays the agent's.",
|
|
4720
|
+
' if git -C "$repo" show-ref --verify --quiet "refs/heads/$branch"; then',
|
|
4721
|
+
' git -C "$repo" worktree add "$path" "$branch" 1>&2 || return 1',
|
|
4722
|
+
" else",
|
|
4723
|
+
' git -C "$repo" worktree add -b "$branch" "$path" HEAD 1>&2 || return 1',
|
|
4724
|
+
" fi",
|
|
4725
|
+
"}"
|
|
4726
|
+
];
|
|
4727
|
+
}
|
|
4728
|
+
function remoteWorktreeEnterLines(worktree, cwd) {
|
|
4729
|
+
const workdir = cwd ?? worktree.cwd;
|
|
4730
|
+
if (worktree.mode === "required") {
|
|
4731
|
+
return [
|
|
4732
|
+
"if ! __openloops_prepare_worktree; then",
|
|
4733
|
+
` echo ${shellQuote("worktree preparation failed (mode=required)")} >&2`,
|
|
4734
|
+
" exit 1",
|
|
4735
|
+
"fi",
|
|
4736
|
+
"__OPENLOOPS_WORKTREE_OK=1",
|
|
4737
|
+
`cd ${shellQuote(workdir)}`
|
|
4738
|
+
];
|
|
4739
|
+
}
|
|
4740
|
+
return [
|
|
4741
|
+
"if __openloops_prepare_worktree; then",
|
|
4742
|
+
" __OPENLOOPS_WORKTREE_OK=1",
|
|
4743
|
+
` cd ${shellQuote(workdir)}`,
|
|
4744
|
+
"else",
|
|
4745
|
+
` echo ${shellQuote(`worktree preparation failed (mode=${worktree.mode}); falling back to ${worktree.originalCwd}`)} >&2`,
|
|
4746
|
+
" __OPENLOOPS_WORKTREE_OK=0",
|
|
4747
|
+
` cd ${shellQuote(worktree.originalCwd)}`,
|
|
4748
|
+
"fi"
|
|
4749
|
+
];
|
|
4750
|
+
}
|
|
4751
|
+
function remoteScript(spec, metadata, fallbackSpec) {
|
|
4752
|
+
const lines = remoteBootstrapLines(spec, metadata, { worktree: true });
|
|
3823
4753
|
let stdinRedirect = "";
|
|
3824
4754
|
if (spec.stdin !== undefined) {
|
|
3825
4755
|
lines.push('__OPENLOOPS_STDIN="$(mktemp -t openloops-stdin.XXXXXX)"', `trap 'rm -f "$__OPENLOOPS_STDIN"' EXIT`);
|
|
3826
4756
|
lines.push(...hereDoc(spec.stdin));
|
|
3827
4757
|
stdinRedirect = ' < "$__OPENLOOPS_STDIN"';
|
|
3828
4758
|
}
|
|
3829
|
-
const
|
|
3830
|
-
|
|
4759
|
+
const invocationFor = (invocationSpec) => invocationSpec.shell ? `sh -c ${shellQuote(commandForShell(invocationSpec))}${stdinRedirect}` : `${[invocationSpec.command, ...invocationSpec.args].map(shellQuote).join(" ")}${stdinRedirect}`;
|
|
4760
|
+
if (spec.worktree?.enabled && spec.worktree.mode === "auto" && fallbackSpec) {
|
|
4761
|
+
lines.push('if [ "${__OPENLOOPS_WORKTREE_OK:-0}" = 1 ]; then', ` ${invocationFor(spec)}`, "else", ` ${invocationFor(fallbackSpec)}`, "fi");
|
|
4762
|
+
} else {
|
|
4763
|
+
lines.push(invocationFor(spec));
|
|
4764
|
+
}
|
|
3831
4765
|
return `${lines.join(`
|
|
3832
4766
|
`)}
|
|
3833
4767
|
`;
|
|
@@ -3859,32 +4793,333 @@ function transportEnv(opts) {
|
|
|
3859
4793
|
env.PATH = normalizeExecutionPath(env);
|
|
3860
4794
|
return env;
|
|
3861
4795
|
}
|
|
3862
|
-
function
|
|
3863
|
-
if (
|
|
3864
|
-
|
|
3865
|
-
|
|
4796
|
+
function assertCodewithProfileListed(profile, result) {
|
|
4797
|
+
if (result.error) {
|
|
4798
|
+
throw new Error(`codewith auth profile preflight failed: ${result.error}`);
|
|
4799
|
+
}
|
|
4800
|
+
if ((result.status ?? 1) !== 0) {
|
|
4801
|
+
const detail = (result.stderr || result.stdout || `exit ${result.status ?? "unknown"}`).trim();
|
|
4802
|
+
throw new Error(`codewith auth profile preflight failed${detail ? `: ${detail}` : ""}`);
|
|
4803
|
+
}
|
|
4804
|
+
const profiles = new Set((result.stdout || "").split(/\r?\n/).slice(1).map((line) => line.trim().split(/\s+/)[0]).filter(Boolean));
|
|
4805
|
+
if (!profiles.has(profile)) {
|
|
4806
|
+
throw new Error(`codewith auth profile not found: ${profile}`);
|
|
4807
|
+
}
|
|
4808
|
+
}
|
|
4809
|
+
function preflightNativeAuthProfileSync(spec, env) {
|
|
4810
|
+
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
3866
4811
|
return;
|
|
3867
|
-
const result =
|
|
4812
|
+
const result = spawnSync4(spec.command, ["profile", "list"], {
|
|
3868
4813
|
encoding: "utf8",
|
|
3869
4814
|
env,
|
|
3870
4815
|
stdio: ["ignore", "pipe", "pipe"],
|
|
3871
4816
|
timeout: 15000
|
|
3872
4817
|
});
|
|
3873
|
-
|
|
3874
|
-
|
|
4818
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, {
|
|
4819
|
+
status: result.status,
|
|
4820
|
+
stdout: result.stdout ?? "",
|
|
4821
|
+
stderr: result.stderr ?? "",
|
|
4822
|
+
error: result.error?.message
|
|
4823
|
+
});
|
|
4824
|
+
}
|
|
4825
|
+
async function preflightNativeAuthProfile(spec, env) {
|
|
4826
|
+
if (spec.nativeAuthProfile?.provider !== "codewith")
|
|
4827
|
+
return;
|
|
4828
|
+
const result = await spawnCapture(spec.command, ["profile", "list"], { env, timeoutMs: 15000 });
|
|
4829
|
+
assertCodewithProfileListed(spec.nativeAuthProfile.profile, result);
|
|
4830
|
+
}
|
|
4831
|
+
function codewithAgentStartArgs(target, idempotencyKey) {
|
|
4832
|
+
const args = providerAdapter(target.provider).buildInvocation(target).args;
|
|
4833
|
+
const startIndex = args.findIndex((arg, index) => arg === "start" && args[index - 1] === "agent");
|
|
4834
|
+
if (startIndex === -1)
|
|
4835
|
+
throw new Error("internal error: codewith durable agent args missing agent start");
|
|
4836
|
+
args.splice(startIndex + 1, 0, "--idempotency-key", idempotencyKey);
|
|
4837
|
+
return args;
|
|
4838
|
+
}
|
|
4839
|
+
function codewithAgentControlArgs(target, command, agentId) {
|
|
4840
|
+
return [
|
|
4841
|
+
...target.authProfile ? ["--auth-profile", target.authProfile] : [],
|
|
4842
|
+
"agent",
|
|
4843
|
+
command,
|
|
4844
|
+
...command === "logs" ? ["--limit", "20"] : [],
|
|
4845
|
+
agentId
|
|
4846
|
+
];
|
|
4847
|
+
}
|
|
4848
|
+
function parseJsonOutput(stdout, label) {
|
|
4849
|
+
try {
|
|
4850
|
+
const value = JSON.parse(stdout || "{}");
|
|
4851
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
4852
|
+
throw new Error("not an object");
|
|
4853
|
+
return value;
|
|
4854
|
+
} catch (error) {
|
|
4855
|
+
throw new Error(`${label} did not return JSON${error instanceof Error ? `: ${error.message}` : ""}`);
|
|
3875
4856
|
}
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
4857
|
+
}
|
|
4858
|
+
function recordField(value, key) {
|
|
4859
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
4860
|
+
return;
|
|
4861
|
+
const field = value[key];
|
|
4862
|
+
return field && typeof field === "object" && !Array.isArray(field) ? field : undefined;
|
|
4863
|
+
}
|
|
4864
|
+
function stringField(value, key) {
|
|
4865
|
+
const field = value?.[key];
|
|
4866
|
+
return typeof field === "string" ? field : undefined;
|
|
4867
|
+
}
|
|
4868
|
+
function numberField(value, key) {
|
|
4869
|
+
const field = value?.[key];
|
|
4870
|
+
return typeof field === "number" && Number.isFinite(field) ? field : undefined;
|
|
4871
|
+
}
|
|
4872
|
+
function codewithAgentStatus(readJson) {
|
|
4873
|
+
return stringField(recordField(readJson, "agent"), "status") ?? stringField(recordField(readJson, "statusSnapshot"), "status");
|
|
4874
|
+
}
|
|
4875
|
+
function codewithAgentEvidence(startJson, readJson, logsJson) {
|
|
4876
|
+
const agent = recordField(readJson, "agent") ?? recordField(startJson, "agent");
|
|
4877
|
+
const statusSnapshot = recordField(readJson, "statusSnapshot");
|
|
4878
|
+
const events = Array.isArray(logsJson?.data) ? logsJson.data.filter((event) => Boolean(event) && typeof event === "object" && !Array.isArray(event)).map((event) => ({
|
|
4879
|
+
seq: numberField(event, "seq"),
|
|
4880
|
+
eventType: stringField(event, "eventType"),
|
|
4881
|
+
createdAt: numberField(event, "createdAt")
|
|
4882
|
+
})) : undefined;
|
|
4883
|
+
return JSON.stringify({
|
|
4884
|
+
codewithAgent: {
|
|
4885
|
+
agentId: stringField(agent, "agentId"),
|
|
4886
|
+
status: stringField(agent, "status"),
|
|
4887
|
+
desiredState: stringField(agent, "desiredState"),
|
|
4888
|
+
statusReason: stringField(agent, "statusReason"),
|
|
4889
|
+
threadId: stringField(agent, "threadId"),
|
|
4890
|
+
rolloutPath: stringField(agent, "rolloutPath"),
|
|
4891
|
+
pid: numberField(agent, "pid"),
|
|
4892
|
+
exitCode: numberField(agent, "exitCode"),
|
|
4893
|
+
created: typeof startJson.created === "boolean" ? startJson.created : undefined
|
|
4894
|
+
},
|
|
4895
|
+
statusSnapshot: statusSnapshot ? {
|
|
4896
|
+
seq: numberField(statusSnapshot, "seq"),
|
|
4897
|
+
status: stringField(statusSnapshot, "status"),
|
|
4898
|
+
summary: stringField(statusSnapshot, "summary"),
|
|
4899
|
+
pendingInteractionCount: numberField(statusSnapshot, "pendingInteractionCount"),
|
|
4900
|
+
lastEventSeq: numberField(statusSnapshot, "lastEventSeq")
|
|
4901
|
+
} : undefined,
|
|
4902
|
+
events
|
|
4903
|
+
}, null, 2);
|
|
4904
|
+
}
|
|
4905
|
+
function sleep(ms) {
|
|
4906
|
+
return new Promise((resolve3) => setTimeout(resolve3, ms));
|
|
4907
|
+
}
|
|
4908
|
+
async function executeCodewithDurableAgent(spec, metadata, opts, env, startedAt) {
|
|
4909
|
+
const target = spec.codewithDurableAgent?.target;
|
|
4910
|
+
if (!target)
|
|
4911
|
+
throw new Error("internal error: missing codewith durable target");
|
|
4912
|
+
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
4913
|
+
const idempotencyKey = codewithAgentIdempotencyKey(metadata);
|
|
4914
|
+
const start = await spawnCapture(spec.command, codewithAgentStartArgs(target, idempotencyKey), {
|
|
4915
|
+
cwd: spec.cwd,
|
|
4916
|
+
env,
|
|
4917
|
+
timeoutMs: 30000,
|
|
4918
|
+
maxOutputBytes
|
|
4919
|
+
});
|
|
4920
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
4921
|
+
stderr.append(start.stderr);
|
|
4922
|
+
if (start.error || (start.status ?? 1) !== 0) {
|
|
4923
|
+
return failureResult(startedAt, start.error ?? `codewith agent start exited with code ${start.status ?? "unknown"}`, {
|
|
4924
|
+
exitCode: start.status ?? undefined,
|
|
4925
|
+
stdout: start.stdout,
|
|
4926
|
+
stderr: stderr.value()
|
|
4927
|
+
});
|
|
3879
4928
|
}
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
4929
|
+
let startJson;
|
|
4930
|
+
try {
|
|
4931
|
+
startJson = parseJsonOutput(start.stdout || "{}", "codewith agent start");
|
|
4932
|
+
} catch (error) {
|
|
4933
|
+
return failureResult(startedAt, error instanceof Error ? error.message : String(error), { stdout: start.stdout, stderr: stderr.value() });
|
|
4934
|
+
}
|
|
4935
|
+
const agentId = stringField(recordField(startJson, "agent"), "agentId");
|
|
4936
|
+
if (!agentId) {
|
|
4937
|
+
return failureResult(startedAt, "codewith agent start did not return agent.agentId", { stdout: start.stdout, stderr: stderr.value() });
|
|
4938
|
+
}
|
|
4939
|
+
const stopAgent = async () => {
|
|
4940
|
+
await spawnCapture(spec.command, codewithAgentControlArgs(target, "stop", agentId), {
|
|
4941
|
+
cwd: spec.cwd,
|
|
4942
|
+
env,
|
|
4943
|
+
timeoutMs: 15000,
|
|
4944
|
+
maxOutputBytes
|
|
4945
|
+
});
|
|
4946
|
+
};
|
|
4947
|
+
const pollMs = Math.max(100, Number(opts.env?.LOOPS_CODEWITH_AGENT_POLL_MS ?? process.env.LOOPS_CODEWITH_AGENT_POLL_MS ?? 2000) || 2000);
|
|
4948
|
+
let lastReadJson = startJson;
|
|
4949
|
+
let lastLogsJson;
|
|
4950
|
+
let lastFingerprint;
|
|
4951
|
+
let lastProgressAt = Date.now();
|
|
4952
|
+
const evidence = () => boundedText(codewithAgentEvidence(startJson, lastReadJson, lastLogsJson), maxOutputBytes);
|
|
4953
|
+
while (true) {
|
|
4954
|
+
if (opts.signal?.aborted) {
|
|
4955
|
+
await stopAgent();
|
|
4956
|
+
return failureResult(startedAt, "cancelled", { stdout: evidence(), stderr: stderr.value() });
|
|
4957
|
+
}
|
|
4958
|
+
const read = await spawnCapture(spec.command, codewithAgentControlArgs(target, "read", agentId), {
|
|
4959
|
+
cwd: spec.cwd,
|
|
4960
|
+
env,
|
|
4961
|
+
timeoutMs: 30000,
|
|
4962
|
+
maxOutputBytes
|
|
4963
|
+
});
|
|
4964
|
+
stderr.append(read.stderr);
|
|
4965
|
+
if (read.error || (read.status ?? 1) !== 0) {
|
|
4966
|
+
return failureResult(startedAt, read.error ?? `codewith agent read exited with code ${read.status ?? "unknown"}`, {
|
|
4967
|
+
exitCode: read.status ?? undefined,
|
|
4968
|
+
stdout: evidence(),
|
|
4969
|
+
stderr: stderr.value()
|
|
4970
|
+
});
|
|
4971
|
+
}
|
|
4972
|
+
try {
|
|
4973
|
+
lastReadJson = parseJsonOutput(read.stdout || "{}", "codewith agent read");
|
|
4974
|
+
} catch (error) {
|
|
4975
|
+
return failureResult(startedAt, error instanceof Error ? error.message : String(error), {
|
|
4976
|
+
stdout: boundedText(read.stdout, maxOutputBytes),
|
|
4977
|
+
stderr: stderr.value()
|
|
4978
|
+
});
|
|
4979
|
+
}
|
|
4980
|
+
const status = codewithAgentStatus(lastReadJson);
|
|
4981
|
+
const fingerprint = JSON.stringify({ status, snapshot: recordField(lastReadJson, "statusSnapshot") ?? null });
|
|
4982
|
+
if (fingerprint !== lastFingerprint) {
|
|
4983
|
+
lastFingerprint = fingerprint;
|
|
4984
|
+
lastProgressAt = Date.now();
|
|
4985
|
+
}
|
|
4986
|
+
if (status === "completed" || status === "failed" || status === "cancelled") {
|
|
4987
|
+
const logs = await spawnCapture(spec.command, codewithAgentControlArgs(target, "logs", agentId), {
|
|
4988
|
+
cwd: spec.cwd,
|
|
4989
|
+
env,
|
|
4990
|
+
timeoutMs: 30000,
|
|
4991
|
+
maxOutputBytes
|
|
4992
|
+
});
|
|
4993
|
+
if (!logs.error && (logs.status ?? 1) === 0) {
|
|
4994
|
+
try {
|
|
4995
|
+
lastLogsJson = parseJsonOutput(logs.stdout || "{}", "codewith agent logs");
|
|
4996
|
+
} catch {
|
|
4997
|
+
lastLogsJson = undefined;
|
|
4998
|
+
}
|
|
4999
|
+
} else {
|
|
5000
|
+
stderr.append(logs.stderr || logs.error || "");
|
|
5001
|
+
}
|
|
5002
|
+
if (status === "completed") {
|
|
5003
|
+
return successResult(startedAt, { exitCode: 0, stdout: evidence(), stderr: stderr.value() });
|
|
5004
|
+
}
|
|
5005
|
+
return failureResult(startedAt, stringField(recordField(lastReadJson, "agent"), "statusReason") ?? `codewith agent ${status}`, { exitCode: 1, stdout: evidence(), stderr: stderr.value() });
|
|
5006
|
+
}
|
|
5007
|
+
if (typeof spec.timeoutMs === "number" && new Date(nowIso()).getTime() - new Date(startedAt).getTime() >= spec.timeoutMs) {
|
|
5008
|
+
await stopAgent();
|
|
5009
|
+
return timeoutResult(startedAt, `timed out after ${spec.timeoutMs}ms`, { stdout: evidence(), stderr: stderr.value() });
|
|
5010
|
+
}
|
|
5011
|
+
if (typeof spec.idleTimeoutMs === "number" && Date.now() - lastProgressAt >= spec.idleTimeoutMs) {
|
|
5012
|
+
await stopAgent();
|
|
5013
|
+
return timeoutResult(startedAt, `idle timed out after ${spec.idleTimeoutMs}ms without agent progress`, {
|
|
5014
|
+
stdout: evidence(),
|
|
5015
|
+
stderr: stderr.value()
|
|
5016
|
+
});
|
|
5017
|
+
}
|
|
5018
|
+
await sleep(pollMs);
|
|
5019
|
+
}
|
|
5020
|
+
}
|
|
5021
|
+
function resolvedDirEquals(left, right) {
|
|
5022
|
+
try {
|
|
5023
|
+
return realpathSync(left) === realpathSync(right);
|
|
5024
|
+
} catch {
|
|
5025
|
+
return false;
|
|
5026
|
+
}
|
|
5027
|
+
}
|
|
5028
|
+
async function ensureLocalWorktree(worktree, env) {
|
|
5029
|
+
const { repoRoot, path, branch } = worktree;
|
|
5030
|
+
if (!repoRoot || !path || !branch) {
|
|
5031
|
+
return { error: "worktree preparation requires repoRoot, path, and branch metadata" };
|
|
5032
|
+
}
|
|
5033
|
+
const git = (args) => spawnCapture("git", args, { env, timeoutMs: WORKTREE_GIT_TIMEOUT_MS });
|
|
5034
|
+
let stats;
|
|
5035
|
+
try {
|
|
5036
|
+
stats = lstatSync(path);
|
|
5037
|
+
} catch {
|
|
5038
|
+
stats = undefined;
|
|
5039
|
+
}
|
|
5040
|
+
if (stats?.isSymbolicLink())
|
|
5041
|
+
return { error: `refusing symlinked worktree path ${path}` };
|
|
5042
|
+
const commonDir = async (base) => {
|
|
5043
|
+
const result = await git(["-C", base, "rev-parse", "--git-common-dir"]);
|
|
5044
|
+
if (result.error || (result.status ?? 1) !== 0)
|
|
5045
|
+
return;
|
|
5046
|
+
const raw = result.stdout.trim();
|
|
5047
|
+
if (!raw)
|
|
5048
|
+
return;
|
|
5049
|
+
try {
|
|
5050
|
+
return realpathSync(resolve2(base, raw));
|
|
5051
|
+
} catch {
|
|
5052
|
+
return resolve2(base, raw);
|
|
5053
|
+
}
|
|
5054
|
+
};
|
|
5055
|
+
if (stats) {
|
|
5056
|
+
const top = await git(["-C", path, "rev-parse", "--show-toplevel"]);
|
|
5057
|
+
if (top.error || (top.status ?? 1) !== 0) {
|
|
5058
|
+
return { error: `refusing to reuse non-worktree path: ${path}` };
|
|
5059
|
+
}
|
|
5060
|
+
if (!resolvedDirEquals(top.stdout.trim(), path)) {
|
|
5061
|
+
return { error: `existing worktree top-level mismatch for ${path}: ${top.stdout.trim()}` };
|
|
5062
|
+
}
|
|
5063
|
+
const expectedCommon = await commonDir(repoRoot);
|
|
5064
|
+
const actualCommon = await commonDir(path);
|
|
5065
|
+
if (!expectedCommon || expectedCommon !== actualCommon) {
|
|
5066
|
+
return { error: `existing worktree ${path} belongs to a different git common dir` };
|
|
5067
|
+
}
|
|
5068
|
+
const current = await git(["-C", path, "branch", "--show-current"]);
|
|
5069
|
+
const actualBranch = current.stdout.trim();
|
|
5070
|
+
if (current.error || (current.status ?? 1) !== 0 || actualBranch !== branch) {
|
|
5071
|
+
return { error: `existing worktree ${path} is on branch ${actualBranch || "unknown"}, expected ${branch}` };
|
|
5072
|
+
}
|
|
5073
|
+
return { cwd: worktree.cwd };
|
|
5074
|
+
}
|
|
5075
|
+
const inside = await git(["-C", repoRoot, "rev-parse", "--is-inside-work-tree"]);
|
|
5076
|
+
if (inside.error || (inside.status ?? 1) !== 0) {
|
|
5077
|
+
return { error: `worktree repoRoot is not a git repository: ${repoRoot}` };
|
|
5078
|
+
}
|
|
5079
|
+
try {
|
|
5080
|
+
mkdirSync5(dirname4(path), { recursive: true });
|
|
5081
|
+
} catch (error) {
|
|
5082
|
+
return { error: `could not create worktree parent directory: ${error instanceof Error ? error.message : String(error)}` };
|
|
5083
|
+
}
|
|
5084
|
+
const hasBranch = await git(["-C", repoRoot, "show-ref", "--verify", "--quiet", `refs/heads/${branch}`]);
|
|
5085
|
+
const add = hasBranch.status === 0 ? await git(["-C", repoRoot, "worktree", "add", path, branch]) : await git(["-C", repoRoot, "worktree", "add", "-b", branch, path, "HEAD"]);
|
|
5086
|
+
if (add.error || (add.status ?? 1) !== 0) {
|
|
5087
|
+
const detail = (add.stderr || add.stdout || add.error || "").toString().trim();
|
|
5088
|
+
return { error: `git worktree add failed for ${path}${detail ? `: ${detail}` : ""}` };
|
|
5089
|
+
}
|
|
5090
|
+
return { cwd: worktree.cwd };
|
|
5091
|
+
}
|
|
5092
|
+
async function enterWorktree(spec, opts, env, startedAt) {
|
|
5093
|
+
const worktree = spec.worktree;
|
|
5094
|
+
if (!worktree?.enabled || worktree.mode === "off" || worktree.mode === "main")
|
|
5095
|
+
return;
|
|
5096
|
+
const prepared = await ensureLocalWorktree(worktree, env);
|
|
5097
|
+
if (prepared.error) {
|
|
5098
|
+
if (worktree.mode === "required") {
|
|
5099
|
+
return { failure: failureResult(startedAt, `worktree preparation failed (mode=required): ${prepared.error}`) };
|
|
5100
|
+
}
|
|
5101
|
+
opts.log?.(`worktree preparation failed (mode=${worktree.mode}); falling back to ${worktree.originalCwd}: ${prepared.error}`);
|
|
5102
|
+
spec.cwd = worktree.originalCwd;
|
|
5103
|
+
return { fallbackCwd: worktree.originalCwd };
|
|
3883
5104
|
}
|
|
5105
|
+
spec.cwd = prepared.cwd ?? spec.cwd;
|
|
5106
|
+
opts.log?.(`entered worktree ${worktree.path ?? spec.cwd}${worktree.branch ? ` branch ${worktree.branch}` : ""}`);
|
|
5107
|
+
return;
|
|
5108
|
+
}
|
|
5109
|
+
function worktreeFallbackSpec(target, opts, fallbackCwd) {
|
|
5110
|
+
if (target.type !== "agent")
|
|
5111
|
+
return;
|
|
5112
|
+
const agentTarget = target;
|
|
5113
|
+
const fallbackTarget = {
|
|
5114
|
+
...agentTarget,
|
|
5115
|
+
cwd: fallbackCwd,
|
|
5116
|
+
worktree: agentTarget.worktree ? { ...agentTarget.worktree, enabled: false, cwd: fallbackCwd, reason: "auto worktree preparation failed" } : undefined
|
|
5117
|
+
};
|
|
5118
|
+
return commandSpec(fallbackTarget, opts);
|
|
3884
5119
|
}
|
|
3885
5120
|
function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
3886
5121
|
const plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
3887
|
-
const result =
|
|
5122
|
+
const result = spawnSync4(plan.command, plan.args, {
|
|
3888
5123
|
encoding: "utf8",
|
|
3889
5124
|
env: transportEnv(opts),
|
|
3890
5125
|
input: remotePreflightScript(spec, metadata),
|
|
@@ -3898,11 +5133,11 @@ function preflightRemoteSpec(spec, machine, metadata, opts) {
|
|
|
3898
5133
|
throw new Error(`remote preflight failed on ${machine.id}${detail ? `: ${detail}` : ""}`);
|
|
3899
5134
|
}
|
|
3900
5135
|
}
|
|
3901
|
-
async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
5136
|
+
async function executeRemoteSpec(spec, machine, metadata, opts, fallbackSpec) {
|
|
3902
5137
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
3903
5138
|
const startedAt = nowIso();
|
|
3904
|
-
|
|
3905
|
-
|
|
5139
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
5140
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
3906
5141
|
let timedOut = false;
|
|
3907
5142
|
let idleTimedOut = false;
|
|
3908
5143
|
let exitCode;
|
|
@@ -3911,25 +5146,16 @@ async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
|
3911
5146
|
let script;
|
|
3912
5147
|
try {
|
|
3913
5148
|
plan = (opts.machineCommandResolver ?? resolveMachineCommand)(machine.id, "bash -s");
|
|
3914
|
-
script = remoteScript(spec, metadata);
|
|
5149
|
+
script = remoteScript(spec, metadata, fallbackSpec);
|
|
3915
5150
|
} catch (err) {
|
|
3916
|
-
return
|
|
3917
|
-
status: "failed",
|
|
3918
|
-
stdout: "",
|
|
3919
|
-
stderr: "",
|
|
3920
|
-
error: err instanceof Error ? err.message : String(err),
|
|
3921
|
-
startedAt,
|
|
3922
|
-
finishedAt: nowIso(),
|
|
3923
|
-
durationMs: 0
|
|
3924
|
-
};
|
|
5151
|
+
return failureResult(startedAt, err instanceof Error ? err.message : String(err));
|
|
3925
5152
|
}
|
|
3926
|
-
const child =
|
|
5153
|
+
const child = spawn2(plan.command, plan.args, {
|
|
3927
5154
|
env: transportEnv(opts),
|
|
3928
5155
|
detached: true,
|
|
3929
5156
|
stdio: ["pipe", "pipe", "pipe"]
|
|
3930
5157
|
});
|
|
3931
|
-
|
|
3932
|
-
opts.onSpawn?.(child.pid);
|
|
5158
|
+
notifySpawn(child.pid, opts);
|
|
3933
5159
|
child.stdin?.on("error", (err) => {
|
|
3934
5160
|
if (err.code !== "EPIPE")
|
|
3935
5161
|
error = err.message;
|
|
@@ -3963,12 +5189,14 @@ async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
|
3963
5189
|
idleTimer.unref();
|
|
3964
5190
|
};
|
|
3965
5191
|
resetIdleTimer();
|
|
5192
|
+
child.stdout?.setEncoding("utf8");
|
|
5193
|
+
child.stderr?.setEncoding("utf8");
|
|
3966
5194
|
child.stdout?.on("data", (chunk) => {
|
|
3967
|
-
stdout
|
|
5195
|
+
stdout.append(chunk);
|
|
3968
5196
|
resetIdleTimer();
|
|
3969
5197
|
});
|
|
3970
5198
|
child.stderr?.on("data", (chunk) => {
|
|
3971
|
-
stderr
|
|
5199
|
+
stderr.append(chunk);
|
|
3972
5200
|
resetIdleTimer();
|
|
3973
5201
|
});
|
|
3974
5202
|
try {
|
|
@@ -3986,47 +5214,17 @@ async function executeRemoteSpec(spec, machine, metadata, opts) {
|
|
|
3986
5214
|
clearTimeout(idleTimer);
|
|
3987
5215
|
opts.signal?.removeEventListener("abort", abortHandler);
|
|
3988
5216
|
}
|
|
3989
|
-
const
|
|
3990
|
-
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
5217
|
+
const fields = { exitCode, stdout: stdout.value(), stderr: stderr.value(), pid: child.pid };
|
|
3991
5218
|
if (timedOut || idleTimedOut) {
|
|
3992
|
-
return {
|
|
3993
|
-
status: "timed_out",
|
|
3994
|
-
exitCode,
|
|
3995
|
-
stdout,
|
|
3996
|
-
stderr,
|
|
3997
|
-
error: idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`,
|
|
3998
|
-
pid: child.pid,
|
|
3999
|
-
startedAt,
|
|
4000
|
-
finishedAt,
|
|
4001
|
-
durationMs
|
|
4002
|
-
};
|
|
5219
|
+
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
4003
5220
|
}
|
|
4004
5221
|
if (error || exitCode !== 0) {
|
|
4005
|
-
return {
|
|
4006
|
-
status: "failed",
|
|
4007
|
-
exitCode,
|
|
4008
|
-
stdout,
|
|
4009
|
-
stderr,
|
|
4010
|
-
error: error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`,
|
|
4011
|
-
pid: child.pid,
|
|
4012
|
-
startedAt,
|
|
4013
|
-
finishedAt,
|
|
4014
|
-
durationMs
|
|
4015
|
-
};
|
|
5222
|
+
return failureResult(startedAt, error ?? `remote process on ${machine.id} exited with code ${exitCode ?? "unknown"}`, fields);
|
|
4016
5223
|
}
|
|
4017
|
-
return
|
|
4018
|
-
status: "succeeded",
|
|
4019
|
-
exitCode,
|
|
4020
|
-
stdout,
|
|
4021
|
-
stderr,
|
|
4022
|
-
pid: child.pid,
|
|
4023
|
-
startedAt,
|
|
4024
|
-
finishedAt,
|
|
4025
|
-
durationMs
|
|
4026
|
-
};
|
|
5224
|
+
return successResult(startedAt, fields);
|
|
4027
5225
|
}
|
|
4028
5226
|
function preflightTarget(target, metadata = {}, opts = {}) {
|
|
4029
|
-
const spec = commandSpec(target);
|
|
5227
|
+
const spec = commandSpec(target, opts);
|
|
4030
5228
|
const machine = resolvedMachine(opts);
|
|
4031
5229
|
if (machine && !machine.local) {
|
|
4032
5230
|
preflightRemoteSpec(spec, machine, metadata, opts);
|
|
@@ -4036,14 +5234,14 @@ function preflightTarget(target, metadata = {}, opts = {}) {
|
|
|
4036
5234
|
accountTool: spec.accountTool
|
|
4037
5235
|
};
|
|
4038
5236
|
}
|
|
4039
|
-
const env =
|
|
5237
|
+
const env = executionEnvSync(spec, metadata, opts);
|
|
4040
5238
|
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
4041
5239
|
throw new Error(commandNotFoundMessage(spec.command, env));
|
|
4042
5240
|
}
|
|
4043
5241
|
if (spec.preflightAnyOf?.length && !spec.preflightAnyOf.some((command) => executableExists(command, env))) {
|
|
4044
5242
|
throw new Error(`none of required executables found: ${spec.preflightAnyOf.join(", ")}`);
|
|
4045
5243
|
}
|
|
4046
|
-
|
|
5244
|
+
preflightNativeAuthProfileSync(spec, env);
|
|
4047
5245
|
return {
|
|
4048
5246
|
command: spec.command,
|
|
4049
5247
|
accountProfile: spec.account?.profile,
|
|
@@ -4051,63 +5249,52 @@ function preflightTarget(target, metadata = {}, opts = {}) {
|
|
|
4051
5249
|
};
|
|
4052
5250
|
}
|
|
4053
5251
|
async function executeTarget(target, metadata = {}, opts = {}) {
|
|
4054
|
-
|
|
5252
|
+
let spec = commandSpec(target, opts);
|
|
4055
5253
|
const machine = resolvedMachine(opts);
|
|
4056
|
-
if (machine && !machine.local)
|
|
4057
|
-
return
|
|
5254
|
+
if (machine && !machine.local && spec.codewithDurableAgent) {
|
|
5255
|
+
return failureResult(nowIso(), "remote Codewith durable background-agent steps require remote status polling support; run this Codewith step locally or add remote durable readback before dispatch");
|
|
5256
|
+
}
|
|
5257
|
+
if (machine && !machine.local) {
|
|
5258
|
+
const remoteFallbackSpec = spec.worktree?.enabled && spec.worktree.mode === "auto" ? worktreeFallbackSpec(target, opts, spec.worktree.originalCwd) : undefined;
|
|
5259
|
+
return executeRemoteSpec(spec, machine, metadata, opts, remoteFallbackSpec);
|
|
5260
|
+
}
|
|
4058
5261
|
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
|
|
4059
5262
|
const startedAt = nowIso();
|
|
4060
|
-
|
|
4061
|
-
|
|
5263
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
5264
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
4062
5265
|
let timedOut = false;
|
|
4063
5266
|
let idleTimedOut = false;
|
|
4064
5267
|
let exitCode;
|
|
4065
5268
|
let error;
|
|
4066
|
-
const env = executionEnv(spec, metadata, opts);
|
|
5269
|
+
const env = await executionEnv(spec, metadata, opts);
|
|
4067
5270
|
if (!spec.shell && !executableExists(spec.command, env)) {
|
|
4068
|
-
return
|
|
4069
|
-
status: "failed",
|
|
4070
|
-
stdout: "",
|
|
4071
|
-
stderr: "",
|
|
4072
|
-
error: commandNotFoundMessage(spec.command, env),
|
|
4073
|
-
startedAt,
|
|
4074
|
-
finishedAt: nowIso(),
|
|
4075
|
-
durationMs: 0
|
|
4076
|
-
};
|
|
5271
|
+
return failureResult(startedAt, commandNotFoundMessage(spec.command, env));
|
|
4077
5272
|
}
|
|
4078
5273
|
if (spec.preflightAnyOf?.length && !spec.preflightAnyOf.some((command) => executableExists(command, env))) {
|
|
4079
|
-
return {
|
|
4080
|
-
status: "failed",
|
|
4081
|
-
stdout,
|
|
4082
|
-
stderr,
|
|
4083
|
-
error: `none of required executables found: ${spec.preflightAnyOf.join(", ")}`,
|
|
4084
|
-
startedAt,
|
|
4085
|
-
finishedAt: nowIso(),
|
|
4086
|
-
durationMs: 0
|
|
4087
|
-
};
|
|
5274
|
+
return failureResult(startedAt, `none of required executables found: ${spec.preflightAnyOf.join(", ")}`);
|
|
4088
5275
|
}
|
|
4089
5276
|
try {
|
|
4090
|
-
preflightNativeAuthProfile(spec, env);
|
|
5277
|
+
await preflightNativeAuthProfile(spec, env);
|
|
4091
5278
|
} catch (err) {
|
|
4092
|
-
return
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
durationMs: 0
|
|
4100
|
-
};
|
|
5279
|
+
return failureResult(startedAt, err instanceof Error ? err.message : String(err));
|
|
5280
|
+
}
|
|
5281
|
+
const worktreeEntry = await enterWorktree(spec, opts, env, startedAt);
|
|
5282
|
+
if (worktreeEntry?.failure)
|
|
5283
|
+
return worktreeEntry.failure;
|
|
5284
|
+
if (worktreeEntry?.fallbackCwd) {
|
|
5285
|
+
spec = worktreeFallbackSpec(target, opts, worktreeEntry.fallbackCwd) ?? spec;
|
|
4101
5286
|
}
|
|
4102
|
-
|
|
5287
|
+
if (spec.codewithDurableAgent) {
|
|
5288
|
+
return executeCodewithDurableAgent(spec, metadata, opts, env, startedAt);
|
|
5289
|
+
}
|
|
5290
|
+
const child = spawn2(spec.command, spec.args, {
|
|
4103
5291
|
cwd: spec.cwd,
|
|
4104
5292
|
env,
|
|
4105
5293
|
shell: spec.shell ?? false,
|
|
4106
5294
|
detached: true,
|
|
4107
5295
|
stdio: spec.stdin === undefined ? ["ignore", "pipe", "pipe"] : ["pipe", "pipe", "pipe"]
|
|
4108
5296
|
});
|
|
4109
|
-
|
|
4110
|
-
opts.onSpawn?.(child.pid);
|
|
5297
|
+
notifySpawn(child.pid, opts);
|
|
4111
5298
|
if (spec.stdin !== undefined && child.stdin) {
|
|
4112
5299
|
child.stdin.on("error", (err) => {
|
|
4113
5300
|
if (err.code !== "EPIPE")
|
|
@@ -4143,12 +5330,14 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
4143
5330
|
idleTimer.unref();
|
|
4144
5331
|
};
|
|
4145
5332
|
resetIdleTimer();
|
|
5333
|
+
child.stdout?.setEncoding("utf8");
|
|
5334
|
+
child.stderr?.setEncoding("utf8");
|
|
4146
5335
|
child.stdout?.on("data", (chunk) => {
|
|
4147
|
-
stdout
|
|
5336
|
+
stdout.append(chunk);
|
|
4148
5337
|
resetIdleTimer();
|
|
4149
5338
|
});
|
|
4150
5339
|
child.stderr?.on("data", (chunk) => {
|
|
4151
|
-
stderr
|
|
5340
|
+
stderr.append(chunk);
|
|
4152
5341
|
resetIdleTimer();
|
|
4153
5342
|
});
|
|
4154
5343
|
try {
|
|
@@ -4166,97 +5355,706 @@ async function executeTarget(target, metadata = {}, opts = {}) {
|
|
|
4166
5355
|
clearTimeout(idleTimer);
|
|
4167
5356
|
opts.signal?.removeEventListener("abort", abortHandler);
|
|
4168
5357
|
}
|
|
4169
|
-
const
|
|
4170
|
-
const durationMs = new Date(finishedAt).getTime() - new Date(startedAt).getTime();
|
|
5358
|
+
const fields = { exitCode, stdout: stdout.value(), stderr: stderr.value(), pid: child.pid };
|
|
4171
5359
|
if (timedOut || idleTimedOut) {
|
|
5360
|
+
return timeoutResult(startedAt, idleTimedOut ? `idle timed out after ${spec.idleTimeoutMs}ms without stdout/stderr` : `timed out after ${spec.timeoutMs}ms`, fields);
|
|
5361
|
+
}
|
|
5362
|
+
if (error || exitCode !== 0) {
|
|
5363
|
+
return failureResult(startedAt, error ?? `process exited with code ${exitCode ?? "unknown"}`, fields);
|
|
5364
|
+
}
|
|
5365
|
+
return successResult(startedAt, fields);
|
|
5366
|
+
}
|
|
5367
|
+
async function executeLoop(loop, run, opts = {}) {
|
|
5368
|
+
if (loop.target.type === "workflow") {
|
|
5369
|
+
throw new Error("workflow loop targets must be executed with executeLoopTarget");
|
|
5370
|
+
}
|
|
5371
|
+
if (loop.target.preflight?.beforeRun) {
|
|
5372
|
+
const startedAt = nowIso();
|
|
5373
|
+
try {
|
|
5374
|
+
preflightTarget(loop.target, {
|
|
5375
|
+
loopId: loop.id,
|
|
5376
|
+
loopName: loop.name,
|
|
5377
|
+
runId: run.id,
|
|
5378
|
+
scheduledFor: run.scheduledFor
|
|
5379
|
+
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5380
|
+
} catch (error) {
|
|
5381
|
+
return failureResult(startedAt, `runtime preflight failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
5382
|
+
}
|
|
5383
|
+
}
|
|
5384
|
+
return executeTarget(loop.target, {
|
|
5385
|
+
loopId: loop.id,
|
|
5386
|
+
loopName: loop.name,
|
|
5387
|
+
runId: run.id,
|
|
5388
|
+
scheduledFor: run.scheduledFor
|
|
5389
|
+
}, { ...opts, machine: opts.machine ?? loop.machine });
|
|
5390
|
+
}
|
|
5391
|
+
|
|
5392
|
+
// src/lib/doctor.ts
|
|
5393
|
+
var PROVIDER_COMMANDS = [
|
|
5394
|
+
"claude",
|
|
5395
|
+
"agent",
|
|
5396
|
+
"codewith",
|
|
5397
|
+
"aicopilot",
|
|
5398
|
+
"opencode",
|
|
5399
|
+
"codex"
|
|
5400
|
+
];
|
|
5401
|
+
function hasCommand(command) {
|
|
5402
|
+
const result = spawnSync5("sh", ["-c", 'command -v "$1" >/dev/null', "sh", command], { stdio: "ignore" });
|
|
5403
|
+
return (result.status ?? 1) === 0;
|
|
5404
|
+
}
|
|
5405
|
+
function commandVersion(command) {
|
|
5406
|
+
const result = spawnSync5(command, ["--version"], {
|
|
5407
|
+
encoding: "utf8",
|
|
5408
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
5409
|
+
});
|
|
5410
|
+
if ((result.status ?? 1) !== 0)
|
|
5411
|
+
return;
|
|
5412
|
+
return (result.stdout || result.stderr).trim().split(/\r?\n/)[0];
|
|
5413
|
+
}
|
|
5414
|
+
function runDoctor(store) {
|
|
5415
|
+
const checks = [];
|
|
5416
|
+
try {
|
|
5417
|
+
const dir = ensureDataDir();
|
|
5418
|
+
accessSync2(dir, constants2.R_OK | constants2.W_OK);
|
|
5419
|
+
checks.push({ id: "data-dir", status: "ok", message: "data directory is writable", detail: dir });
|
|
5420
|
+
} catch (error) {
|
|
5421
|
+
checks.push({
|
|
5422
|
+
id: "data-dir",
|
|
5423
|
+
status: "fail",
|
|
5424
|
+
message: "data directory is not writable",
|
|
5425
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
5426
|
+
});
|
|
5427
|
+
}
|
|
5428
|
+
const bunVersion = commandVersion("bun");
|
|
5429
|
+
checks.push(bunVersion ? { id: "bun", status: "ok", message: "bun is available", detail: bunVersion } : { id: "bun", status: "fail", message: "bun is not available on PATH" });
|
|
5430
|
+
const accountsVersion = commandVersion("accounts");
|
|
5431
|
+
checks.push(accountsVersion ? { id: "accounts", status: "ok", message: "accounts is available", detail: accountsVersion } : { id: "accounts", status: "warn", message: "accounts CLI is not available; account-routed steps will fail" });
|
|
5432
|
+
try {
|
|
5433
|
+
const machines = listOpenMachines();
|
|
5434
|
+
const local = machines.find((machine) => machine.local);
|
|
5435
|
+
checks.push({
|
|
5436
|
+
id: "machines",
|
|
5437
|
+
status: "ok",
|
|
5438
|
+
message: `OpenMachines topology available (${machines.length} machine(s))`,
|
|
5439
|
+
detail: local ? `local=${local.id}` : undefined
|
|
5440
|
+
});
|
|
5441
|
+
} catch (error) {
|
|
5442
|
+
checks.push({
|
|
5443
|
+
id: "machines",
|
|
5444
|
+
status: "warn",
|
|
5445
|
+
message: "OpenMachines topology is not available; machine-assigned loops will fail",
|
|
5446
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
5447
|
+
});
|
|
5448
|
+
}
|
|
5449
|
+
for (const command of PROVIDER_COMMANDS) {
|
|
5450
|
+
checks.push(hasCommand(command) ? { id: `provider:${command}`, status: "ok", message: `${command} is available` } : { id: `provider:${command}`, status: "warn", message: `${command} is not on PATH` });
|
|
5451
|
+
}
|
|
5452
|
+
const status = daemonStatus(store);
|
|
5453
|
+
checks.push(status.running ? { id: "daemon", status: "ok", message: `daemon is running pid=${status.pid}` } : { id: "daemon", status: status.stale ? "warn" : "ok", message: status.stale ? "daemon pid file is stale" : "daemon is not running" });
|
|
5454
|
+
const failedRuns = store.countRuns("failed");
|
|
5455
|
+
checks.push(failedRuns === 0 ? { id: "loop-runs", status: "ok", message: "no failed loop runs recorded" } : { id: "loop-runs", status: "warn", message: `${failedRuns} failed loop run(s) recorded` });
|
|
5456
|
+
for (const loop of store.listLoops({ status: "active" })) {
|
|
5457
|
+
try {
|
|
5458
|
+
if (loop.target.type === "workflow") {
|
|
5459
|
+
const workflow = store.requireWorkflow(loop.target.workflowId);
|
|
5460
|
+
for (const step of workflowExecutionOrder(workflow)) {
|
|
5461
|
+
preflightTarget({
|
|
5462
|
+
...step.target,
|
|
5463
|
+
account: step.account ?? step.target.account,
|
|
5464
|
+
timeoutMs: step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs
|
|
5465
|
+
}, { loopId: loop.id, loopName: loop.name, workflowId: workflow.id, workflowName: workflow.name, workflowStepId: step.id }, { machine: loop.machine });
|
|
5466
|
+
}
|
|
5467
|
+
} else {
|
|
5468
|
+
preflightTarget(loop.target, { loopId: loop.id, loopName: loop.name }, { machine: loop.machine });
|
|
5469
|
+
}
|
|
5470
|
+
checks.push({ id: `loop:${loop.id}:preflight`, status: "ok", message: `active loop target is ready: ${loop.name}` });
|
|
5471
|
+
} catch (error) {
|
|
5472
|
+
checks.push({
|
|
5473
|
+
id: `loop:${loop.id}:preflight`,
|
|
5474
|
+
status: "fail",
|
|
5475
|
+
message: `active loop target preflight failed: ${loop.name}`,
|
|
5476
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
5477
|
+
});
|
|
5478
|
+
}
|
|
5479
|
+
}
|
|
5480
|
+
return {
|
|
5481
|
+
ok: checks.every((check) => check.status !== "fail"),
|
|
5482
|
+
checks
|
|
5483
|
+
};
|
|
5484
|
+
}
|
|
5485
|
+
|
|
5486
|
+
// src/lib/health.ts
|
|
5487
|
+
import { createHash as createHash2 } from "crypto";
|
|
5488
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4 } from "fs";
|
|
5489
|
+
|
|
5490
|
+
// src/lib/format.ts
|
|
5491
|
+
var TEXT_OUTPUT_LIMIT = 32 * 1024;
|
|
5492
|
+
var SENSITIVE_PAYLOAD_KEYS = new Set(["env", "error", "prompt", "reason", "stderr", "stdout"]);
|
|
5493
|
+
function redact(value, visible = 0) {
|
|
5494
|
+
if (!value)
|
|
5495
|
+
return value;
|
|
5496
|
+
if (value.length <= visible)
|
|
5497
|
+
return value;
|
|
5498
|
+
if (visible <= 0)
|
|
5499
|
+
return `[redacted ${value.length} chars]`;
|
|
5500
|
+
return `${value.slice(0, visible)}... [redacted ${value.length - visible} chars]`;
|
|
5501
|
+
}
|
|
5502
|
+
function truncateTextOutput(value) {
|
|
5503
|
+
if (value.length <= TEXT_OUTPUT_LIMIT)
|
|
5504
|
+
return value;
|
|
5505
|
+
return `${value.slice(0, TEXT_OUTPUT_LIMIT)}
|
|
5506
|
+
[truncated ${value.length - TEXT_OUTPUT_LIMIT} chars]`;
|
|
5507
|
+
}
|
|
5508
|
+
function redactSensitivePayload(value, key) {
|
|
5509
|
+
if (key && SENSITIVE_PAYLOAD_KEYS.has(key)) {
|
|
5510
|
+
if (typeof value === "string")
|
|
5511
|
+
return redact(value);
|
|
5512
|
+
if (value === undefined || value === null)
|
|
5513
|
+
return value;
|
|
5514
|
+
return "[redacted]";
|
|
5515
|
+
}
|
|
5516
|
+
if (Array.isArray(value))
|
|
5517
|
+
return value.map((item) => redactSensitivePayload(item));
|
|
5518
|
+
if (value && typeof value === "object") {
|
|
5519
|
+
return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [entryKey, redactSensitivePayload(entryValue, entryKey)]));
|
|
5520
|
+
}
|
|
5521
|
+
return value;
|
|
5522
|
+
}
|
|
5523
|
+
function textOutputBlocks(value, opts = {}) {
|
|
5524
|
+
const indent = opts.indent ?? "";
|
|
5525
|
+
const nested = `${indent} `;
|
|
5526
|
+
const blocks = [];
|
|
5527
|
+
for (const [label, output] of [
|
|
5528
|
+
["stdout", value.stdout],
|
|
5529
|
+
["stderr", value.stderr]
|
|
5530
|
+
]) {
|
|
5531
|
+
if (!output)
|
|
5532
|
+
continue;
|
|
5533
|
+
blocks.push(`${indent}${label}:`);
|
|
5534
|
+
for (const line of truncateTextOutput(output).replace(/\s+$/, "").split(/\r?\n/)) {
|
|
5535
|
+
blocks.push(`${nested}${line}`);
|
|
5536
|
+
}
|
|
5537
|
+
}
|
|
5538
|
+
return blocks;
|
|
5539
|
+
}
|
|
5540
|
+
function publicLoop(loop) {
|
|
5541
|
+
const target = loop.target.type === "command" ? { ...loop.target, env: loop.target.env ? "[redacted]" : undefined } : loop.target.type === "agent" ? { ...loop.target, prompt: redact(loop.target.prompt) } : loop.target;
|
|
5542
|
+
return {
|
|
5543
|
+
...loop,
|
|
5544
|
+
target
|
|
5545
|
+
};
|
|
5546
|
+
}
|
|
5547
|
+
function publicRun(run, showOutput = false) {
|
|
5548
|
+
return {
|
|
5549
|
+
...run,
|
|
5550
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
5551
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined
|
|
5552
|
+
};
|
|
5553
|
+
}
|
|
5554
|
+
function publicExecutorResult(result, showOutput = false) {
|
|
5555
|
+
return {
|
|
5556
|
+
...result,
|
|
5557
|
+
stdout: showOutput ? result.stdout : result.stdout ? `[redacted ${result.stdout.length} chars]` : undefined,
|
|
5558
|
+
stderr: showOutput ? result.stderr : result.stderr ? `[redacted ${result.stderr.length} chars]` : undefined,
|
|
5559
|
+
error: redact(result.error)
|
|
5560
|
+
};
|
|
5561
|
+
}
|
|
5562
|
+
function publicWorkflow(workflow) {
|
|
5563
|
+
return {
|
|
5564
|
+
...workflow,
|
|
5565
|
+
steps: workflow.steps.map((step) => ({
|
|
5566
|
+
...step,
|
|
5567
|
+
target: step.target.type === "agent" ? { ...step.target, prompt: redact(step.target.prompt) } : step.target.type === "command" && step.target.env ? { ...step.target, env: "[redacted]" } : step.target
|
|
5568
|
+
}))
|
|
5569
|
+
};
|
|
5570
|
+
}
|
|
5571
|
+
function publicWorkflowRun(run) {
|
|
5572
|
+
return { ...run, error: redact(run.error) };
|
|
5573
|
+
}
|
|
5574
|
+
function publicWorkflowInvocation(invocation) {
|
|
5575
|
+
return redactSensitivePayload(invocation);
|
|
5576
|
+
}
|
|
5577
|
+
function publicWorkflowWorkItem(item) {
|
|
5578
|
+
return { ...item, lastReason: redact(item.lastReason, 240) };
|
|
5579
|
+
}
|
|
5580
|
+
function publicWorkflowStepRun(run, showOutput = false) {
|
|
5581
|
+
return {
|
|
5582
|
+
...run,
|
|
5583
|
+
stdout: showOutput ? run.stdout : run.stdout ? `[redacted ${run.stdout.length} chars]` : undefined,
|
|
5584
|
+
stderr: showOutput ? run.stderr : run.stderr ? `[redacted ${run.stderr.length} chars]` : undefined,
|
|
5585
|
+
error: redact(run.error)
|
|
5586
|
+
};
|
|
5587
|
+
}
|
|
5588
|
+
function publicWorkflowEvent(event) {
|
|
5589
|
+
return { ...event, payload: redactSensitivePayload(event.payload) };
|
|
5590
|
+
}
|
|
5591
|
+
function publicGoal(goal) {
|
|
5592
|
+
return {
|
|
5593
|
+
...goal,
|
|
5594
|
+
objective: redact(goal.objective, 120)
|
|
5595
|
+
};
|
|
5596
|
+
}
|
|
5597
|
+
function publicGoalRun(run) {
|
|
5598
|
+
return {
|
|
5599
|
+
...run,
|
|
5600
|
+
evidence: redactSensitivePayload(run.evidence),
|
|
5601
|
+
rawResponse: redactSensitivePayload(run.rawResponse)
|
|
5602
|
+
};
|
|
5603
|
+
}
|
|
5604
|
+
|
|
5605
|
+
// src/lib/health.ts
|
|
5606
|
+
var EVIDENCE_CHARS = 2000;
|
|
5607
|
+
var FINGERPRINT_EVIDENCE_CHARS = 120;
|
|
5608
|
+
var CLASSIFICATIONS = [
|
|
5609
|
+
"rate_limit",
|
|
5610
|
+
"auth",
|
|
5611
|
+
"model_not_found",
|
|
5612
|
+
"context_length",
|
|
5613
|
+
"schema_response_format",
|
|
5614
|
+
"node_init",
|
|
5615
|
+
"preflight",
|
|
5616
|
+
"route_functional",
|
|
5617
|
+
"timeout",
|
|
5618
|
+
"sigsegv",
|
|
5619
|
+
"skipped_previous_active",
|
|
5620
|
+
"circuit_breaker",
|
|
5621
|
+
"unknown"
|
|
5622
|
+
];
|
|
5623
|
+
function bounded(value, limit = EVIDENCE_CHARS) {
|
|
5624
|
+
if (!value)
|
|
5625
|
+
return;
|
|
5626
|
+
if (value.length <= limit)
|
|
5627
|
+
return value;
|
|
5628
|
+
return `${value.slice(0, limit)}
|
|
5629
|
+
[truncated ${value.length - limit} chars]`;
|
|
5630
|
+
}
|
|
5631
|
+
function redactedEvidence(value) {
|
|
5632
|
+
return redact(bounded(value));
|
|
5633
|
+
}
|
|
5634
|
+
function searchableText(run) {
|
|
5635
|
+
return [run.error, run.stderr, run.stdout].filter(Boolean).join(`
|
|
5636
|
+
`).toLowerCase();
|
|
5637
|
+
}
|
|
5638
|
+
function isRecord(value) {
|
|
5639
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
5640
|
+
}
|
|
5641
|
+
function stringValue(value) {
|
|
5642
|
+
return typeof value === "string" && value.trim() ? value : undefined;
|
|
5643
|
+
}
|
|
5644
|
+
function objectField(value, key) {
|
|
5645
|
+
if (!value)
|
|
5646
|
+
return;
|
|
5647
|
+
const field = value[key];
|
|
5648
|
+
return isRecord(field) ? field : undefined;
|
|
5649
|
+
}
|
|
5650
|
+
function tagsFromValue(value) {
|
|
5651
|
+
if (Array.isArray(value))
|
|
5652
|
+
return value.map((entry) => String(entry).trim()).filter(Boolean);
|
|
5653
|
+
if (typeof value === "string")
|
|
5654
|
+
return value.split(",").map((entry) => entry.trim()).filter(Boolean);
|
|
5655
|
+
return [];
|
|
5656
|
+
}
|
|
5657
|
+
function stableFingerprint(parts) {
|
|
5658
|
+
return createHash2("sha256").update(parts.join(`
|
|
5659
|
+
`)).digest("hex").slice(0, 16);
|
|
5660
|
+
}
|
|
5661
|
+
function stableFailureFingerprint(run, classification) {
|
|
5662
|
+
return stableFingerprint([
|
|
5663
|
+
run.loopId,
|
|
5664
|
+
classification,
|
|
5665
|
+
String(run.status),
|
|
5666
|
+
String(run.exitCode ?? ""),
|
|
5667
|
+
(run.error ?? run.stderr ?? run.stdout ?? "").replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
5668
|
+
]);
|
|
5669
|
+
}
|
|
5670
|
+
function stableRouteFunctionalFingerprint(loop, reason) {
|
|
5671
|
+
return stableFingerprint([
|
|
5672
|
+
loop.id,
|
|
5673
|
+
"route_functional",
|
|
5674
|
+
reason.replace(/\d{4}-\d{2}-\d{2}T\S+/g, "<timestamp>").slice(0, FINGERPRINT_EVIDENCE_CHARS)
|
|
5675
|
+
]);
|
|
5676
|
+
}
|
|
5677
|
+
function healthRun(run) {
|
|
5678
|
+
return {
|
|
5679
|
+
...run,
|
|
5680
|
+
error: redactedEvidence(run.error),
|
|
5681
|
+
stdout: redactedEvidence(run.stdout),
|
|
5682
|
+
stderr: redactedEvidence(run.stderr)
|
|
5683
|
+
};
|
|
5684
|
+
}
|
|
5685
|
+
function classifyRunFailure(run) {
|
|
5686
|
+
if (run.status === "succeeded" || run.status === "running")
|
|
5687
|
+
return;
|
|
5688
|
+
const text = searchableText(run);
|
|
5689
|
+
let classification = "unknown";
|
|
5690
|
+
if (run.status === "timed_out")
|
|
5691
|
+
classification = "timeout";
|
|
5692
|
+
else if (run.status === "skipped" && /circuit breaker open/.test(text))
|
|
5693
|
+
classification = "circuit_breaker";
|
|
5694
|
+
else if (run.status === "skipped" && /previous run still active/.test(text))
|
|
5695
|
+
classification = "skipped_previous_active";
|
|
5696
|
+
else if (/runtime preflight failed|preflight failed|executable not found in path|none of required executables found|auth profile preflight failed|profile not found/.test(text))
|
|
5697
|
+
classification = "preflight";
|
|
5698
|
+
else if (/rate limit|too many requests|429\b|quota exceeded/.test(text))
|
|
5699
|
+
classification = "rate_limit";
|
|
5700
|
+
else if (/unauthorized|authentication|auth\b|api key|invalid token|permission denied|401\b|403\b/.test(text))
|
|
5701
|
+
classification = "auth";
|
|
5702
|
+
else if (/model .*not found|model_not_found|unknown model|invalid model|404.*model/.test(text))
|
|
5703
|
+
classification = "model_not_found";
|
|
5704
|
+
else if (/context length|context_length|context window|maximum context|token limit|too many tokens/.test(text))
|
|
5705
|
+
classification = "context_length";
|
|
5706
|
+
else if (/response_format|json schema|schema validation|invalid schema|structured output/.test(text))
|
|
5707
|
+
classification = "schema_response_format";
|
|
5708
|
+
else if (/cannot find module|module not found|node:internal|bun: command not found|node: command not found|npm err!|err_module_not_found/.test(text))
|
|
5709
|
+
classification = "node_init";
|
|
5710
|
+
else if (/sigsegv|segmentation fault|signal 11/.test(text))
|
|
5711
|
+
classification = "sigsegv";
|
|
5712
|
+
return {
|
|
5713
|
+
classification,
|
|
5714
|
+
fingerprint: stableFailureFingerprint(run, classification),
|
|
5715
|
+
evidence: {
|
|
5716
|
+
error: redactedEvidence(run.error),
|
|
5717
|
+
stdout: redactedEvidence(run.stdout),
|
|
5718
|
+
stderr: redactedEvidence(run.stderr),
|
|
5719
|
+
exitCode: run.exitCode
|
|
5720
|
+
}
|
|
5721
|
+
};
|
|
5722
|
+
}
|
|
5723
|
+
var ROUTE_FUNCTIONAL_DISALLOWED_TAGS = new Set([
|
|
5724
|
+
"no-auto",
|
|
5725
|
+
"manual",
|
|
5726
|
+
"manual-required",
|
|
5727
|
+
"approval-required",
|
|
5728
|
+
"blocked",
|
|
5729
|
+
"completed",
|
|
5730
|
+
"done",
|
|
5731
|
+
"cancelled",
|
|
5732
|
+
"canceled",
|
|
5733
|
+
"failed",
|
|
5734
|
+
"archived"
|
|
5735
|
+
]);
|
|
5736
|
+
var ROUTE_FUNCTIONAL_DISALLOWED_STATUSES = new Set([
|
|
5737
|
+
"blocked",
|
|
5738
|
+
"completed",
|
|
5739
|
+
"done",
|
|
5740
|
+
"cancelled",
|
|
5741
|
+
"canceled",
|
|
5742
|
+
"failed",
|
|
5743
|
+
"archived"
|
|
5744
|
+
]);
|
|
5745
|
+
function parseJsonObject(raw) {
|
|
5746
|
+
if (!raw?.trim())
|
|
5747
|
+
return;
|
|
5748
|
+
try {
|
|
5749
|
+
const parsed = JSON.parse(raw);
|
|
5750
|
+
return isRecord(parsed) ? parsed : undefined;
|
|
5751
|
+
} catch {
|
|
5752
|
+
return;
|
|
5753
|
+
}
|
|
5754
|
+
}
|
|
5755
|
+
function routeEvidenceReport(run) {
|
|
5756
|
+
const stdoutReport = parseJsonObject(run.stdout);
|
|
5757
|
+
const evidencePath = stringValue(stdoutReport?.evidencePath);
|
|
5758
|
+
if (evidencePath && existsSync4(evidencePath)) {
|
|
5759
|
+
try {
|
|
5760
|
+
return parseJsonObject(readFileSync4(evidencePath, "utf8")) ?? stdoutReport;
|
|
5761
|
+
} catch {
|
|
5762
|
+
return stdoutReport;
|
|
5763
|
+
}
|
|
5764
|
+
}
|
|
5765
|
+
return stdoutReport;
|
|
5766
|
+
}
|
|
5767
|
+
function commandName(command) {
|
|
5768
|
+
return command.split(/[\\/]/).at(-1) ?? command;
|
|
5769
|
+
}
|
|
5770
|
+
function argsContainSequence(args, sequence) {
|
|
5771
|
+
for (let index = 0;index <= args.length - sequence.length; index += 1) {
|
|
5772
|
+
if (sequence.every((part, offset) => args[index + offset] === part))
|
|
5773
|
+
return true;
|
|
5774
|
+
}
|
|
5775
|
+
return false;
|
|
5776
|
+
}
|
|
5777
|
+
function isRouteDrainLoop(loop) {
|
|
5778
|
+
if (loop.target.type !== "command")
|
|
5779
|
+
return false;
|
|
5780
|
+
if (commandName(loop.target.command) !== "loops")
|
|
5781
|
+
return false;
|
|
5782
|
+
const args = loop.target.args ?? [];
|
|
5783
|
+
return argsContainSequence(args, ["events", "drain", "todos-task"]) || argsContainSequence(args, ["routes", "drain", "todos-task"]) || argsContainSequence(args, ["route", "drain", "todos-task"]);
|
|
5784
|
+
}
|
|
5785
|
+
function routeResultTaskState(result) {
|
|
5786
|
+
const event = objectField(result, "event");
|
|
5787
|
+
const data = objectField(event, "data");
|
|
5788
|
+
const task = objectField(data, "task");
|
|
5789
|
+
const payload = objectField(data, "payload");
|
|
5790
|
+
const payloadTask = objectField(payload, "task");
|
|
5791
|
+
const metadata = objectField(data, "metadata");
|
|
5792
|
+
const records = [data, task, payload, payloadTask, metadata].filter(isRecord);
|
|
5793
|
+
const tags = new Set;
|
|
5794
|
+
for (const record of records) {
|
|
5795
|
+
for (const tag of tagsFromValue(record.tags ?? record.task_tags ?? record.taskTags)) {
|
|
5796
|
+
tags.add(tag.toLowerCase());
|
|
5797
|
+
}
|
|
5798
|
+
}
|
|
5799
|
+
const status = records.map((record) => stringValue(record.status ?? record.task_status ?? record.taskStatus)?.toLowerCase()).find(Boolean);
|
|
5800
|
+
return {
|
|
5801
|
+
taskId: stringValue(event?.subject) ?? stringValue(data?.id) ?? stringValue(task?.id) ?? stringValue(payloadTask?.id),
|
|
5802
|
+
tags: [...tags],
|
|
5803
|
+
status
|
|
5804
|
+
};
|
|
5805
|
+
}
|
|
5806
|
+
function detectRouteFunctionalFailure(store, loop, run) {
|
|
5807
|
+
if (run.status !== "succeeded")
|
|
5808
|
+
return;
|
|
5809
|
+
if (!isRouteDrainLoop(loop))
|
|
5810
|
+
return;
|
|
5811
|
+
const report = routeEvidenceReport(run);
|
|
5812
|
+
const rawResults = Array.isArray(report?.results) ? report.results.filter(isRecord) : [];
|
|
5813
|
+
for (const result of rawResults) {
|
|
5814
|
+
const kind = stringValue(result.kind);
|
|
5815
|
+
const task = routeResultTaskState(result);
|
|
5816
|
+
const disallowedTag = task.tags.find((tag) => ROUTE_FUNCTIONAL_DISALLOWED_TAGS.has(tag));
|
|
5817
|
+
if (kind && kind !== "skipped" && disallowedTag) {
|
|
5818
|
+
const reason2 = `route drain ${kind} task ${task.taskId ?? "unknown"} with disallowed tag ${disallowedTag}`;
|
|
5819
|
+
return {
|
|
5820
|
+
classification: "route_functional",
|
|
5821
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, reason2),
|
|
5822
|
+
evidence: { error: reason2, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5823
|
+
};
|
|
5824
|
+
}
|
|
5825
|
+
if (kind && kind !== "skipped" && task.status && ROUTE_FUNCTIONAL_DISALLOWED_STATUSES.has(task.status)) {
|
|
5826
|
+
const reason2 = `route drain ${kind} task ${task.taskId ?? "unknown"} with non-routable status ${task.status}`;
|
|
5827
|
+
return {
|
|
5828
|
+
classification: "route_functional",
|
|
5829
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, reason2),
|
|
5830
|
+
evidence: { error: reason2, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5831
|
+
};
|
|
5832
|
+
}
|
|
5833
|
+
const reason = stringValue(result.reason);
|
|
5834
|
+
if (kind === "skipped" && reason === "task metadata requires manual or approval-gated handling") {
|
|
5835
|
+
const message = `route drain skipped task ${task.taskId ?? "unknown"} with ambiguous manual-gate reason`;
|
|
5836
|
+
return {
|
|
5837
|
+
classification: "route_functional",
|
|
5838
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, message),
|
|
5839
|
+
evidence: { error: message, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5840
|
+
};
|
|
5841
|
+
}
|
|
5842
|
+
const sourceTaskUpdate = objectField(result, "sourceTaskUpdate");
|
|
5843
|
+
if (kind === "skipped" && sourceTaskUpdate && sourceTaskUpdate.ok === false) {
|
|
5844
|
+
const updateError = stringValue(sourceTaskUpdate.error);
|
|
5845
|
+
const message = `route drain skipped task ${task.taskId ?? "unknown"} but failed to update source task${updateError ? `: ${updateError}` : ""}`;
|
|
5846
|
+
return {
|
|
5847
|
+
classification: "route_functional",
|
|
5848
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, message),
|
|
5849
|
+
evidence: { error: message, stdout: redactedEvidence(run.stdout), exitCode: run.exitCode }
|
|
5850
|
+
};
|
|
5851
|
+
}
|
|
5852
|
+
const childLoopId = stringValue(objectField(result, "loop")?.id) ?? stringValue(result.loopId);
|
|
5853
|
+
const childRun = childLoopId ? store.listRuns({ loopId: childLoopId, limit: 1 })[0] : undefined;
|
|
5854
|
+
if (childRun && !["succeeded", "running"].includes(childRun.status)) {
|
|
5855
|
+
const message = `route drain ${kind ?? "handled"} task ${task.taskId ?? "unknown"} but child loop ${childLoopId} latest run is ${childRun.status}`;
|
|
5856
|
+
return {
|
|
5857
|
+
classification: "route_functional",
|
|
5858
|
+
fingerprint: stableRouteFunctionalFingerprint(loop, message),
|
|
5859
|
+
evidence: { error: message, stdout: redactedEvidence(run.stdout), stderr: redactedEvidence(childRun.stderr), exitCode: childRun.exitCode }
|
|
5860
|
+
};
|
|
5861
|
+
}
|
|
5862
|
+
}
|
|
5863
|
+
return;
|
|
5864
|
+
}
|
|
5865
|
+
function targetRoute(loop) {
|
|
5866
|
+
if (loop.target.type === "agent") {
|
|
4172
5867
|
return {
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
startedAt,
|
|
4180
|
-
finishedAt,
|
|
4181
|
-
durationMs
|
|
5868
|
+
source: "openloops",
|
|
5869
|
+
kind: "loop_expectation",
|
|
5870
|
+
loopId: loop.id,
|
|
5871
|
+
loopName: loop.name,
|
|
5872
|
+
cwd: loop.target.cwd,
|
|
5873
|
+
provider: loop.target.provider
|
|
4182
5874
|
};
|
|
4183
5875
|
}
|
|
4184
|
-
if (
|
|
5876
|
+
if (loop.target.type === "command") {
|
|
4185
5877
|
return {
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
pid: child.pid,
|
|
4192
|
-
startedAt,
|
|
4193
|
-
finishedAt,
|
|
4194
|
-
durationMs
|
|
5878
|
+
source: "openloops",
|
|
5879
|
+
kind: "loop_expectation",
|
|
5880
|
+
loopId: loop.id,
|
|
5881
|
+
loopName: loop.name,
|
|
5882
|
+
cwd: loop.target.cwd
|
|
4195
5883
|
};
|
|
4196
5884
|
}
|
|
4197
5885
|
return {
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
pid: child.pid,
|
|
4203
|
-
startedAt,
|
|
4204
|
-
finishedAt,
|
|
4205
|
-
durationMs
|
|
5886
|
+
source: "openloops",
|
|
5887
|
+
kind: "loop_expectation",
|
|
5888
|
+
loopId: loop.id,
|
|
5889
|
+
loopName: loop.name
|
|
4206
5890
|
};
|
|
4207
5891
|
}
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
5892
|
+
function recommendedTask(loop, run, failure, route) {
|
|
5893
|
+
const title = `BUG: open-loops loop failure - ${loop.name}`;
|
|
5894
|
+
const description = [
|
|
5895
|
+
`OpenLoops expectation failed for loop ${loop.name} (${loop.id}).`,
|
|
5896
|
+
`Run: ${run.id}`,
|
|
5897
|
+
`Status: ${run.status}`,
|
|
5898
|
+
`Classification: ${failure.classification}`,
|
|
5899
|
+
`Fingerprint: ${failure.fingerprint}`,
|
|
5900
|
+
`No-tmux routing: Do not dispatch or paste prompts into tmux panes; use task-triggered headless worker/verifier workflows only.`,
|
|
5901
|
+
route.cwd ? `Route cwd: ${route.cwd}` : undefined,
|
|
5902
|
+
route.provider ? `Provider: ${route.provider}` : undefined,
|
|
5903
|
+
failure.evidence.error ? `Error:
|
|
5904
|
+
${failure.evidence.error}` : undefined,
|
|
5905
|
+
failure.evidence.stderr ? `Stderr:
|
|
5906
|
+
${failure.evidence.stderr}` : undefined
|
|
5907
|
+
].filter(Boolean).join(`
|
|
5908
|
+
|
|
5909
|
+
`);
|
|
5910
|
+
const dedupeKey = `openloops:${loop.id}:${failure.fingerprint}`;
|
|
5911
|
+
const tags = ["bug", "openloops", "loop-health", failure.classification];
|
|
5912
|
+
const priority = failure.classification === "auth" || failure.classification === "rate_limit" ? "high" : "medium";
|
|
5913
|
+
return {
|
|
5914
|
+
title,
|
|
5915
|
+
description,
|
|
5916
|
+
priority,
|
|
5917
|
+
tags,
|
|
5918
|
+
dedupeKey,
|
|
5919
|
+
search: { query: dedupeKey },
|
|
5920
|
+
compatibilityFallback: {
|
|
5921
|
+
search: ["todos", "search", dedupeKey, "--json"],
|
|
5922
|
+
add: ["todos", "add", title, "--description", description, "--tag", tags.join(","), "--priority", priority],
|
|
5923
|
+
comment: ["todos", "comment", "<task-id>", description]
|
|
5924
|
+
},
|
|
5925
|
+
futureNativeUpsert: {
|
|
5926
|
+
command: "todos upsert",
|
|
5927
|
+
fields: {
|
|
5928
|
+
title,
|
|
5929
|
+
description,
|
|
5930
|
+
priority,
|
|
5931
|
+
tags,
|
|
5932
|
+
dedupeKey,
|
|
5933
|
+
routeSource: route.source,
|
|
5934
|
+
routeKind: route.kind,
|
|
5935
|
+
routeLoopId: route.loopId,
|
|
5936
|
+
routeLoopName: route.loopName
|
|
5937
|
+
}
|
|
5938
|
+
}
|
|
5939
|
+
};
|
|
5940
|
+
}
|
|
5941
|
+
function expectationForLoop(store, loop) {
|
|
5942
|
+
const latestRun = store.listRuns({ loopId: loop.id, limit: 1 })[0];
|
|
5943
|
+
const route = targetRoute(loop);
|
|
5944
|
+
if (!latestRun) {
|
|
5945
|
+
return {
|
|
5946
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5947
|
+
ok: true,
|
|
5948
|
+
check: { id: "latest-run-succeeded", status: "warn", message: "loop has no recorded runs yet" },
|
|
5949
|
+
route
|
|
5950
|
+
};
|
|
4211
5951
|
}
|
|
4212
|
-
|
|
4213
|
-
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
5952
|
+
const routeFailure = detectRouteFunctionalFailure(store, loop, latestRun);
|
|
5953
|
+
if (routeFailure) {
|
|
5954
|
+
return {
|
|
5955
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5956
|
+
ok: false,
|
|
5957
|
+
check: { id: "route-functional-health", status: "fail", message: routeFailure.evidence.error ?? "route functional blocker detected" },
|
|
5958
|
+
latestRun: healthRun(latestRun),
|
|
5959
|
+
failure: routeFailure,
|
|
5960
|
+
route,
|
|
5961
|
+
recommendedTask: recommendedTask(loop, latestRun, routeFailure, route)
|
|
5962
|
+
};
|
|
5963
|
+
}
|
|
5964
|
+
if (latestRun.status === "succeeded") {
|
|
5965
|
+
return {
|
|
5966
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5967
|
+
ok: true,
|
|
5968
|
+
check: { id: "latest-run-succeeded", status: "pass", message: "latest run succeeded" },
|
|
5969
|
+
latestRun: healthRun(latestRun),
|
|
5970
|
+
route
|
|
5971
|
+
};
|
|
5972
|
+
}
|
|
5973
|
+
const failure = classifyRunFailure(latestRun);
|
|
5974
|
+
if (failure?.classification === "circuit_breaker") {
|
|
5975
|
+
if (loop.status !== "paused") {
|
|
4223
5976
|
return {
|
|
4224
|
-
status:
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
finishedAt,
|
|
4230
|
-
durationMs: new Date(finishedAt).getTime() - new Date(startedAt).getTime()
|
|
5977
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5978
|
+
ok: true,
|
|
5979
|
+
check: { id: "latest-run-succeeded", status: "warn", message: "circuit breaker cleared by resume; awaiting next run" },
|
|
5980
|
+
latestRun: healthRun(latestRun),
|
|
5981
|
+
route
|
|
4231
5982
|
};
|
|
4232
5983
|
}
|
|
5984
|
+
return {
|
|
5985
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5986
|
+
ok: false,
|
|
5987
|
+
check: { id: "latest-run-succeeded", status: "fail", message: latestRun.error ?? "circuit breaker open; loop auto-paused" },
|
|
5988
|
+
latestRun: healthRun(latestRun),
|
|
5989
|
+
failure,
|
|
5990
|
+
route,
|
|
5991
|
+
recommendedTask: recommendedTask(loop, latestRun, failure, route)
|
|
5992
|
+
};
|
|
4233
5993
|
}
|
|
4234
|
-
return
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
5994
|
+
return {
|
|
5995
|
+
loop: { id: loop.id, name: loop.name, status: loop.status, nextRunAt: loop.nextRunAt },
|
|
5996
|
+
ok: false,
|
|
5997
|
+
check: { id: "latest-run-succeeded", status: "fail", message: `latest run is ${latestRun.status}` },
|
|
5998
|
+
latestRun: healthRun(latestRun),
|
|
5999
|
+
failure,
|
|
6000
|
+
route,
|
|
6001
|
+
recommendedTask: failure ? recommendedTask(loop, latestRun, failure, route) : undefined
|
|
6002
|
+
};
|
|
4240
6003
|
}
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
|
|
4249
|
-
function resolveGoalModel(opts = {}) {
|
|
4250
|
-
const env = opts.env ?? process.env;
|
|
4251
|
-
const apiKey = opts.apiKey ?? env.OPENROUTER_API_KEY;
|
|
4252
|
-
if (!apiKey) {
|
|
4253
|
-
throw new Error("OPENROUTER_API_KEY is required to run goals with the OpenRouter AI SDK provider");
|
|
6004
|
+
function buildHealthReport(store, opts = {}) {
|
|
6005
|
+
const loops = store.listLoops({ includeArchived: opts.includeArchived, limit: opts.limit ?? 200 }).filter((loop) => opts.includeInactive || loop.status === "active" || loop.status === "paused");
|
|
6006
|
+
const expectations = loops.map((loop) => expectationForLoop(store, loop));
|
|
6007
|
+
const classifications = Object.fromEntries(CLASSIFICATIONS.map((key) => [key, 0]));
|
|
6008
|
+
for (const expectation of expectations) {
|
|
6009
|
+
if (expectation.failure)
|
|
6010
|
+
classifications[expectation.failure.classification] += 1;
|
|
4254
6011
|
}
|
|
4255
|
-
const
|
|
4256
|
-
|
|
4257
|
-
|
|
4258
|
-
|
|
4259
|
-
|
|
6012
|
+
const unhealthy = expectations.filter((expectation) => !expectation.ok).length;
|
|
6013
|
+
const warnings = expectations.filter((expectation) => expectation.check.status === "warn").length;
|
|
6014
|
+
return {
|
|
6015
|
+
ok: unhealthy === 0,
|
|
6016
|
+
generatedAt: new Date().toISOString(),
|
|
6017
|
+
summary: {
|
|
6018
|
+
loops: expectations.length,
|
|
6019
|
+
healthy: expectations.length - unhealthy,
|
|
6020
|
+
unhealthy,
|
|
6021
|
+
warnings
|
|
6022
|
+
},
|
|
6023
|
+
classifications,
|
|
6024
|
+
expectations
|
|
6025
|
+
};
|
|
6026
|
+
}
|
|
6027
|
+
|
|
6028
|
+
// src/lib/goal/metadata.ts
|
|
6029
|
+
function goalExecutionContext(parts) {
|
|
6030
|
+
return {
|
|
6031
|
+
loopId: parts.loop?.id,
|
|
6032
|
+
loopName: parts.loop?.name,
|
|
6033
|
+
loopRunId: parts.loopRun?.id,
|
|
6034
|
+
scheduledFor: parts.loopRun?.scheduledFor ?? parts.scheduledFor,
|
|
6035
|
+
workflowId: parts.workflow?.id,
|
|
6036
|
+
workflowName: parts.workflow?.name,
|
|
6037
|
+
workflowRunId: parts.workflowRunId,
|
|
6038
|
+
workflowStepId: parts.workflowStepId
|
|
6039
|
+
};
|
|
6040
|
+
}
|
|
6041
|
+
function executionMetadata(context, goal, node) {
|
|
6042
|
+
return {
|
|
6043
|
+
loopId: context?.loopId,
|
|
6044
|
+
loopName: context?.loopName,
|
|
6045
|
+
runId: context?.loopRunId,
|
|
6046
|
+
scheduledFor: context?.scheduledFor,
|
|
6047
|
+
workflowId: context?.workflowId,
|
|
6048
|
+
workflowName: context?.workflowName,
|
|
6049
|
+
workflowRunId: context?.workflowRunId,
|
|
6050
|
+
workflowStepId: context?.workflowStepId,
|
|
6051
|
+
goalId: goal?.goalId,
|
|
6052
|
+
goalObjective: goal?.objective,
|
|
6053
|
+
goalNodeKey: node?.key
|
|
6054
|
+
};
|
|
6055
|
+
}
|
|
6056
|
+
function withGoalNodeEnv(env, node) {
|
|
6057
|
+
return { ...env ?? process.env, LOOPS_GOAL_NODE_KEY: node.key, LOOPS_GOAL_NODE_OBJECTIVE: node.objective };
|
|
4260
6058
|
}
|
|
4261
6059
|
|
|
4262
6060
|
// src/lib/goal/prompts.ts
|
|
@@ -4272,6 +6070,23 @@ function planPrompt(spec) {
|
|
|
4272
6070
|
].join(`
|
|
4273
6071
|
`);
|
|
4274
6072
|
}
|
|
6073
|
+
function iterationPrompt(goal, node) {
|
|
6074
|
+
return [
|
|
6075
|
+
"Continue executing the goal plan without losing budget discipline.",
|
|
6076
|
+
`Goal: ${goal.objective}`,
|
|
6077
|
+
`Plan node: ${node.key}`,
|
|
6078
|
+
`Node objective: ${node.objective}`,
|
|
6079
|
+
"Work only on this node's objective in this run; other plan nodes run in their own iterations."
|
|
6080
|
+
].join(`
|
|
6081
|
+
`);
|
|
6082
|
+
}
|
|
6083
|
+
function goalNodeTarget(target, goal, node) {
|
|
6084
|
+
if (target.type !== "agent")
|
|
6085
|
+
return target;
|
|
6086
|
+
return { ...target, prompt: `${target.prompt}
|
|
6087
|
+
|
|
6088
|
+
${iterationPrompt(goal, node)}` };
|
|
6089
|
+
}
|
|
4275
6090
|
function achievementPrompt(goal, nodes, evidence) {
|
|
4276
6091
|
return [
|
|
4277
6092
|
"Run an adversarial achievement audit.",
|
|
@@ -4291,6 +6106,79 @@ ${evidence.join(`
|
|
|
4291
6106
|
`);
|
|
4292
6107
|
}
|
|
4293
6108
|
|
|
6109
|
+
// src/lib/goal/runner.ts
|
|
6110
|
+
import { generateObject } from "ai";
|
|
6111
|
+
import { z } from "zod";
|
|
6112
|
+
|
|
6113
|
+
// src/lib/run-envelope.ts
|
|
6114
|
+
var ENVELOPE_EXCERPT_CHARS = 2048;
|
|
6115
|
+
var BLOCKED_STEP_ERROR_PREFIX = "blocked:";
|
|
6116
|
+
function boundedExcerpt(text, limit = ENVELOPE_EXCERPT_CHARS) {
|
|
6117
|
+
if (!text)
|
|
6118
|
+
return;
|
|
6119
|
+
const scrubbed = scrubSecrets(text);
|
|
6120
|
+
if (scrubbed.length <= limit * 2)
|
|
6121
|
+
return scrubbed;
|
|
6122
|
+
const omitted = scrubbed.length - limit * 2;
|
|
6123
|
+
return `${scrubbed.slice(0, limit)}
|
|
6124
|
+
[... ${omitted} chars omitted ...]
|
|
6125
|
+
${scrubbed.slice(-limit)}`;
|
|
6126
|
+
}
|
|
6127
|
+
function summarizeOutput(stdout, stderr) {
|
|
6128
|
+
return {
|
|
6129
|
+
stdoutBytes: stdout ? Buffer.byteLength(stdout, "utf8") : 0,
|
|
6130
|
+
stderrBytes: stderr ? Buffer.byteLength(stderr, "utf8") : 0,
|
|
6131
|
+
stdoutExcerpt: boundedExcerpt(stdout),
|
|
6132
|
+
stderrExcerpt: boundedExcerpt(stderr)
|
|
6133
|
+
};
|
|
6134
|
+
}
|
|
6135
|
+
function summarizeExecutorResult(result) {
|
|
6136
|
+
return {
|
|
6137
|
+
...summarizeOutput(result.stdout, result.stderr),
|
|
6138
|
+
status: result.status,
|
|
6139
|
+
exitCode: result.exitCode,
|
|
6140
|
+
durationMs: result.durationMs,
|
|
6141
|
+
error: boundedExcerpt(result.error)
|
|
6142
|
+
};
|
|
6143
|
+
}
|
|
6144
|
+
function isBlockedStepRun(step) {
|
|
6145
|
+
return step?.status === "skipped" && (step.error?.startsWith(BLOCKED_STEP_ERROR_PREFIX) ?? false);
|
|
6146
|
+
}
|
|
6147
|
+
function summarizeWorkflowStepRun(step) {
|
|
6148
|
+
return {
|
|
6149
|
+
...summarizeOutput(step.stdout, step.stderr),
|
|
6150
|
+
stepId: step.stepId,
|
|
6151
|
+
status: step.status,
|
|
6152
|
+
exitCode: step.exitCode,
|
|
6153
|
+
durationMs: step.durationMs,
|
|
6154
|
+
error: boundedExcerpt(step.error),
|
|
6155
|
+
blocked: isBlockedStepRun(step)
|
|
6156
|
+
};
|
|
6157
|
+
}
|
|
6158
|
+
function workflowRunEnvelope(run, steps) {
|
|
6159
|
+
return JSON.stringify({ workflowRun: run, steps: steps.map(summarizeWorkflowStepRun) }, null, 2);
|
|
6160
|
+
}
|
|
6161
|
+
|
|
6162
|
+
// src/lib/goal/model-factory.ts
|
|
6163
|
+
import { createOpenRouter } from "@openrouter/ai-sdk-provider";
|
|
6164
|
+
var DEFAULT_GOAL_MODEL = "openai/gpt-4o-mini";
|
|
6165
|
+
function resolveGoalModel(opts = {}) {
|
|
6166
|
+
const env = opts.env ?? process.env;
|
|
6167
|
+
const apiKey = opts.apiKey ?? env.OPENROUTER_API_KEY;
|
|
6168
|
+
if (!apiKey) {
|
|
6169
|
+
throw new Error("OPENROUTER_API_KEY is required to run goals with the OpenRouter AI SDK provider");
|
|
6170
|
+
}
|
|
6171
|
+
const provider = createOpenRouter({
|
|
6172
|
+
apiKey,
|
|
6173
|
+
baseURL: opts.baseURL ?? env.LOOPS_GOAL_BASE_URL ?? env.OPENROUTER_BASE_URL
|
|
6174
|
+
});
|
|
6175
|
+
return provider.chat(opts.model ?? env.LOOPS_GOAL_MODEL ?? DEFAULT_GOAL_MODEL);
|
|
6176
|
+
}
|
|
6177
|
+
function resolveGoalVerifierModel(opts = {}) {
|
|
6178
|
+
const env = opts.env ?? process.env;
|
|
6179
|
+
return resolveGoalModel({ ...opts, model: opts.model ?? env.LOOPS_GOAL_VERIFIER_MODEL });
|
|
6180
|
+
}
|
|
6181
|
+
|
|
4294
6182
|
// src/lib/goal/runner.ts
|
|
4295
6183
|
var DEFAULT_MAX_TURNS = 10;
|
|
4296
6184
|
var PlanNodeSchema = z.object({
|
|
@@ -4418,34 +6306,39 @@ function noReadyDiagnostic(goal, nodes) {
|
|
|
4418
6306
|
incompleteNodes: incompleteDetails
|
|
4419
6307
|
};
|
|
4420
6308
|
}
|
|
4421
|
-
function metadataFor(goal, node, context) {
|
|
4422
|
-
return {
|
|
4423
|
-
loopId: context?.loopId,
|
|
4424
|
-
loopName: context?.loopName,
|
|
4425
|
-
runId: context?.loopRunId,
|
|
4426
|
-
scheduledFor: context?.scheduledFor,
|
|
4427
|
-
workflowId: context?.workflowId,
|
|
4428
|
-
workflowName: context?.workflowName,
|
|
4429
|
-
workflowRunId: context?.workflowRunId,
|
|
4430
|
-
workflowStepId: context?.workflowStepId,
|
|
4431
|
-
goalId: goal.goalId,
|
|
4432
|
-
goalObjective: goal.objective,
|
|
4433
|
-
goalNodeKey: node.key
|
|
4434
|
-
};
|
|
4435
|
-
}
|
|
4436
6309
|
async function executeUnderlyingTarget(target, goal, node, opts) {
|
|
4437
|
-
const metadata =
|
|
6310
|
+
const metadata = executionMetadata(opts.context, goal, node);
|
|
4438
6311
|
if (opts.executeNode)
|
|
4439
6312
|
return opts.executeNode(node, metadata);
|
|
4440
6313
|
if (!target)
|
|
4441
6314
|
throw new Error("runGoal requires either target or executeNode");
|
|
4442
|
-
return executeTarget(target, metadata, {
|
|
4443
|
-
env: opts.env,
|
|
6315
|
+
return executeTarget(goalNodeTarget(target, goal, node), metadata, {
|
|
6316
|
+
env: withGoalNodeEnv(opts.env, node),
|
|
4444
6317
|
daemonLeaseId: opts.daemonLeaseId,
|
|
4445
6318
|
beforePersist: opts.beforePersist,
|
|
4446
6319
|
signal: opts.signal
|
|
4447
6320
|
});
|
|
4448
6321
|
}
|
|
6322
|
+
function verifierModelFor(spec, opts, planner) {
|
|
6323
|
+
if (opts.verifierModel)
|
|
6324
|
+
return opts.verifierModel;
|
|
6325
|
+
const env = opts.env ?? process.env;
|
|
6326
|
+
const configured = spec.verifierModel ?? env.LOOPS_GOAL_VERIFIER_MODEL;
|
|
6327
|
+
if (!configured)
|
|
6328
|
+
return planner;
|
|
6329
|
+
return resolveGoalVerifierModel({ model: configured, env: opts.env });
|
|
6330
|
+
}
|
|
6331
|
+
function nodeEvidence(node, result) {
|
|
6332
|
+
const summary = summarizeExecutorResult(result);
|
|
6333
|
+
return [
|
|
6334
|
+
`node ${node.key} ${summary.status} (exit ${summary.exitCode ?? "unknown"}, ${summary.durationMs}ms, ` + `stdout ${summary.stdoutBytes}B, stderr ${summary.stderrBytes}B)`,
|
|
6335
|
+
summary.stdoutExcerpt ? `stdout excerpt:
|
|
6336
|
+
${summary.stdoutExcerpt}` : undefined,
|
|
6337
|
+
summary.stderrExcerpt ? `stderr excerpt:
|
|
6338
|
+
${summary.stderrExcerpt}` : undefined
|
|
6339
|
+
].filter(Boolean).join(`
|
|
6340
|
+
`);
|
|
6341
|
+
}
|
|
4449
6342
|
async function planGoal(store, goal, spec, model, opts) {
|
|
4450
6343
|
const existing = store.listGoalPlanNodes(goal.goalId);
|
|
4451
6344
|
if (existing.length > 0)
|
|
@@ -4479,18 +6372,19 @@ async function planGoal(store, goal, spec, model, opts) {
|
|
|
4479
6372
|
return store.createGoalPlanNodes(goal.goalId, rawNodes, { daemonLeaseId: opts.daemonLeaseId });
|
|
4480
6373
|
}
|
|
4481
6374
|
function stdoutFor(goal, nodes, evidence, validation, diagnostics) {
|
|
4482
|
-
return JSON.stringify({
|
|
6375
|
+
return scrubSecrets(JSON.stringify(scrubSecretsDeep({
|
|
4483
6376
|
goal,
|
|
4484
6377
|
rollup: rollupSummary(nodes),
|
|
4485
6378
|
nodes,
|
|
4486
6379
|
evidence,
|
|
4487
6380
|
validation,
|
|
4488
6381
|
diagnostics
|
|
4489
|
-
}, null, 2);
|
|
6382
|
+
}), null, 2));
|
|
4490
6383
|
}
|
|
4491
6384
|
async function runGoal(store, input, opts = {}) {
|
|
4492
6385
|
const spec = normalizeGoalSpec2(input);
|
|
4493
6386
|
const model = opts.model ?? resolveGoalModel({ model: spec.model, env: opts.env });
|
|
6387
|
+
const verifier = verifierModelFor(spec, opts, model);
|
|
4494
6388
|
const startedAt = nowIso();
|
|
4495
6389
|
const existing = store.findGoalByContext({
|
|
4496
6390
|
loopRunId: opts.context?.loopRunId,
|
|
@@ -4523,6 +6417,13 @@ async function runGoal(store, input, opts = {}) {
|
|
|
4523
6417
|
goal = store.updateGoalStatus(goal.goalId, "budgetLimited", { daemonLeaseId: opts.daemonLeaseId });
|
|
4524
6418
|
return resultFromGoal(goal, "failed", stdoutFor(goal, nodes, evidence), "goal token budget exhausted after planning", startedAt);
|
|
4525
6419
|
}
|
|
6420
|
+
if ((goal.autoExecute ?? spec.autoExecute) === "off") {
|
|
6421
|
+
nodes = syncReadyFlags(store, goal, nodes, opts);
|
|
6422
|
+
return resultFromGoal(goal, "succeeded", stdoutFor(goal, nodes, evidence, undefined, {
|
|
6423
|
+
autoExecute: "off",
|
|
6424
|
+
note: "autoExecute is off: goal plan persisted without executing any nodes"
|
|
6425
|
+
}), undefined, startedAt);
|
|
6426
|
+
}
|
|
4526
6427
|
for (let turn = 1;turn <= (spec.maxTurns ?? DEFAULT_MAX_TURNS); turn++) {
|
|
4527
6428
|
if (opts.signal?.aborted) {
|
|
4528
6429
|
goal = store.updateGoalStatus(goal.goalId, "cancelled", { daemonLeaseId: opts.daemonLeaseId });
|
|
@@ -4561,11 +6462,7 @@ async function runGoal(store, input, opts = {}) {
|
|
|
4561
6462
|
}
|
|
4562
6463
|
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
4563
6464
|
if (result.status === "succeeded") {
|
|
4564
|
-
evidence.push(
|
|
4565
|
-
stdout:
|
|
4566
|
-
${result.stdout}
|
|
4567
|
-
stderr:
|
|
4568
|
-
${result.stderr}`);
|
|
6465
|
+
evidence.push(nodeEvidence(node, result));
|
|
4569
6466
|
store.updateGoalPlanNode(goal.goalId, node.key, {
|
|
4570
6467
|
status: "complete",
|
|
4571
6468
|
timeUsedSeconds: Math.round(result.durationMs / 1000)
|
|
@@ -4592,7 +6489,7 @@ ${result.stderr}`);
|
|
|
4592
6489
|
}
|
|
4593
6490
|
if (nodes.every((node) => node.status === "complete")) {
|
|
4594
6491
|
const judged = await generateObject({
|
|
4595
|
-
model,
|
|
6492
|
+
model: verifier,
|
|
4596
6493
|
schema: AchievementSchema,
|
|
4597
6494
|
temperature: 0,
|
|
4598
6495
|
prompt: achievementPrompt(goal, nodes, evidence),
|
|
@@ -4661,19 +6558,31 @@ ${result.stderr}`);
|
|
|
4661
6558
|
}
|
|
4662
6559
|
|
|
4663
6560
|
// src/lib/workflow-runner.ts
|
|
4664
|
-
|
|
6561
|
+
var DEFAULT_BLOCKED_EXIT_CODES = [12];
|
|
6562
|
+
var GATE_STEP_PATTERN = /(^|[^a-z])gate([^a-z]|$)/i;
|
|
6563
|
+
function blockedExitCodesForStep(step) {
|
|
6564
|
+
const explicit = step.blockedExitCodes;
|
|
6565
|
+
if (explicit !== undefined)
|
|
6566
|
+
return explicit.filter((code) => Number.isInteger(code));
|
|
6567
|
+
return GATE_STEP_PATTERN.test(step.name ? `${step.id} ${step.name}` : step.id) ? DEFAULT_BLOCKED_EXIT_CODES : [];
|
|
6568
|
+
}
|
|
6569
|
+
function targetWithStepAccount(step, goalNodePrompt) {
|
|
4665
6570
|
const account = step.account ?? step.target.account;
|
|
4666
6571
|
const timeoutMs = step.timeoutMs !== undefined ? step.timeoutMs : step.target.timeoutMs;
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
6572
|
+
const target = !account && timeoutMs === step.target.timeoutMs ? step.target : { ...step.target, account, timeoutMs };
|
|
6573
|
+
if (goalNodePrompt && target.type === "agent") {
|
|
6574
|
+
return { ...target, prompt: `${target.prompt}
|
|
6575
|
+
|
|
6576
|
+
${goalNodePrompt}` };
|
|
6577
|
+
}
|
|
6578
|
+
return target;
|
|
4670
6579
|
}
|
|
4671
|
-
function workflowResult(workflowRun, status, startedAt, finishedAt,
|
|
6580
|
+
function workflowResult(workflowRun, status, startedAt, finishedAt, steps, error) {
|
|
4672
6581
|
const executorStatus = status === "succeeded" ? "succeeded" : status === "timed_out" ? "timed_out" : "failed";
|
|
4673
6582
|
return {
|
|
4674
6583
|
status: executorStatus,
|
|
4675
6584
|
exitCode: executorStatus === "succeeded" ? 0 : 1,
|
|
4676
|
-
stdout,
|
|
6585
|
+
stdout: workflowRunEnvelope(workflowRun, steps),
|
|
4677
6586
|
stderr: "",
|
|
4678
6587
|
error,
|
|
4679
6588
|
startedAt,
|
|
@@ -4683,20 +6592,16 @@ function workflowResult(workflowRun, status, startedAt, finishedAt, stdout, erro
|
|
|
4683
6592
|
}
|
|
4684
6593
|
async function executeWorkflow(store, workflow, opts = {}) {
|
|
4685
6594
|
if (workflow.goal) {
|
|
6595
|
+
const goalSpec = workflow.goal;
|
|
4686
6596
|
const workflowWithoutGoal = { ...workflow, goal: undefined };
|
|
4687
|
-
return runGoal(store,
|
|
6597
|
+
return runGoal(store, goalSpec, {
|
|
4688
6598
|
...opts,
|
|
4689
6599
|
model: opts.goalModel,
|
|
4690
|
-
context: {
|
|
4691
|
-
loopId: opts.loop?.id,
|
|
4692
|
-
loopName: opts.loop?.name,
|
|
4693
|
-
loopRunId: opts.loopRun?.id,
|
|
4694
|
-
scheduledFor: opts.loopRun?.scheduledFor ?? opts.scheduledFor,
|
|
4695
|
-
workflowId: workflow.id,
|
|
4696
|
-
workflowName: workflow.name
|
|
4697
|
-
},
|
|
6600
|
+
context: goalExecutionContext({ loop: opts.loop, loopRun: opts.loopRun, scheduledFor: opts.scheduledFor, workflow }),
|
|
4698
6601
|
executeNode: async (node) => executeWorkflow(store, workflowWithoutGoal, {
|
|
4699
6602
|
...opts,
|
|
6603
|
+
env: withGoalNodeEnv(opts.env, node),
|
|
6604
|
+
goalNodePrompt: iterationPrompt(goalSpec, node),
|
|
4700
6605
|
idempotencyKey: `${opts.idempotencyKey ?? workflow.id}:goal:${node.key}`
|
|
4701
6606
|
})
|
|
4702
6607
|
});
|
|
@@ -4711,8 +6616,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4711
6616
|
});
|
|
4712
6617
|
const startedAt = run.startedAt ?? nowIso();
|
|
4713
6618
|
if (run.status === "succeeded" || run.status === "failed" || run.status === "timed_out" || run.status === "cancelled") {
|
|
4714
|
-
|
|
4715
|
-
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), JSON.stringify({ workflowRun: run, steps: steps2 }, null, 2), run.error);
|
|
6619
|
+
return workflowResult(run, run.status, startedAt, run.finishedAt ?? nowIso(), store.listWorkflowStepRuns(run.id), run.error);
|
|
4716
6620
|
}
|
|
4717
6621
|
const ordered = workflowExecutionOrder(workflow);
|
|
4718
6622
|
const byId = new Map(workflow.steps.map((step) => [step.id, step]));
|
|
@@ -4742,6 +6646,12 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4742
6646
|
});
|
|
4743
6647
|
if (blockedBy) {
|
|
4744
6648
|
opts.beforePersist?.();
|
|
6649
|
+
if (isBlockedStepRun(store.getWorkflowStepRun(run.id, blockedBy))) {
|
|
6650
|
+
store.skipWorkflowStepRun(run.id, step.id, `${BLOCKED_STEP_ERROR_PREFIX} upstream step ${blockedBy} was blocked`, {
|
|
6651
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6652
|
+
});
|
|
6653
|
+
continue;
|
|
6654
|
+
}
|
|
4745
6655
|
store.skipWorkflowStepRun(run.id, step.id, `dependency did not succeed: ${blockedBy}`, { daemonLeaseId: opts.daemonLeaseId });
|
|
4746
6656
|
blockingError ??= `step ${step.id} blocked by dependency ${blockedBy}`;
|
|
4747
6657
|
terminalStatus = "failed";
|
|
@@ -4754,16 +6664,14 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4754
6664
|
blockingError = `step ${step.id} could not start because workflow is no longer running`;
|
|
4755
6665
|
break;
|
|
4756
6666
|
}
|
|
4757
|
-
const
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
workflowId: workflow.id,
|
|
4763
|
-
workflowName: workflow.name,
|
|
6667
|
+
const stepContext = goalExecutionContext({
|
|
6668
|
+
loop: opts.loop,
|
|
6669
|
+
loopRun: opts.loopRun,
|
|
6670
|
+
scheduledFor: opts.scheduledFor,
|
|
6671
|
+
workflow,
|
|
4764
6672
|
workflowRunId: run.id,
|
|
4765
6673
|
workflowStepId: step.id
|
|
4766
|
-
};
|
|
6674
|
+
});
|
|
4767
6675
|
let result;
|
|
4768
6676
|
const controller = new AbortController;
|
|
4769
6677
|
const externalAbort = () => controller.abort();
|
|
@@ -4782,19 +6690,10 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4782
6690
|
model: opts.goalModel,
|
|
4783
6691
|
target: targetWithStepAccount(step),
|
|
4784
6692
|
signal: controller.signal,
|
|
4785
|
-
context:
|
|
4786
|
-
loopId: opts.loop?.id,
|
|
4787
|
-
loopName: opts.loop?.name,
|
|
4788
|
-
loopRunId: opts.loopRun?.id,
|
|
4789
|
-
scheduledFor: opts.loopRun?.scheduledFor ?? opts.scheduledFor,
|
|
4790
|
-
workflowId: workflow.id,
|
|
4791
|
-
workflowName: workflow.name,
|
|
4792
|
-
workflowRunId: run.id,
|
|
4793
|
-
workflowStepId: step.id
|
|
4794
|
-
}
|
|
6693
|
+
context: stepContext
|
|
4795
6694
|
});
|
|
4796
6695
|
} else {
|
|
4797
|
-
result = await executeTarget(targetWithStepAccount(step),
|
|
6696
|
+
result = await executeTarget(targetWithStepAccount(step, opts.goalNodePrompt), executionMetadata(stepContext), {
|
|
4798
6697
|
...opts,
|
|
4799
6698
|
machine: opts.machine ?? opts.loop?.machine,
|
|
4800
6699
|
signal: controller.signal,
|
|
@@ -4830,6 +6729,21 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4830
6729
|
break;
|
|
4831
6730
|
}
|
|
4832
6731
|
opts.beforePersist?.();
|
|
6732
|
+
const blockedExit = result.status === "failed" && result.exitCode !== undefined && blockedExitCodesForStep(step).includes(result.exitCode);
|
|
6733
|
+
if (blockedExit) {
|
|
6734
|
+
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
6735
|
+
status: "skipped",
|
|
6736
|
+
finishedAt: result.finishedAt,
|
|
6737
|
+
durationMs: result.durationMs,
|
|
6738
|
+
stdout: result.stdout,
|
|
6739
|
+
stderr: result.stderr,
|
|
6740
|
+
exitCode: result.exitCode,
|
|
6741
|
+
error: `${BLOCKED_STEP_ERROR_PREFIX} step ${step.id} exited with blocked exit code ${result.exitCode}`
|
|
6742
|
+
}, {
|
|
6743
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
6744
|
+
});
|
|
6745
|
+
continue;
|
|
6746
|
+
}
|
|
4833
6747
|
store.finalizeWorkflowStepRun(run.id, step.id, {
|
|
4834
6748
|
status: result.status,
|
|
4835
6749
|
finishedAt: result.finishedAt,
|
|
@@ -4860,8 +6774,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4860
6774
|
const finishedAt = nowIso();
|
|
4861
6775
|
if (store.isWorkflowRunTerminal(run.id)) {
|
|
4862
6776
|
const terminalRun = store.requireWorkflowRun(run.id);
|
|
4863
|
-
|
|
4864
|
-
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, JSON.stringify({ workflowRun: terminalRun, steps: steps2 }, null, 2), terminalRun.error ?? blockingError);
|
|
6777
|
+
return workflowResult(terminalRun, terminalRun.status, startedAt, terminalRun.finishedAt ?? finishedAt, store.listWorkflowStepRuns(run.id), terminalRun.error ?? blockingError);
|
|
4865
6778
|
}
|
|
4866
6779
|
opts.beforePersist?.();
|
|
4867
6780
|
const finalRun = store.finalizeWorkflowRun(run.id, terminalStatus, {
|
|
@@ -4871,8 +6784,7 @@ async function executeWorkflow(store, workflow, opts = {}) {
|
|
|
4871
6784
|
}, {
|
|
4872
6785
|
daemonLeaseId: opts.daemonLeaseId
|
|
4873
6786
|
});
|
|
4874
|
-
|
|
4875
|
-
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, JSON.stringify({ workflowRun: finalRun, steps }, null, 2), blockingError);
|
|
6787
|
+
return workflowResult(finalRun, terminalStatus, startedAt, finishedAt, store.listWorkflowStepRuns(run.id), blockingError);
|
|
4876
6788
|
}
|
|
4877
6789
|
function preflightWorkflow(workflow, opts = {}) {
|
|
4878
6790
|
return workflowExecutionOrder(workflow).map((step) => {
|
|
@@ -4923,12 +6835,7 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
4923
6835
|
...opts,
|
|
4924
6836
|
model: opts.goalModel,
|
|
4925
6837
|
target: loop.target,
|
|
4926
|
-
context: {
|
|
4927
|
-
loopId: loop.id,
|
|
4928
|
-
loopName: loop.name,
|
|
4929
|
-
loopRunId: run.id,
|
|
4930
|
-
scheduledFor: run.scheduledFor
|
|
4931
|
-
}
|
|
6838
|
+
context: goalExecutionContext({ loop, loopRun: run })
|
|
4932
6839
|
});
|
|
4933
6840
|
}
|
|
4934
6841
|
return executeLoop(loop, run, opts);
|
|
@@ -4943,23 +6850,19 @@ async function executeLoopTarget(store, loop, run, opts = {}) {
|
|
|
4943
6850
|
}
|
|
4944
6851
|
}
|
|
4945
6852
|
if (loop.goal) {
|
|
6853
|
+
const loopGoal = loop.goal;
|
|
4946
6854
|
const workflowForLoopGoal = workflow.goal ? { ...workflow, goal: undefined } : workflow;
|
|
4947
|
-
return runGoal(store,
|
|
6855
|
+
return runGoal(store, loopGoal, {
|
|
4948
6856
|
...opts,
|
|
4949
6857
|
model: opts.goalModel,
|
|
4950
|
-
context: {
|
|
4951
|
-
loopId: loop.id,
|
|
4952
|
-
loopName: loop.name,
|
|
4953
|
-
loopRunId: run.id,
|
|
4954
|
-
scheduledFor: run.scheduledFor,
|
|
4955
|
-
workflowId: workflow.id,
|
|
4956
|
-
workflowName: workflow.name
|
|
4957
|
-
},
|
|
6858
|
+
context: goalExecutionContext({ loop, loopRun: run, workflow }),
|
|
4958
6859
|
executeNode: async (node) => executeWorkflow(store, workflowForLoopGoal, {
|
|
4959
6860
|
...opts,
|
|
4960
6861
|
loop,
|
|
4961
6862
|
loopRun: run,
|
|
4962
6863
|
scheduledFor: run.scheduledFor,
|
|
6864
|
+
env: withGoalNodeEnv(opts.env, node),
|
|
6865
|
+
goalNodePrompt: iterationPrompt(loopGoal, node),
|
|
4963
6866
|
idempotencyKey: `${loop.id}:${run.scheduledFor}:attempt:${run.attempt}:goal:${node.key}`
|
|
4964
6867
|
})
|
|
4965
6868
|
});
|
|
@@ -5024,12 +6927,129 @@ function manualRunSource(loop, scheduledFor, now = new Date) {
|
|
|
5024
6927
|
return "retry_slot";
|
|
5025
6928
|
return "due_slot";
|
|
5026
6929
|
}
|
|
5027
|
-
|
|
5028
|
-
|
|
6930
|
+
var INLINE_RUNNER_ID_PATTERN = /^[^:\s]+:(\d+)$/;
|
|
6931
|
+
function inlineRunnerOwnerPid(claimedBy) {
|
|
6932
|
+
const match = claimedBy ? INLINE_RUNNER_ID_PATTERN.exec(claimedBy) : null;
|
|
6933
|
+
return match ? Number(match[1]) : undefined;
|
|
6934
|
+
}
|
|
6935
|
+
async function runLoopNow(deps) {
|
|
6936
|
+
const { store, runnerId } = deps;
|
|
6937
|
+
const loop = store.requireLoop(deps.idOrName);
|
|
6938
|
+
if (loop.archivedAt)
|
|
6939
|
+
throw new LoopArchivedError(loop.name || deps.idOrName);
|
|
6940
|
+
const now = deps.now?.() ?? new Date;
|
|
6941
|
+
if (deps.mode === "schedule") {
|
|
6942
|
+
const scheduledFor2 = now.toISOString();
|
|
6943
|
+
const updated = store.updateLoop(loop.id, { status: "active", nextRunAt: scheduledFor2 });
|
|
6944
|
+
return { mode: "schedule", loop: updated, scheduledFor: scheduledFor2 };
|
|
6945
|
+
}
|
|
6946
|
+
let scheduledFor = manualRunScheduledFor(loop, now);
|
|
6947
|
+
let source = manualRunSource(loop, scheduledFor, now);
|
|
6948
|
+
let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
|
|
6949
|
+
let claim = store.claimRun(loop, scheduledFor, runnerId, now);
|
|
6950
|
+
if (!claim && shouldAdvance) {
|
|
6951
|
+
const existing = store.getRunBySlot(loop.id, scheduledFor);
|
|
6952
|
+
if (existing && existing.status !== "running") {
|
|
6953
|
+
scheduledFor = now.toISOString();
|
|
6954
|
+
source = "ad_hoc";
|
|
6955
|
+
shouldAdvance = false;
|
|
6956
|
+
claim = store.claimRun(loop, scheduledFor, runnerId, now);
|
|
6957
|
+
}
|
|
6958
|
+
}
|
|
6959
|
+
if (!claim)
|
|
6960
|
+
throw new Error(`could not claim manual run for ${deps.idOrName}`);
|
|
6961
|
+
const run = await executeClaimedRun({
|
|
6962
|
+
store,
|
|
6963
|
+
runnerId,
|
|
6964
|
+
loop: claim.loop,
|
|
6965
|
+
run: claim.run,
|
|
6966
|
+
now: deps.now,
|
|
6967
|
+
execute: deps.execute
|
|
6968
|
+
});
|
|
6969
|
+
if (shouldAdvance) {
|
|
6970
|
+
advanceLoop(store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
|
|
6971
|
+
}
|
|
6972
|
+
return { mode: "inline", loop: claim.loop, run, source, advancedLoop: shouldAdvance };
|
|
6973
|
+
}
|
|
6974
|
+
var MAX_RETRY_DELAY_MS = 6 * 60 * 60 * 1000;
|
|
6975
|
+
var DEFAULT_CIRCUIT_BREAKER_THRESHOLD = 5;
|
|
6976
|
+
var CIRCUIT_BREAKER_REASON_PREFIX = "circuit breaker open";
|
|
6977
|
+
var MAX_SKIPS_PER_LOOP_PER_TICK = 10;
|
|
6978
|
+
var THROTTLED_RETRY_MULTIPLIER = 4;
|
|
6979
|
+
var MAX_RETRY_EXPONENT = 20;
|
|
6980
|
+
function retryBackoffDelayMs(loop, run, random = Math.random) {
|
|
6981
|
+
const attempt = Math.max(1, run.attempt);
|
|
6982
|
+
const failure = classifyRunFailure(run);
|
|
6983
|
+
const throttled = failure?.classification === "rate_limit" || failure?.classification === "auth";
|
|
6984
|
+
const growth = 2 ** Math.min(attempt - 1, MAX_RETRY_EXPONENT);
|
|
6985
|
+
const base = loop.retryDelayMs * growth * (throttled ? THROTTLED_RETRY_MULTIPLIER : 1);
|
|
6986
|
+
const jitter = 0.5 + random();
|
|
6987
|
+
return Math.min(MAX_RETRY_DELAY_MS, Math.round(base * jitter));
|
|
6988
|
+
}
|
|
6989
|
+
function nextAfterRetry(loop, run, now, random) {
|
|
6990
|
+
return new Date(now.getTime() + retryBackoffDelayMs(loop, run, random)).toISOString();
|
|
5029
6991
|
}
|
|
5030
6992
|
function isDaemonLeaseLost(error) {
|
|
5031
6993
|
return error instanceof Error && error.message === "daemon lease lost";
|
|
5032
6994
|
}
|
|
6995
|
+
function resolveBreakerThreshold(loop, override) {
|
|
6996
|
+
const perLoop = loop.circuitBreakerThreshold;
|
|
6997
|
+
if (typeof perLoop === "number" && Number.isFinite(perLoop))
|
|
6998
|
+
return Math.floor(perLoop);
|
|
6999
|
+
const resolved = typeof override === "function" ? override(loop) : override;
|
|
7000
|
+
if (typeof resolved === "number" && Number.isFinite(resolved))
|
|
7001
|
+
return Math.floor(resolved);
|
|
7002
|
+
return DEFAULT_CIRCUIT_BREAKER_THRESHOLD;
|
|
7003
|
+
}
|
|
7004
|
+
function consecutiveFailureCount(store, loopId, maxAttempts = 1, scanLimit = 50) {
|
|
7005
|
+
const runs = store.listRuns({ loopId, limit: scanLimit });
|
|
7006
|
+
let watermark;
|
|
7007
|
+
for (const run of runs) {
|
|
7008
|
+
if (run.status !== "skipped" || !run.error?.startsWith(CIRCUIT_BREAKER_REASON_PREFIX))
|
|
7009
|
+
continue;
|
|
7010
|
+
const at = new Date(run.scheduledFor).getTime();
|
|
7011
|
+
if (watermark === undefined || at > watermark)
|
|
7012
|
+
watermark = at;
|
|
7013
|
+
}
|
|
7014
|
+
let count = 0;
|
|
7015
|
+
for (const run of runs) {
|
|
7016
|
+
if (run.status === "running" || run.status === "skipped")
|
|
7017
|
+
continue;
|
|
7018
|
+
if (watermark !== undefined && new Date(run.scheduledFor).getTime() <= watermark)
|
|
7019
|
+
continue;
|
|
7020
|
+
if (run.status === "succeeded")
|
|
7021
|
+
break;
|
|
7022
|
+
if (run.attempt < maxAttempts)
|
|
7023
|
+
continue;
|
|
7024
|
+
count += 1;
|
|
7025
|
+
}
|
|
7026
|
+
return count;
|
|
7027
|
+
}
|
|
7028
|
+
function awaitStrictlyNewerRunTimestamp(store, loopId) {
|
|
7029
|
+
const latest = store.listRuns({ loopId, limit: 1 })[0];
|
|
7030
|
+
if (!latest)
|
|
7031
|
+
return;
|
|
7032
|
+
const latestMs = new Date(latest.createdAt).getTime();
|
|
7033
|
+
for (let spin = 0;spin < 1e6 && Date.now() <= latestMs; spin += 1) {}
|
|
7034
|
+
}
|
|
7035
|
+
function tripCircuitBreaker(store, loop, run, finishedAt, failures, opts) {
|
|
7036
|
+
awaitStrictlyNewerRunTimestamp(store, loop.id);
|
|
7037
|
+
const reason = `${CIRCUIT_BREAKER_REASON_PREFIX}: ${failures} consecutive failed runs; loop auto-paused (resume with 'loops resume ${loop.name}')`;
|
|
7038
|
+
let markerAtMs = finishedAt.getTime();
|
|
7039
|
+
for (let probe = 0;probe < 1000 && store.getRunBySlot(loop.id, new Date(markerAtMs).toISOString()); probe += 1) {
|
|
7040
|
+
markerAtMs += 1;
|
|
7041
|
+
}
|
|
7042
|
+
const marker = store.createSkippedRun(loop, new Date(markerAtMs).toISOString(), reason, {
|
|
7043
|
+
daemonLeaseId: opts.daemonLeaseId
|
|
7044
|
+
});
|
|
7045
|
+
const nextRunAt = computeNextAfter(loop.schedule, new Date(run.scheduledFor), finishedAt);
|
|
7046
|
+
store.updateLoop(loop.id, {
|
|
7047
|
+
status: "paused",
|
|
7048
|
+
nextRunAt,
|
|
7049
|
+
retryScheduledFor: undefined
|
|
7050
|
+
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
7051
|
+
opts.onRun?.(marker);
|
|
7052
|
+
}
|
|
5033
7053
|
function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
5034
7054
|
if (run.status === "running")
|
|
5035
7055
|
return;
|
|
@@ -5042,7 +7062,7 @@ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
|
5042
7062
|
if (shouldRetry) {
|
|
5043
7063
|
store.updateLoop(current.id, {
|
|
5044
7064
|
status: "active",
|
|
5045
|
-
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
7065
|
+
nextRunAt: nextAfterRetry(current, run, finishedAt, opts.random),
|
|
5046
7066
|
retryScheduledFor: run.scheduledFor
|
|
5047
7067
|
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
5048
7068
|
return;
|
|
@@ -5051,11 +7071,21 @@ function advanceLoop(store, loop, run, finishedAt, succeeded, opts = {}) {
|
|
|
5051
7071
|
if (deferredRetry) {
|
|
5052
7072
|
store.updateLoop(current.id, {
|
|
5053
7073
|
status: "active",
|
|
5054
|
-
nextRunAt: nextAfterRetry(current, finishedAt),
|
|
7074
|
+
nextRunAt: nextAfterRetry(current, deferredRetry, finishedAt, opts.random),
|
|
5055
7075
|
retryScheduledFor: deferredRetry.scheduledFor
|
|
5056
7076
|
}, { daemonLeaseId: opts.daemonLeaseId });
|
|
5057
7077
|
return;
|
|
5058
7078
|
}
|
|
7079
|
+
if (!succeeded) {
|
|
7080
|
+
const threshold = resolveBreakerThreshold(current, opts.circuitBreakerThreshold);
|
|
7081
|
+
if (threshold > 0) {
|
|
7082
|
+
const failures = consecutiveFailureCount(store, current.id, current.maxAttempts, Math.max(threshold * 4, 50));
|
|
7083
|
+
if (failures >= threshold) {
|
|
7084
|
+
tripCircuitBreaker(store, current, run, finishedAt, failures, opts);
|
|
7085
|
+
return;
|
|
7086
|
+
}
|
|
7087
|
+
}
|
|
7088
|
+
}
|
|
5059
7089
|
const nextRunAt = computeNextAfter(current.schedule, new Date(run.scheduledFor), finishedAt);
|
|
5060
7090
|
store.updateLoop(current.id, {
|
|
5061
7091
|
status: nextRunAt ? "active" : "stopped",
|
|
@@ -5117,6 +7147,14 @@ async function executeClaimedRun(deps) {
|
|
|
5117
7147
|
clearInterval(heartbeat);
|
|
5118
7148
|
}
|
|
5119
7149
|
}
|
|
7150
|
+
function advanceOptions(deps) {
|
|
7151
|
+
return {
|
|
7152
|
+
daemonLeaseId: deps.daemonLeaseId,
|
|
7153
|
+
random: deps.random,
|
|
7154
|
+
circuitBreakerThreshold: deps.circuitBreakerThreshold,
|
|
7155
|
+
onRun: deps.onRun
|
|
7156
|
+
};
|
|
7157
|
+
}
|
|
5120
7158
|
async function runSlot(deps, loop, scheduledFor) {
|
|
5121
7159
|
const now = deps.now?.() ?? new Date;
|
|
5122
7160
|
deps.beforeRun?.(loop, scheduledFor);
|
|
@@ -5131,7 +7169,7 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
5131
7169
|
return;
|
|
5132
7170
|
throw error;
|
|
5133
7171
|
}
|
|
5134
|
-
advanceLoop(deps.store, loop, skipped, now, true,
|
|
7172
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
5135
7173
|
deps.onRun?.(skipped);
|
|
5136
7174
|
return skipped;
|
|
5137
7175
|
}
|
|
@@ -5158,7 +7196,7 @@ async function runSlot(deps, loop, scheduledFor) {
|
|
|
5158
7196
|
daemonLeaseId: deps.daemonLeaseId,
|
|
5159
7197
|
onError: deps.onError
|
|
5160
7198
|
});
|
|
5161
|
-
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded",
|
|
7199
|
+
advanceLoop(deps.store, claim.loop, finalRun, new Date(finalRun.finishedAt ?? new Date), finalRun.status === "succeeded", advanceOptions(deps));
|
|
5162
7200
|
deps.onRun?.(finalRun);
|
|
5163
7201
|
return finalRun;
|
|
5164
7202
|
}
|
|
@@ -5178,7 +7216,7 @@ function claimSlot(deps, loop, scheduledFor) {
|
|
|
5178
7216
|
return;
|
|
5179
7217
|
throw error;
|
|
5180
7218
|
}
|
|
5181
|
-
advanceLoop(deps.store, loop, skipped, now, true,
|
|
7219
|
+
advanceLoop(deps.store, loop, skipped, now, true, advanceOptions(deps));
|
|
5182
7220
|
deps.onRun?.(skipped);
|
|
5183
7221
|
return skipped;
|
|
5184
7222
|
}
|
|
@@ -5196,8 +7234,7 @@ function claimSlot(deps, loop, scheduledFor) {
|
|
|
5196
7234
|
deps.onRun?.(claim.run);
|
|
5197
7235
|
return claim;
|
|
5198
7236
|
}
|
|
5199
|
-
function
|
|
5200
|
-
const now = deps.now?.() ?? new Date;
|
|
7237
|
+
function recoverAndExpire(deps, now) {
|
|
5201
7238
|
const recovered = deps.store.recoverExpiredRunLeases(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
5202
7239
|
const recoveredByLoop = new Map;
|
|
5203
7240
|
for (const run of recovered) {
|
|
@@ -5209,21 +7246,22 @@ function claimDueRuns(deps) {
|
|
|
5209
7246
|
continue;
|
|
5210
7247
|
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
5211
7248
|
if (retryable) {
|
|
5212
|
-
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false,
|
|
5213
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5214
|
-
});
|
|
7249
|
+
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, advanceOptions(deps));
|
|
5215
7250
|
continue;
|
|
5216
7251
|
}
|
|
5217
7252
|
for (const run of runs) {
|
|
5218
7253
|
const current = deps.store.getLoop(run.loopId);
|
|
5219
7254
|
if (current) {
|
|
5220
|
-
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false,
|
|
5221
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5222
|
-
});
|
|
7255
|
+
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, advanceOptions(deps));
|
|
5223
7256
|
}
|
|
5224
7257
|
}
|
|
5225
7258
|
}
|
|
5226
7259
|
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
7260
|
+
return { recovered, expired };
|
|
7261
|
+
}
|
|
7262
|
+
function claimDueRuns(deps) {
|
|
7263
|
+
const now = deps.now?.() ?? new Date;
|
|
7264
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
5227
7265
|
const claims = [];
|
|
5228
7266
|
const claimed = [];
|
|
5229
7267
|
const skipped = [];
|
|
@@ -5231,11 +7269,14 @@ function claimDueRuns(deps) {
|
|
|
5231
7269
|
if (maxClaims === 0)
|
|
5232
7270
|
return { claims, claimed, completed: [], skipped, recovered, expired };
|
|
5233
7271
|
for (const loop of deps.store.dueLoops(now)) {
|
|
5234
|
-
if (claims.length
|
|
7272
|
+
if (claims.length >= maxClaims)
|
|
5235
7273
|
break;
|
|
5236
7274
|
const plan = dueSlots(loop, now);
|
|
7275
|
+
let loopSkips = 0;
|
|
5237
7276
|
for (const slot of plan.slots) {
|
|
5238
|
-
if (claims.length
|
|
7277
|
+
if (claims.length >= maxClaims)
|
|
7278
|
+
break;
|
|
7279
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
5239
7280
|
break;
|
|
5240
7281
|
const run = claimSlot(deps, loop, slot);
|
|
5241
7282
|
if (!run)
|
|
@@ -5245,6 +7286,7 @@ function claimDueRuns(deps) {
|
|
|
5245
7286
|
claimed.push(run.run);
|
|
5246
7287
|
} else if (run.status === "skipped") {
|
|
5247
7288
|
skipped.push(run);
|
|
7289
|
+
loopSkips += 1;
|
|
5248
7290
|
}
|
|
5249
7291
|
}
|
|
5250
7292
|
}
|
|
@@ -5252,46 +7294,25 @@ function claimDueRuns(deps) {
|
|
|
5252
7294
|
}
|
|
5253
7295
|
async function tick(deps) {
|
|
5254
7296
|
const now = deps.now?.() ?? new Date;
|
|
5255
|
-
const recovered = deps
|
|
5256
|
-
const recoveredByLoop = new Map;
|
|
5257
|
-
for (const run of recovered) {
|
|
5258
|
-
recoveredByLoop.set(run.loopId, [...recoveredByLoop.get(run.loopId) ?? [], run]);
|
|
5259
|
-
}
|
|
5260
|
-
for (const runs of recoveredByLoop.values()) {
|
|
5261
|
-
const loop = deps.store.getLoop(runs[0].loopId);
|
|
5262
|
-
if (!loop)
|
|
5263
|
-
continue;
|
|
5264
|
-
const retryable = runs.filter((run) => run.attempt < loop.maxAttempts).sort((a, b) => new Date(a.scheduledFor).getTime() - new Date(b.scheduledFor).getTime())[0];
|
|
5265
|
-
if (retryable) {
|
|
5266
|
-
advanceLoop(deps.store, loop, retryable, new Date(retryable.finishedAt ?? now), false, {
|
|
5267
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5268
|
-
});
|
|
5269
|
-
continue;
|
|
5270
|
-
}
|
|
5271
|
-
for (const run of runs) {
|
|
5272
|
-
const current = deps.store.getLoop(run.loopId);
|
|
5273
|
-
if (current) {
|
|
5274
|
-
advanceLoop(deps.store, current, run, new Date(run.finishedAt ?? now), false, {
|
|
5275
|
-
daemonLeaseId: deps.daemonLeaseId
|
|
5276
|
-
});
|
|
5277
|
-
}
|
|
5278
|
-
}
|
|
5279
|
-
}
|
|
5280
|
-
const expired = deps.store.expireLoops(now, { daemonLeaseId: deps.daemonLeaseId });
|
|
7297
|
+
const { recovered, expired } = recoverAndExpire(deps, now);
|
|
5281
7298
|
const claimed = [];
|
|
5282
7299
|
const completed = [];
|
|
5283
7300
|
const skipped = [];
|
|
5284
7301
|
for (const loop of deps.store.dueLoops(now)) {
|
|
5285
7302
|
const plan = dueSlots(loop, now);
|
|
7303
|
+
let loopSkips = 0;
|
|
5286
7304
|
for (const slot of plan.slots) {
|
|
7305
|
+
if (loopSkips >= MAX_SKIPS_PER_LOOP_PER_TICK)
|
|
7306
|
+
break;
|
|
5287
7307
|
const run = await runSlot(deps, loop, slot);
|
|
5288
7308
|
if (!run)
|
|
5289
7309
|
continue;
|
|
5290
7310
|
if (run.status === "running")
|
|
5291
7311
|
claimed.push(run);
|
|
5292
|
-
else if (run.status === "skipped")
|
|
7312
|
+
else if (run.status === "skipped") {
|
|
5293
7313
|
skipped.push(run);
|
|
5294
|
-
|
|
7314
|
+
loopSkips += 1;
|
|
7315
|
+
} else
|
|
5295
7316
|
completed.push(run);
|
|
5296
7317
|
if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts)
|
|
5297
7318
|
break;
|
|
@@ -5313,29 +7334,25 @@ class LoopsClient {
|
|
|
5313
7334
|
create(input) {
|
|
5314
7335
|
return this.store.createLoop(input);
|
|
5315
7336
|
}
|
|
5316
|
-
list() {
|
|
5317
|
-
return this.store.listLoops(
|
|
7337
|
+
list(filters = {}) {
|
|
7338
|
+
return this.store.listLoops({
|
|
7339
|
+
status: filters.status,
|
|
7340
|
+
limit: filters.limit,
|
|
7341
|
+
includeArchived: filters.includeArchived,
|
|
7342
|
+
archived: filters.archivedOnly
|
|
7343
|
+
});
|
|
5318
7344
|
}
|
|
5319
7345
|
get(idOrName) {
|
|
5320
7346
|
return this.store.requireLoop(idOrName);
|
|
5321
7347
|
}
|
|
5322
7348
|
pause(idOrName) {
|
|
5323
|
-
|
|
5324
|
-
if (loop.archivedAt)
|
|
5325
|
-
throw new Error(`loop is archived; unarchive it before pausing: ${idOrName}`);
|
|
5326
|
-
return this.store.updateLoop(loop.id, { status: "paused" });
|
|
7349
|
+
return this.store.updateLoop(this.get(idOrName).id, { status: "paused" });
|
|
5327
7350
|
}
|
|
5328
7351
|
resume(idOrName) {
|
|
5329
|
-
|
|
5330
|
-
if (loop.archivedAt)
|
|
5331
|
-
throw new Error(`loop is archived; unarchive it before resuming: ${idOrName}`);
|
|
5332
|
-
return this.store.updateLoop(loop.id, { status: "active" });
|
|
7352
|
+
return this.store.updateLoop(this.get(idOrName).id, { status: "active" });
|
|
5333
7353
|
}
|
|
5334
7354
|
stop(idOrName) {
|
|
5335
|
-
|
|
5336
|
-
if (loop.archivedAt)
|
|
5337
|
-
throw new Error(`loop is archived; unarchive it before stopping: ${idOrName}`);
|
|
5338
|
-
return this.store.updateLoop(loop.id, { status: "stopped", nextRunAt: undefined });
|
|
7355
|
+
return this.store.updateLoop(this.get(idOrName).id, { status: "stopped", nextRunAt: undefined });
|
|
5339
7356
|
}
|
|
5340
7357
|
archive(idOrName) {
|
|
5341
7358
|
return this.store.archiveLoop(idOrName);
|
|
@@ -5346,8 +7363,24 @@ class LoopsClient {
|
|
|
5346
7363
|
delete(idOrName) {
|
|
5347
7364
|
return this.store.deleteLoop(idOrName);
|
|
5348
7365
|
}
|
|
5349
|
-
runs(
|
|
5350
|
-
|
|
7366
|
+
runs(idOrName, filters = {}) {
|
|
7367
|
+
let loopId;
|
|
7368
|
+
if (idOrName) {
|
|
7369
|
+
try {
|
|
7370
|
+
loopId = this.get(idOrName).id;
|
|
7371
|
+
} catch (error) {
|
|
7372
|
+
if (error instanceof LoopNotFoundError)
|
|
7373
|
+
return [];
|
|
7374
|
+
throw error;
|
|
7375
|
+
}
|
|
7376
|
+
}
|
|
7377
|
+
return this.store.listRuns({ loopId, status: filters.status, limit: filters.limit });
|
|
7378
|
+
}
|
|
7379
|
+
doctor() {
|
|
7380
|
+
return runDoctor(this.store);
|
|
7381
|
+
}
|
|
7382
|
+
health(opts = {}) {
|
|
7383
|
+
return buildHealthReport(this.store, opts);
|
|
5351
7384
|
}
|
|
5352
7385
|
goal(idOrName) {
|
|
5353
7386
|
const goal = this.store.getGoal(idOrName) ?? this.store.findGoalByLoop(idOrName) ?? this.store.findGoalByRunId(idOrName);
|
|
@@ -5360,28 +7393,8 @@ class LoopsClient {
|
|
|
5360
7393
|
return tick({ store: this.store, runnerId: this.runnerId });
|
|
5361
7394
|
}
|
|
5362
7395
|
async runNow(idOrName) {
|
|
5363
|
-
const
|
|
5364
|
-
|
|
5365
|
-
throw new Error(`loop is archived; unarchive it before running: ${idOrName}`);
|
|
5366
|
-
const now = new Date;
|
|
5367
|
-
let scheduledFor = manualRunScheduledFor(loop, now);
|
|
5368
|
-
let shouldAdvance = shouldAdvanceManualRun(loop, scheduledFor, now);
|
|
5369
|
-
let claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
|
|
5370
|
-
if (!claim && shouldAdvance) {
|
|
5371
|
-
const existing = this.store.getRunBySlot(loop.id, scheduledFor);
|
|
5372
|
-
if (existing && existing.status !== "running") {
|
|
5373
|
-
scheduledFor = now.toISOString();
|
|
5374
|
-
shouldAdvance = false;
|
|
5375
|
-
claim = this.store.claimRun(loop, scheduledFor, this.runnerId, now);
|
|
5376
|
-
}
|
|
5377
|
-
}
|
|
5378
|
-
if (!claim)
|
|
5379
|
-
throw new Error(`could not claim manual run for ${idOrName}`);
|
|
5380
|
-
const run = await executeClaimedRun({ store: this.store, runnerId: this.runnerId, loop: claim.loop, run: claim.run });
|
|
5381
|
-
if (shouldAdvance) {
|
|
5382
|
-
advanceLoop(this.store, claim.loop, run, new Date(run.finishedAt ?? new Date), run.status === "succeeded");
|
|
5383
|
-
}
|
|
5384
|
-
return run;
|
|
7396
|
+
const result = await runLoopNow({ store: this.store, idOrName, runnerId: this.runnerId });
|
|
7397
|
+
return result.run;
|
|
5385
7398
|
}
|
|
5386
7399
|
close() {
|
|
5387
7400
|
if (this.ownStore)
|