@hasna/loops 0.3.59 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +247 -0
- package/LICENSE +197 -13
- package/README.md +28 -60
- package/dist/cli/index.js +7269 -5776
- package/dist/daemon/control.d.ts +21 -1
- package/dist/daemon/daemon.d.ts +5 -0
- package/dist/daemon/index.js +2985 -1072
- package/dist/daemon/install.d.ts +1 -1
- package/dist/index.d.ts +21 -8
- package/dist/index.js +5906 -4580
- package/dist/lib/accounts.d.ts +6 -1
- package/dist/lib/agent-adapter.d.ts +74 -0
- package/dist/lib/backup.d.ts +25 -0
- package/dist/lib/errors.d.ts +21 -0
- package/dist/lib/executor.d.ts +10 -1
- package/dist/lib/goal/metadata.d.ts +16 -0
- package/dist/lib/goal/model-factory.d.ts +1 -0
- package/dist/lib/goal/prompts.d.ts +3 -1
- package/dist/lib/goal/types.d.ts +3 -0
- package/dist/lib/health.d.ts +1 -1
- package/dist/lib/ids.d.ts +8 -1
- package/dist/lib/machines.d.ts +6 -0
- package/dist/lib/process-identity.d.ts +16 -0
- package/dist/lib/{schedule.d.ts → recurrence.d.ts} +1 -0
- package/dist/lib/redact.d.ts +21 -0
- package/dist/lib/route/cursors.d.ts +18 -0
- package/dist/lib/route/drain.d.ts +6 -0
- package/dist/lib/route/fields.d.ts +27 -0
- package/dist/lib/route/gates.d.ts +26 -0
- package/dist/lib/route/index.d.ts +18 -0
- package/dist/lib/route/options.d.ts +31 -0
- package/dist/lib/route/parse.d.ts +8 -0
- package/dist/lib/route/pr-review.d.ts +11 -0
- package/dist/lib/route/provider.d.ts +41 -0
- package/dist/lib/route/route-event.d.ts +23 -0
- package/dist/lib/route/route-tasks.d.ts +75 -0
- package/dist/lib/route/throttle.d.ts +47 -0
- package/dist/lib/route/todos-cli.d.ts +21 -0
- package/dist/lib/route/types.d.ts +88 -0
- package/dist/lib/run-artifacts.d.ts +17 -2
- package/dist/lib/run-envelope.d.ts +41 -0
- package/dist/lib/scheduler.d.ts +78 -2
- package/dist/lib/store.d.ts +63 -0
- package/dist/lib/store.js +1011 -266
- package/dist/lib/template-kit.d.ts +70 -0
- package/dist/lib/templates-custom.d.ts +34 -0
- package/dist/lib/templates.d.ts +13 -81
- package/dist/lib/workflow-runner.d.ts +2 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.js +3313 -1165
- package/dist/sdk/index.d.ts +29 -3
- package/dist/sdk/index.js +2910 -897
- package/dist/test-helpers.d.ts +37 -0
- package/dist/types.d.ts +2 -0
- package/docs/USAGE.md +40 -18
- package/package.json +6 -3
package/dist/lib/store.js
CHANGED
|
@@ -1,14 +1,50 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// src/lib/store.ts
|
|
3
3
|
import { Database } from "bun:sqlite";
|
|
4
|
-
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync } from "fs";
|
|
4
|
+
import { chmodSync, existsSync, mkdirSync as mkdirSync3, mkdtempSync, rmSync as rmSync2 } from "fs";
|
|
5
5
|
import { tmpdir } from "os";
|
|
6
|
-
import { dirname, join as join3 } from "path";
|
|
6
|
+
import { basename as basename2, dirname as dirname2, join as join3 } from "path";
|
|
7
|
+
|
|
8
|
+
// src/lib/errors.ts
|
|
9
|
+
class CodedError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
constructor(code, message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = new.target.name;
|
|
14
|
+
this.code = code;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
class LoopNotFoundError extends CodedError {
|
|
19
|
+
constructor(idOrName) {
|
|
20
|
+
super("LOOP_NOT_FOUND", `loop not found: ${idOrName}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
class LoopArchivedError extends CodedError {
|
|
25
|
+
constructor(idOrName) {
|
|
26
|
+
super("LOOP_ARCHIVED", `loop is archived: ${idOrName}; unarchive it before modifying`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
class AmbiguousNameError extends CodedError {
|
|
31
|
+
constructor(name) {
|
|
32
|
+
super("AMBIGUOUS_NAME", `ambiguous loop name: ${name}; use a loop id`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class ValidationError extends CodedError {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super("VALIDATION_ERROR", message);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
7
41
|
|
|
8
42
|
// src/lib/ids.ts
|
|
9
43
|
import { randomBytes } from "crypto";
|
|
10
|
-
|
|
11
|
-
|
|
44
|
+
var TIME_HEX_LENGTH = 12;
|
|
45
|
+
function genId() {
|
|
46
|
+
const time = Date.now().toString(16).padStart(TIME_HEX_LENGTH, "0");
|
|
47
|
+
return `${time}${randomBytes(10).toString("hex")}`;
|
|
12
48
|
}
|
|
13
49
|
function nowIso() {
|
|
14
50
|
return new Date().toISOString();
|
|
@@ -42,7 +78,166 @@ function launchdPlistPath() {
|
|
|
42
78
|
return join(homedir(), "Library", "LaunchAgents", "com.hasna.loops.daemon.plist");
|
|
43
79
|
}
|
|
44
80
|
|
|
45
|
-
// src/lib/
|
|
81
|
+
// src/lib/process-identity.ts
|
|
82
|
+
import { readFileSync } from "fs";
|
|
83
|
+
import { spawnSync } from "child_process";
|
|
84
|
+
var START_TIME_TOLERANCE_MS = 5000;
|
|
85
|
+
var clockTicksCache;
|
|
86
|
+
function clockTicksPerSecond() {
|
|
87
|
+
if (clockTicksCache !== undefined)
|
|
88
|
+
return clockTicksCache;
|
|
89
|
+
try {
|
|
90
|
+
const run = spawnSync("getconf", ["CLK_TCK"], { encoding: "utf8" });
|
|
91
|
+
const value = Number(run.stdout.trim());
|
|
92
|
+
clockTicksCache = run.status === 0 && Number.isFinite(value) && value > 0 ? value : 100;
|
|
93
|
+
} catch {
|
|
94
|
+
clockTicksCache = 100;
|
|
95
|
+
}
|
|
96
|
+
return clockTicksCache;
|
|
97
|
+
}
|
|
98
|
+
var bootTimeCache;
|
|
99
|
+
var bootTimeResolved = false;
|
|
100
|
+
function bootTimeMs() {
|
|
101
|
+
if (bootTimeResolved)
|
|
102
|
+
return bootTimeCache;
|
|
103
|
+
bootTimeResolved = true;
|
|
104
|
+
try {
|
|
105
|
+
const match = readFileSync("/proc/stat", "utf8").match(/^btime (\d+)$/m);
|
|
106
|
+
bootTimeCache = match ? Number(match[1]) * 1000 : undefined;
|
|
107
|
+
} catch {
|
|
108
|
+
bootTimeCache = undefined;
|
|
109
|
+
}
|
|
110
|
+
return bootTimeCache;
|
|
111
|
+
}
|
|
112
|
+
function procStatFields(path) {
|
|
113
|
+
try {
|
|
114
|
+
const stat = readFileSync(path, "utf8");
|
|
115
|
+
const closeParen = stat.lastIndexOf(")");
|
|
116
|
+
if (closeParen < 0)
|
|
117
|
+
return;
|
|
118
|
+
return stat.slice(closeParen + 2).split(" ");
|
|
119
|
+
} catch {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function processStartTimeMs(pid) {
|
|
124
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
125
|
+
return;
|
|
126
|
+
if (process.platform === "linux") {
|
|
127
|
+
const fields = procStatFields(`/proc/${pid}/stat`);
|
|
128
|
+
const startTicks = fields ? Number(fields[19]) : Number.NaN;
|
|
129
|
+
const boot = bootTimeMs();
|
|
130
|
+
if (Number.isFinite(startTicks) && boot !== undefined) {
|
|
131
|
+
return boot + Math.round(startTicks / clockTicksPerSecond() * 1000);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
try {
|
|
135
|
+
const run = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" });
|
|
136
|
+
if (run.status === 0) {
|
|
137
|
+
const parsed = Date.parse(run.stdout.trim());
|
|
138
|
+
if (Number.isFinite(parsed))
|
|
139
|
+
return parsed;
|
|
140
|
+
}
|
|
141
|
+
} catch {}
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
function sameProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
145
|
+
if (recorded === undefined || actualMs === undefined)
|
|
146
|
+
return true;
|
|
147
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
148
|
+
if (!Number.isFinite(recordedMs))
|
|
149
|
+
return true;
|
|
150
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
151
|
+
}
|
|
152
|
+
function verifiedProcessStart(recorded, actualMs, toleranceMs = START_TIME_TOLERANCE_MS) {
|
|
153
|
+
if (recorded === undefined || actualMs === undefined)
|
|
154
|
+
return false;
|
|
155
|
+
const recordedMs = typeof recorded === "number" ? recorded : Date.parse(recorded);
|
|
156
|
+
if (!Number.isFinite(recordedMs))
|
|
157
|
+
return false;
|
|
158
|
+
return Math.abs(recordedMs - actualMs) <= toleranceMs;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/lib/redact.ts
|
|
162
|
+
var SCRUBBED = "[SCRUBBED]";
|
|
163
|
+
var TOKEN_PATTERNS = [
|
|
164
|
+
/\bsk-ant-[A-Za-z0-9_-]{8,}/g,
|
|
165
|
+
/\bsk-proj-[A-Za-z0-9_-]{8,}/g,
|
|
166
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
167
|
+
/\bghp_[A-Za-z0-9]{16,}\b/g,
|
|
168
|
+
/\bgithub_pat_[A-Za-z0-9_]{16,}\b/g,
|
|
169
|
+
/\bxox[a-z]-[A-Za-z0-9-]{8,}/g,
|
|
170
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{10,}\b/g
|
|
171
|
+
];
|
|
172
|
+
var AUTHORIZATION_PATTERN = /(\bAuthorization(?:\\?["'])?\s*[:=]\s*(?:\\?["'])?(?:Bearer|Basic)\s+)[A-Za-z0-9._~+/=-]{16,}/gi;
|
|
173
|
+
var PEM_PATTERN = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|$)/g;
|
|
174
|
+
var ASSIGNMENT_PATTERN = /([A-Za-z0-9_.-]{0,64}(?:key|token|secret|passwd|password|passphrase|credentials?)(?:\\?["'])?\s*[:=]\s*)(\\"([^"\\\r\n]{12,})\\"|"((?:[^"\\\r\n]|\\[^\r\n]){12,})"|'([^'\\\r\n]{12,})'|([A-Za-z0-9+/_.=-]{16,}))/gi;
|
|
175
|
+
var BENIGN_KEY_NAME_PATTERN = /(?:^|[._-])(?:idempotency|dedupe|route)[_-]?key$/i;
|
|
176
|
+
var ASSIGNMENT_HINTS = ["key", "token", "secret", "pass", "credential"];
|
|
177
|
+
function mayContainAssignment(text) {
|
|
178
|
+
const lowered = text.toLowerCase();
|
|
179
|
+
return ASSIGNMENT_HINTS.some((hint) => lowered.includes(hint));
|
|
180
|
+
}
|
|
181
|
+
function shannonEntropy(value) {
|
|
182
|
+
const counts = new Map;
|
|
183
|
+
for (const char of value)
|
|
184
|
+
counts.set(char, (counts.get(char) ?? 0) + 1);
|
|
185
|
+
let entropy = 0;
|
|
186
|
+
for (const count of counts.values()) {
|
|
187
|
+
const probability = count / value.length;
|
|
188
|
+
entropy -= probability * Math.log2(probability);
|
|
189
|
+
}
|
|
190
|
+
return entropy;
|
|
191
|
+
}
|
|
192
|
+
function isLikelySecretValue(value) {
|
|
193
|
+
if (value.length < 12)
|
|
194
|
+
return false;
|
|
195
|
+
if (value.includes(SCRUBBED))
|
|
196
|
+
return false;
|
|
197
|
+
if (value.startsWith("/") || value.startsWith("~") || value.startsWith("./"))
|
|
198
|
+
return false;
|
|
199
|
+
return shannonEntropy(value) >= 3.2;
|
|
200
|
+
}
|
|
201
|
+
function scrubSecrets(text) {
|
|
202
|
+
if (!text)
|
|
203
|
+
return text;
|
|
204
|
+
let scrubbed = text;
|
|
205
|
+
for (const pattern of TOKEN_PATTERNS)
|
|
206
|
+
scrubbed = scrubbed.replace(pattern, SCRUBBED);
|
|
207
|
+
scrubbed = scrubbed.replace(AUTHORIZATION_PATTERN, `$1${SCRUBBED}`);
|
|
208
|
+
scrubbed = scrubbed.replace(PEM_PATTERN, SCRUBBED);
|
|
209
|
+
if (!mayContainAssignment(scrubbed))
|
|
210
|
+
return scrubbed;
|
|
211
|
+
scrubbed = scrubbed.replace(ASSIGNMENT_PATTERN, (match, prefix, quotedValue, escapedQuoted, doubleQuoted, singleQuoted, bare) => {
|
|
212
|
+
const value = escapedQuoted ?? doubleQuoted ?? singleQuoted ?? bare ?? "";
|
|
213
|
+
const keyName = /^[A-Za-z0-9_.-]*/.exec(prefix)?.[0] ?? "";
|
|
214
|
+
if (BENIGN_KEY_NAME_PATTERN.test(keyName))
|
|
215
|
+
return match;
|
|
216
|
+
if (!isLikelySecretValue(value))
|
|
217
|
+
return match;
|
|
218
|
+
const quote = quotedValue.startsWith("\\\"") ? "\\\"" : quotedValue.startsWith('"') ? '"' : quotedValue.startsWith("'") ? "'" : "";
|
|
219
|
+
return `${prefix}${quote}${SCRUBBED}${quote}`;
|
|
220
|
+
});
|
|
221
|
+
return scrubbed;
|
|
222
|
+
}
|
|
223
|
+
function scrubSecretsDeep(value) {
|
|
224
|
+
if (typeof value === "string")
|
|
225
|
+
return scrubSecrets(value);
|
|
226
|
+
if (Array.isArray(value))
|
|
227
|
+
return value.map((entry) => scrubSecretsDeep(entry));
|
|
228
|
+
if (value !== null && typeof value === "object") {
|
|
229
|
+
const toJSON = value.toJSON;
|
|
230
|
+
if (typeof toJSON === "function")
|
|
231
|
+
return scrubSecretsDeep(toJSON.call(value));
|
|
232
|
+
const scrubbed = {};
|
|
233
|
+
for (const [key, entry] of Object.entries(value))
|
|
234
|
+
scrubbed[key] = scrubSecretsDeep(entry);
|
|
235
|
+
return scrubbed;
|
|
236
|
+
}
|
|
237
|
+
return value;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// src/lib/recurrence.ts
|
|
46
241
|
function assertDate(value, label) {
|
|
47
242
|
const date = new Date(value);
|
|
48
243
|
if (Number.isNaN(date.getTime()))
|
|
@@ -79,7 +274,20 @@ function parseField(expr, min, max) {
|
|
|
79
274
|
}
|
|
80
275
|
return out;
|
|
81
276
|
}
|
|
277
|
+
var MINUTE_MS = 60000;
|
|
278
|
+
var PARSED_CRON_CACHE_LIMIT = 256;
|
|
279
|
+
var parsedCronCache = new Map;
|
|
280
|
+
var emittedScheduleWarnings = new Set;
|
|
281
|
+
function warnOnce(key, message) {
|
|
282
|
+
if (emittedScheduleWarnings.has(key))
|
|
283
|
+
return;
|
|
284
|
+
emittedScheduleWarnings.add(key);
|
|
285
|
+
console.warn(`[open-loops] WARN ${message}`);
|
|
286
|
+
}
|
|
82
287
|
function parseCron(expr) {
|
|
288
|
+
const cached = parsedCronCache.get(expr);
|
|
289
|
+
if (cached)
|
|
290
|
+
return cached;
|
|
83
291
|
const fields = expr.trim().split(/\s+/);
|
|
84
292
|
if (fields.length !== 5)
|
|
85
293
|
throw new Error(`cron must have 5 fields, got ${fields.length}: "${expr}"`);
|
|
@@ -89,7 +297,7 @@ function parseCron(expr) {
|
|
|
89
297
|
dowSet.delete(7);
|
|
90
298
|
dowSet.add(0);
|
|
91
299
|
}
|
|
92
|
-
|
|
300
|
+
const parsed = {
|
|
93
301
|
minute: parseField(minute, 0, 59),
|
|
94
302
|
hour: parseField(hour, 0, 23),
|
|
95
303
|
dom: parseField(dom, 1, 31),
|
|
@@ -98,6 +306,13 @@ function parseCron(expr) {
|
|
|
98
306
|
domRestricted: dom !== "*",
|
|
99
307
|
dowRestricted: dow !== "*"
|
|
100
308
|
};
|
|
309
|
+
if (parsedCronCache.size >= PARSED_CRON_CACHE_LIMIT) {
|
|
310
|
+
const oldest = parsedCronCache.keys().next().value;
|
|
311
|
+
if (oldest !== undefined)
|
|
312
|
+
parsedCronCache.delete(oldest);
|
|
313
|
+
}
|
|
314
|
+
parsedCronCache.set(expr, parsed);
|
|
315
|
+
return parsed;
|
|
101
316
|
}
|
|
102
317
|
function cronMatches(cron, date) {
|
|
103
318
|
if (!cron.minute.has(date.getMinutes()))
|
|
@@ -171,14 +386,59 @@ function latestIntervalSlot(first, now, everyMs) {
|
|
|
171
386
|
const steps = Math.floor((now.getTime() - first.getTime()) / everyMs);
|
|
172
387
|
return new Date(first.getTime() + steps * everyMs);
|
|
173
388
|
}
|
|
389
|
+
function floorToMinute(date) {
|
|
390
|
+
const floored = new Date(date.getTime());
|
|
391
|
+
floored.setSeconds(0, 0);
|
|
392
|
+
return floored;
|
|
393
|
+
}
|
|
394
|
+
var LATEST_CRON_SCAN_LIMIT_MINUTES = 366 * 24 * 60;
|
|
395
|
+
function latestDailyCronSlot(cron, now) {
|
|
396
|
+
const hoursDesc = [...cron.hour].sort((a, b) => b - a);
|
|
397
|
+
const minutesDesc = [...cron.minute].sort((a, b) => b - a);
|
|
398
|
+
const candidate = floorToMinute(now);
|
|
399
|
+
for (let dayOffset = 0;dayOffset < 2; dayOffset += 1) {
|
|
400
|
+
const isToday = dayOffset === 0;
|
|
401
|
+
for (const hour of hoursDesc) {
|
|
402
|
+
if (isToday && hour > now.getHours())
|
|
403
|
+
continue;
|
|
404
|
+
const minuteCap = isToday && hour === now.getHours() ? now.getMinutes() : 59;
|
|
405
|
+
const minute = minutesDesc.find((value) => value <= minuteCap);
|
|
406
|
+
if (minute === undefined)
|
|
407
|
+
continue;
|
|
408
|
+
candidate.setHours(hour, minute, 0, 0);
|
|
409
|
+
return candidate;
|
|
410
|
+
}
|
|
411
|
+
candidate.setDate(candidate.getDate() - 1);
|
|
412
|
+
}
|
|
413
|
+
throw new Error("unreachable: daily cron pattern must match within two days");
|
|
414
|
+
}
|
|
174
415
|
function latestCronSlot(first, now, expression) {
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
416
|
+
if (first.getTime() > now.getTime())
|
|
417
|
+
return first;
|
|
418
|
+
const cron = parseCron(expression);
|
|
419
|
+
if (!cron.domRestricted && !cron.dowRestricted && cron.month.size === 12) {
|
|
420
|
+
const latest = latestDailyCronSlot(cron, now);
|
|
421
|
+
return latest.getTime() >= first.getTime() ? latest : first;
|
|
422
|
+
}
|
|
423
|
+
const scanFloorMs = Math.max(first.getTime(), now.getTime() - LATEST_CRON_SCAN_LIMIT_MINUTES * MINUTE_MS);
|
|
424
|
+
const cursor = floorToMinute(now);
|
|
425
|
+
while (cursor.getTime() >= scanFloorMs) {
|
|
426
|
+
if (cronMatches(cron, cursor))
|
|
427
|
+
return cursor;
|
|
428
|
+
cursor.setMinutes(cursor.getMinutes() - 1);
|
|
429
|
+
}
|
|
430
|
+
if (first.getTime() < scanFloorMs) {
|
|
431
|
+
warnOnce(`latest-cron:${expression}`, `latestCronSlot: no match for "${expression}" within the ${LATEST_CRON_SCAN_LIMIT_MINUTES}-minute scan window; using the stored next run as the latest slot`);
|
|
432
|
+
}
|
|
433
|
+
return first;
|
|
434
|
+
}
|
|
435
|
+
var MAX_CATCH_UP_SLOTS = 1000;
|
|
436
|
+
function catchUpSlotLimit(loop) {
|
|
437
|
+
if (loop.catchUpLimit > MAX_CATCH_UP_SLOTS) {
|
|
438
|
+
warnOnce(`catch-up-limit:${loop.id}`, `dueSlots: loop ${loop.id} catchUpLimit ${loop.catchUpLimit} exceeds the per-plan cap; using ${MAX_CATCH_UP_SLOTS}`);
|
|
439
|
+
return MAX_CATCH_UP_SLOTS;
|
|
181
440
|
}
|
|
441
|
+
return loop.catchUpLimit;
|
|
182
442
|
}
|
|
183
443
|
function dueSlots(loop, now) {
|
|
184
444
|
if (!loop.nextRunAt || loop.status !== "active")
|
|
@@ -197,9 +457,10 @@ function dueSlots(loop, now) {
|
|
|
197
457
|
return { slots: [next.toISOString()] };
|
|
198
458
|
case "interval": {
|
|
199
459
|
if (catchUp === "all") {
|
|
460
|
+
const limit = catchUpSlotLimit(loop);
|
|
200
461
|
const slots = [];
|
|
201
462
|
let cursor = next;
|
|
202
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
463
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
203
464
|
slots.push(cursor.toISOString());
|
|
204
465
|
cursor = new Date(cursor.getTime() + loop.schedule.everyMs);
|
|
205
466
|
}
|
|
@@ -211,9 +472,10 @@ function dueSlots(loop, now) {
|
|
|
211
472
|
}
|
|
212
473
|
case "cron": {
|
|
213
474
|
if (catchUp === "all") {
|
|
475
|
+
const limit = catchUpSlotLimit(loop);
|
|
214
476
|
const slots = [];
|
|
215
477
|
let cursor = next;
|
|
216
|
-
while (cursor.getTime() <= now.getTime() && slots.length <
|
|
478
|
+
while (cursor.getTime() <= now.getTime() && slots.length < limit) {
|
|
217
479
|
slots.push(cursor.toISOString());
|
|
218
480
|
cursor = nextCronRun(loop.schedule.expression, cursor);
|
|
219
481
|
}
|
|
@@ -312,8 +574,311 @@ function updateReadyFlags(nodes, planStatus) {
|
|
|
312
574
|
}
|
|
313
575
|
|
|
314
576
|
// src/lib/workflow-spec.ts
|
|
315
|
-
import { readFileSync } from "fs";
|
|
577
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
316
578
|
import { isAbsolute, resolve } from "path";
|
|
579
|
+
|
|
580
|
+
// src/lib/agent-adapter.ts
|
|
581
|
+
import { spawn } from "child_process";
|
|
582
|
+
var UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS = new Set([
|
|
583
|
+
"e",
|
|
584
|
+
"exec",
|
|
585
|
+
"agent",
|
|
586
|
+
"start",
|
|
587
|
+
"--ephemeral",
|
|
588
|
+
"--ignore-rules",
|
|
589
|
+
"--skip-git-repo-check",
|
|
590
|
+
"--json",
|
|
591
|
+
"--output-last-message",
|
|
592
|
+
"-o",
|
|
593
|
+
"--output-schema",
|
|
594
|
+
"--dangerously-bypass-approvals-and-sandbox"
|
|
595
|
+
]);
|
|
596
|
+
var CODEX_LIKE_SANDBOXES = ["read-only", "workspace-write", "danger-full-access"];
|
|
597
|
+
var CURSOR_SANDBOXES = ["enabled", "disabled"];
|
|
598
|
+
var PERMISSION_MODES = ["default", "plan", "auto", "bypass"];
|
|
599
|
+
function assertOptionalNonEmptyString(value, label) {
|
|
600
|
+
if (value === undefined)
|
|
601
|
+
return;
|
|
602
|
+
if (typeof value !== "string" || value.trim() === "")
|
|
603
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
604
|
+
}
|
|
605
|
+
function validateAgentOptions(target, label, capabilities) {
|
|
606
|
+
const provider = target.provider;
|
|
607
|
+
if (typeof target.prompt !== "string" || target.prompt.trim() === "") {
|
|
608
|
+
throw new Error(`${label}.prompt must be a non-empty string`);
|
|
609
|
+
}
|
|
610
|
+
assertOptionalNonEmptyString(target.model, `${label}.model`);
|
|
611
|
+
assertOptionalNonEmptyString(target.variant, `${label}.variant`);
|
|
612
|
+
assertOptionalNonEmptyString(target.agent, `${label}.agent`);
|
|
613
|
+
assertOptionalNonEmptyString(target.authProfile, `${label}.authProfile`);
|
|
614
|
+
assertOptionalNonEmptyString(target.configIsolation, `${label}.configIsolation`);
|
|
615
|
+
if (target.configIsolation !== undefined && target.configIsolation !== "safe" && target.configIsolation !== "none") {
|
|
616
|
+
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
617
|
+
}
|
|
618
|
+
if (target.authProfile !== undefined && provider !== "codewith") {
|
|
619
|
+
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
620
|
+
}
|
|
621
|
+
if (provider === "opencode" && (typeof target.model !== "string" || target.model.trim() === "")) {
|
|
622
|
+
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
623
|
+
}
|
|
624
|
+
if (provider === "cursor" && target.variant !== undefined) {
|
|
625
|
+
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
626
|
+
}
|
|
627
|
+
if (provider === "codex" && target.agent !== undefined) {
|
|
628
|
+
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
629
|
+
}
|
|
630
|
+
if (provider === "codewith" && target.agent !== undefined) {
|
|
631
|
+
throw new Error(`${label}.agent is not supported for provider codewith durable background-agent execution`);
|
|
632
|
+
}
|
|
633
|
+
if (provider === "codewith") {
|
|
634
|
+
const unsafe = target.extraArgs?.find((arg) => UNSAFE_CODEWITH_DURABLE_EXTRA_ARGS.has(arg));
|
|
635
|
+
if (unsafe) {
|
|
636
|
+
throw new Error(`${label}.extraArgs cannot include ${unsafe}; codewith agent steps use durable agent start, not exec/ephemeral flags`);
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (target.addDirs?.length && !["codewith", "codex"].includes(provider)) {
|
|
640
|
+
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
641
|
+
}
|
|
642
|
+
if (target.permissionMode !== undefined) {
|
|
643
|
+
if (!PERMISSION_MODES.includes(target.permissionMode)) {
|
|
644
|
+
throw new Error(`${label}.permissionMode must be one of ${PERMISSION_MODES.join(", ")}`);
|
|
645
|
+
}
|
|
646
|
+
if (target.permissionMode === "plan" && !["claude", "cursor"].includes(provider)) {
|
|
647
|
+
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
648
|
+
}
|
|
649
|
+
if (target.permissionMode === "auto" && provider !== "claude") {
|
|
650
|
+
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (target.sandbox !== undefined) {
|
|
654
|
+
if (!capabilities.sandbox.length) {
|
|
655
|
+
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
656
|
+
}
|
|
657
|
+
if (!capabilities.sandbox.includes(target.sandbox)) {
|
|
658
|
+
throw new Error(`${label}.sandbox must be one of ${capabilities.sandbox.join(", ")}`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function codewithLikeSandbox(target) {
|
|
663
|
+
return target.sandbox ?? (target.permissionMode === "bypass" ? "danger-full-access" : "workspace-write");
|
|
664
|
+
}
|
|
665
|
+
function configStringValue(value) {
|
|
666
|
+
return JSON.stringify(value);
|
|
667
|
+
}
|
|
668
|
+
function buildAgentInvocation(target) {
|
|
669
|
+
const isolation = target.configIsolation ?? "safe";
|
|
670
|
+
const permissionMode = target.permissionMode ?? "default";
|
|
671
|
+
const args = [];
|
|
672
|
+
switch (target.provider) {
|
|
673
|
+
case "claude": {
|
|
674
|
+
if (isolation === "safe")
|
|
675
|
+
args.push("--safe-mode", "--setting-sources", "local", "--no-session-persistence");
|
|
676
|
+
if (permissionMode !== "default") {
|
|
677
|
+
const mode = permissionMode === "bypass" ? "bypassPermissions" : permissionMode;
|
|
678
|
+
args.push("--permission-mode", mode);
|
|
679
|
+
}
|
|
680
|
+
args.push("-p", "--output-format", "json");
|
|
681
|
+
if (target.model)
|
|
682
|
+
args.push("--model", target.model);
|
|
683
|
+
if (target.variant)
|
|
684
|
+
args.push("--effort", target.variant);
|
|
685
|
+
if (target.agent)
|
|
686
|
+
args.push("--agent", target.agent);
|
|
687
|
+
args.push(...target.extraArgs ?? []);
|
|
688
|
+
return { command: "claude", args, stdin: target.prompt };
|
|
689
|
+
}
|
|
690
|
+
case "cursor": {
|
|
691
|
+
args.push("-c", [
|
|
692
|
+
"set -eu",
|
|
693
|
+
"if command -v agent >/dev/null 2>&1; then",
|
|
694
|
+
' exec agent "$@"',
|
|
695
|
+
"else",
|
|
696
|
+
" echo 'Executable not found in PATH: agent' >&2",
|
|
697
|
+
" exit 127",
|
|
698
|
+
"fi"
|
|
699
|
+
].join(`
|
|
700
|
+
`), "openloops-cursor", "-p");
|
|
701
|
+
if (permissionMode === "plan")
|
|
702
|
+
args.push("--mode", "plan");
|
|
703
|
+
if (permissionMode === "bypass")
|
|
704
|
+
args.push("--force");
|
|
705
|
+
const cursorSandbox = target.sandbox ?? (isolation === "safe" ? "enabled" : undefined);
|
|
706
|
+
if (cursorSandbox)
|
|
707
|
+
args.push("--sandbox", cursorSandbox);
|
|
708
|
+
if (target.model)
|
|
709
|
+
args.push("--model", target.model);
|
|
710
|
+
if (target.agent)
|
|
711
|
+
args.push("--agent", target.agent);
|
|
712
|
+
args.push(...target.extraArgs ?? []);
|
|
713
|
+
return { command: "sh", args, stdin: target.prompt, preflightAnyOf: ["agent"] };
|
|
714
|
+
}
|
|
715
|
+
case "codewith": {
|
|
716
|
+
args.push(...target.authProfile ? ["--auth-profile", target.authProfile] : []);
|
|
717
|
+
if (target.variant)
|
|
718
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
719
|
+
args.push("--ask-for-approval", "never", "--sandbox", codewithLikeSandbox(target));
|
|
720
|
+
if (target.cwd)
|
|
721
|
+
args.push("--cd", target.cwd);
|
|
722
|
+
for (const dir of target.addDirs ?? [])
|
|
723
|
+
args.push("--add-dir", dir);
|
|
724
|
+
if (target.model)
|
|
725
|
+
args.push("--model", target.model);
|
|
726
|
+
args.push(...target.extraArgs ?? []);
|
|
727
|
+
args.push("agent", "start");
|
|
728
|
+
if (target.cwd)
|
|
729
|
+
args.push("--cwd", target.cwd);
|
|
730
|
+
args.push(target.prompt);
|
|
731
|
+
return { command: "codewith", args };
|
|
732
|
+
}
|
|
733
|
+
case "codex": {
|
|
734
|
+
if (target.variant)
|
|
735
|
+
args.push("-c", `model_reasoning_effort=${configStringValue(target.variant)}`);
|
|
736
|
+
args.push("--ask-for-approval", "never", "exec", "--json", "--ephemeral", "--sandbox", codewithLikeSandbox(target), "--skip-git-repo-check");
|
|
737
|
+
if (isolation === "safe")
|
|
738
|
+
args.push("--ignore-rules");
|
|
739
|
+
if (target.cwd)
|
|
740
|
+
args.push("--cd", target.cwd);
|
|
741
|
+
for (const dir of target.addDirs ?? [])
|
|
742
|
+
args.push("--add-dir", dir);
|
|
743
|
+
if (target.model)
|
|
744
|
+
args.push("--model", target.model);
|
|
745
|
+
args.push(...target.extraArgs ?? []);
|
|
746
|
+
return { command: "codex", args, stdin: target.prompt };
|
|
747
|
+
}
|
|
748
|
+
case "aicopilot":
|
|
749
|
+
case "opencode": {
|
|
750
|
+
args.push("run", "--format", "json");
|
|
751
|
+
if (isolation === "safe")
|
|
752
|
+
args.push("--pure");
|
|
753
|
+
if (permissionMode === "bypass")
|
|
754
|
+
args.push("--dangerously-skip-permissions");
|
|
755
|
+
if (target.cwd)
|
|
756
|
+
args.push("--dir", target.cwd);
|
|
757
|
+
if (target.model)
|
|
758
|
+
args.push("--model", target.model);
|
|
759
|
+
if (target.variant)
|
|
760
|
+
args.push("--variant", target.variant);
|
|
761
|
+
if (target.agent)
|
|
762
|
+
args.push("--agent", target.agent);
|
|
763
|
+
args.push(...target.extraArgs ?? []);
|
|
764
|
+
return { command: target.provider, args, stdin: target.prompt };
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
function adapterFor(provider, capabilities) {
|
|
769
|
+
return {
|
|
770
|
+
provider,
|
|
771
|
+
capabilities,
|
|
772
|
+
validate(target, label = provider) {
|
|
773
|
+
validateAgentOptions(target, label, capabilities);
|
|
774
|
+
},
|
|
775
|
+
buildInvocation(target) {
|
|
776
|
+
validateAgentOptions(target, provider, capabilities);
|
|
777
|
+
return buildAgentInvocation(target);
|
|
778
|
+
}
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
var PROVIDER_ADAPTERS = {
|
|
782
|
+
claude: adapterFor("claude", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
783
|
+
cursor: adapterFor("cursor", { sandbox: CURSOR_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
784
|
+
codewith: adapterFor("codewith", { sandbox: CODEX_LIKE_SANDBOXES, durable: true, remote: false, promptChannel: "argv" }),
|
|
785
|
+
codex: adapterFor("codex", { sandbox: CODEX_LIKE_SANDBOXES, durable: false, remote: true, promptChannel: "stdin" }),
|
|
786
|
+
aicopilot: adapterFor("aicopilot", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" }),
|
|
787
|
+
opencode: adapterFor("opencode", { sandbox: [], durable: false, remote: true, promptChannel: "stdin" })
|
|
788
|
+
};
|
|
789
|
+
var AGENT_PROVIDERS = Object.keys(PROVIDER_ADAPTERS);
|
|
790
|
+
function providerAdapter(provider) {
|
|
791
|
+
const adapter = PROVIDER_ADAPTERS[provider];
|
|
792
|
+
if (!adapter)
|
|
793
|
+
throw new Error(`unsupported agent provider: ${String(provider)}`);
|
|
794
|
+
return adapter;
|
|
795
|
+
}
|
|
796
|
+
var DEFAULT_CAPTURE_MAX_OUTPUT_BYTES = 256 * 1024;
|
|
797
|
+
function killProcessGroup(pgid) {
|
|
798
|
+
try {
|
|
799
|
+
process.kill(-pgid, "SIGTERM");
|
|
800
|
+
} catch {
|
|
801
|
+
try {
|
|
802
|
+
process.kill(pgid, "SIGTERM");
|
|
803
|
+
} catch {}
|
|
804
|
+
}
|
|
805
|
+
setTimeout(() => {
|
|
806
|
+
try {
|
|
807
|
+
process.kill(-pgid, "SIGKILL");
|
|
808
|
+
} catch {
|
|
809
|
+
try {
|
|
810
|
+
process.kill(pgid, "SIGKILL");
|
|
811
|
+
} catch {}
|
|
812
|
+
}
|
|
813
|
+
}, 2000).unref();
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
class BoundedOutputBuffer {
|
|
817
|
+
maxBytes;
|
|
818
|
+
text = "";
|
|
819
|
+
truncatedBytes = 0;
|
|
820
|
+
constructor(maxBytes) {
|
|
821
|
+
this.maxBytes = maxBytes;
|
|
822
|
+
}
|
|
823
|
+
append(chunk) {
|
|
824
|
+
if (!chunk)
|
|
825
|
+
return;
|
|
826
|
+
this.text += chunk;
|
|
827
|
+
if (Buffer.byteLength(this.text, "utf8") <= this.maxBytes)
|
|
828
|
+
return;
|
|
829
|
+
this.text = scrubSecrets(this.text);
|
|
830
|
+
const encoded = Buffer.from(this.text, "utf8");
|
|
831
|
+
if (encoded.length <= this.maxBytes)
|
|
832
|
+
return;
|
|
833
|
+
let start = encoded.length - this.maxBytes;
|
|
834
|
+
while (start < encoded.length && (encoded[start] & 192) === 128)
|
|
835
|
+
start++;
|
|
836
|
+
this.truncatedBytes += start;
|
|
837
|
+
this.text = encoded.subarray(start).toString("utf8");
|
|
838
|
+
}
|
|
839
|
+
value() {
|
|
840
|
+
if (this.truncatedBytes === 0)
|
|
841
|
+
return this.text;
|
|
842
|
+
return `[truncated ${this.truncatedBytes} bytes]
|
|
843
|
+
${this.text}`;
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
async function spawnCapture(command, args, opts) {
|
|
847
|
+
const maxOutputBytes = opts.maxOutputBytes ?? DEFAULT_CAPTURE_MAX_OUTPUT_BYTES;
|
|
848
|
+
const stdout = new BoundedOutputBuffer(maxOutputBytes);
|
|
849
|
+
const stderr = new BoundedOutputBuffer(maxOutputBytes);
|
|
850
|
+
let timedOut = false;
|
|
851
|
+
let error;
|
|
852
|
+
const child = spawn(command, args, {
|
|
853
|
+
cwd: opts.cwd,
|
|
854
|
+
env: opts.env,
|
|
855
|
+
detached: true,
|
|
856
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
857
|
+
});
|
|
858
|
+
const timer = setTimeout(() => {
|
|
859
|
+
timedOut = true;
|
|
860
|
+
if (child.pid)
|
|
861
|
+
killProcessGroup(child.pid);
|
|
862
|
+
}, opts.timeoutMs);
|
|
863
|
+
timer.unref();
|
|
864
|
+
child.stdout?.setEncoding("utf8");
|
|
865
|
+
child.stderr?.setEncoding("utf8");
|
|
866
|
+
child.stdout?.on("data", (chunk) => stdout.append(chunk));
|
|
867
|
+
child.stderr?.on("data", (chunk) => stderr.append(chunk));
|
|
868
|
+
const [status, signal] = await new Promise((resolve) => {
|
|
869
|
+
child.once("error", (err) => {
|
|
870
|
+
error = err.message;
|
|
871
|
+
resolve([null, null]);
|
|
872
|
+
});
|
|
873
|
+
child.once("close", (code, sig) => resolve([code, sig]));
|
|
874
|
+
});
|
|
875
|
+
clearTimeout(timer);
|
|
876
|
+
if (timedOut && !error)
|
|
877
|
+
error = `timed out after ${opts.timeoutMs}ms`;
|
|
878
|
+
return { status, signal, stdout: stdout.value(), stderr: stderr.value(), error, timedOut };
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
// src/lib/workflow-spec.ts
|
|
317
882
|
function assertObject(value, label) {
|
|
318
883
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
319
884
|
throw new Error(`${label} must be an object`);
|
|
@@ -376,7 +941,7 @@ function readPromptFile(rawPath, label, opts) {
|
|
|
376
941
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
377
942
|
let prompt;
|
|
378
943
|
try {
|
|
379
|
-
prompt =
|
|
944
|
+
prompt = readFileSync2(path, "utf8");
|
|
380
945
|
} catch (error) {
|
|
381
946
|
const code = error && typeof error === "object" && "code" in error ? String(error.code) : undefined;
|
|
382
947
|
throw new Error(`${label}.promptFile could not be read${code ? `: ${code}` : ""}`);
|
|
@@ -395,7 +960,7 @@ function matchingPromptSource(value, prompt, opts) {
|
|
|
395
960
|
return;
|
|
396
961
|
const path = promptFilePath(rawPath.trim(), opts);
|
|
397
962
|
try {
|
|
398
|
-
return
|
|
963
|
+
return readFileSync2(path, "utf8") === prompt ? { type: "file", path } : undefined;
|
|
399
964
|
} catch {
|
|
400
965
|
return;
|
|
401
966
|
}
|
|
@@ -442,65 +1007,14 @@ function validateTarget(value, label, opts) {
|
|
|
442
1007
|
if (!hasPrompt && !hasPromptFile)
|
|
443
1008
|
throw new Error(`${label}.prompt must be a non-empty string or ${label}.promptFile must be set`);
|
|
444
1009
|
const promptFields = hasPrompt ? { prompt: value.prompt, promptSource: matchingPromptSource(value.promptSource, value.prompt, opts) } : readPromptFile(value.promptFile, label, opts);
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
1010
|
+
if (!AGENT_PROVIDERS.includes(value.provider)) {
|
|
1011
|
+
throw new Error(`${label}.provider must be one of ${AGENT_PROVIDERS.join(", ")}`);
|
|
1012
|
+
}
|
|
448
1013
|
optionalTimeoutMs(value.timeoutMs, `${label}.timeoutMs`);
|
|
449
1014
|
optionalPositiveInteger(value.idleTimeoutMs, `${label}.idleTimeoutMs`);
|
|
450
|
-
|
|
451
|
-
assertString(value.authProfile, `${label}.authProfile`);
|
|
452
|
-
if (value.provider !== "codewith")
|
|
453
|
-
throw new Error(`${label}.authProfile is currently supported only for provider codewith`);
|
|
454
|
-
}
|
|
455
|
-
if (value.configIsolation !== undefined) {
|
|
456
|
-
assertString(value.configIsolation, `${label}.configIsolation`);
|
|
457
|
-
if (!["safe", "none"].includes(value.configIsolation))
|
|
458
|
-
throw new Error(`${label}.configIsolation must be safe or none`);
|
|
459
|
-
}
|
|
460
|
-
if (value.model !== undefined)
|
|
461
|
-
assertString(value.model, `${label}.model`);
|
|
462
|
-
if (value.provider === "opencode" && (typeof value.model !== "string" || value.model.trim() === "")) {
|
|
463
|
-
throw new Error(`${label}.model is required for provider opencode; pass a provider/model id such as openrouter/google/gemini-2.5-flash`);
|
|
464
|
-
}
|
|
465
|
-
if (value.variant !== undefined)
|
|
466
|
-
assertString(value.variant, `${label}.variant`);
|
|
467
|
-
if (value.agent !== undefined)
|
|
468
|
-
assertString(value.agent, `${label}.agent`);
|
|
469
|
-
if (value.provider === "cursor" && value.variant !== undefined)
|
|
470
|
-
throw new Error(`${label}.variant is not supported for provider cursor`);
|
|
471
|
-
if (value.provider === "codex" && value.agent !== undefined)
|
|
472
|
-
throw new Error(`${label}.agent is not supported for provider codex`);
|
|
1015
|
+
const extraArgs = optionalStringArray(value.extraArgs, `${label}.extraArgs`);
|
|
473
1016
|
optionalStringArray(value.addDirs, `${label}.addDirs`);
|
|
474
|
-
|
|
475
|
-
throw new Error(`${label}.addDirs is currently supported only for provider codewith or codex`);
|
|
476
|
-
}
|
|
477
|
-
if (value.permissionMode !== undefined) {
|
|
478
|
-
assertString(value.permissionMode, `${label}.permissionMode`);
|
|
479
|
-
const permissionModes = ["default", "plan", "auto", "bypass"];
|
|
480
|
-
if (!permissionModes.includes(value.permissionMode)) {
|
|
481
|
-
throw new Error(`${label}.permissionMode must be one of ${permissionModes.join(", ")}`);
|
|
482
|
-
}
|
|
483
|
-
if (value.permissionMode === "plan" && !["claude", "cursor"].includes(value.provider)) {
|
|
484
|
-
throw new Error(`${label}.permissionMode plan is currently supported only for provider claude or cursor`);
|
|
485
|
-
}
|
|
486
|
-
if (value.permissionMode === "auto" && value.provider !== "claude") {
|
|
487
|
-
throw new Error(`${label}.permissionMode auto is currently supported only for provider claude`);
|
|
488
|
-
}
|
|
489
|
-
}
|
|
490
|
-
if (value.sandbox !== undefined) {
|
|
491
|
-
assertString(value.sandbox, `${label}.sandbox`);
|
|
492
|
-
const codexLike = ["read-only", "workspace-write", "danger-full-access"];
|
|
493
|
-
const cursorLike = ["enabled", "disabled"];
|
|
494
|
-
if (["codewith", "codex"].includes(value.provider)) {
|
|
495
|
-
if (!codexLike.includes(value.sandbox))
|
|
496
|
-
throw new Error(`${label}.sandbox must be one of ${codexLike.join(", ")}`);
|
|
497
|
-
} else if (value.provider === "cursor") {
|
|
498
|
-
if (!cursorLike.includes(value.sandbox))
|
|
499
|
-
throw new Error(`${label}.sandbox must be one of ${cursorLike.join(", ")}`);
|
|
500
|
-
} else {
|
|
501
|
-
throw new Error(`${label}.sandbox is currently supported only for provider codewith, codex, or cursor`);
|
|
502
|
-
}
|
|
503
|
-
}
|
|
1017
|
+
providerAdapter(value.provider).validate({ ...value, extraArgs, ...promptFields }, label);
|
|
504
1018
|
if (value.allowlist !== undefined) {
|
|
505
1019
|
assertObject(value.allowlist, `${label}.allowlist`);
|
|
506
1020
|
optionalStringArray(value.allowlist.tools, `${label}.allowlist.tools`);
|
|
@@ -545,7 +1059,9 @@ function validateTarget(value, label, opts) {
|
|
|
545
1059
|
if (value.routing.eventSource !== undefined)
|
|
546
1060
|
assertString(value.routing.eventSource, `${label}.routing.eventSource`);
|
|
547
1061
|
}
|
|
548
|
-
const target = { ...value };
|
|
1062
|
+
const target = { ...value, extraArgs };
|
|
1063
|
+
if (!extraArgs)
|
|
1064
|
+
delete target.extraArgs;
|
|
549
1065
|
delete target.promptFile;
|
|
550
1066
|
delete target.promptSource;
|
|
551
1067
|
return { ...target, ...promptFields };
|
|
@@ -628,8 +1144,8 @@ function workflowBodyFromJson(value, fallbackName, opts = {}) {
|
|
|
628
1144
|
|
|
629
1145
|
// src/lib/run-artifacts.ts
|
|
630
1146
|
import { createHash } from "crypto";
|
|
631
|
-
import { mkdirSync as mkdirSync2, writeFileSync } from "fs";
|
|
632
|
-
import { basename, join as join2 } from "path";
|
|
1147
|
+
import { mkdirSync as mkdirSync2, renameSync, rmdirSync, rmSync, writeFileSync } from "fs";
|
|
1148
|
+
import { basename, dirname, join as join2 } from "path";
|
|
633
1149
|
function shortHash(value) {
|
|
634
1150
|
return createHash("sha256").update(value).digest("hex").slice(0, 12);
|
|
635
1151
|
}
|
|
@@ -650,13 +1166,14 @@ function workflowRunProjectSlug(projectKey) {
|
|
|
650
1166
|
return "global";
|
|
651
1167
|
return safeRunPathSlug(projectKey.startsWith("/") ? basename(projectKey) : projectKey, "project");
|
|
652
1168
|
}
|
|
653
|
-
function
|
|
1169
|
+
function stageWorkflowRunManifest(args) {
|
|
654
1170
|
const projectSlug = workflowRunProjectSlug(args.projectKey);
|
|
655
1171
|
const subjectKey = workflowRunSubjectKey(args.subjectKind, args.rawSubjectRef);
|
|
656
1172
|
const dir = join2(args.loopsDataDir, "runs", projectSlug, subjectKey, args.workflowRunId);
|
|
657
1173
|
mkdirSync2(dir, { recursive: true, mode: 448 });
|
|
658
1174
|
const manifestPath = join2(dir, "manifest.json");
|
|
659
|
-
|
|
1175
|
+
const tmpPath = `${manifestPath}.tmp`;
|
|
1176
|
+
writeFileSync(tmpPath, JSON.stringify({
|
|
660
1177
|
version: 1,
|
|
661
1178
|
workflowRunId: args.workflowRunId,
|
|
662
1179
|
workflowId: args.workflowId,
|
|
@@ -669,13 +1186,26 @@ function writeWorkflowRunManifest(args) {
|
|
|
669
1186
|
createdAt: new Date().toISOString(),
|
|
670
1187
|
...args.payload
|
|
671
1188
|
}, null, 2), { mode: 384 });
|
|
672
|
-
return manifestPath;
|
|
1189
|
+
return { manifestPath, tmpPath };
|
|
1190
|
+
}
|
|
1191
|
+
function commitWorkflowRunManifest(staged) {
|
|
1192
|
+
renameSync(staged.tmpPath, staged.manifestPath);
|
|
1193
|
+
return staged.manifestPath;
|
|
1194
|
+
}
|
|
1195
|
+
function discardWorkflowRunManifest(staged) {
|
|
1196
|
+
rmSync(staged.tmpPath, { force: true });
|
|
1197
|
+
try {
|
|
1198
|
+
rmdirSync(dirname(staged.manifestPath));
|
|
1199
|
+
} catch {}
|
|
673
1200
|
}
|
|
674
1201
|
|
|
675
1202
|
// src/lib/store.ts
|
|
676
1203
|
var DEFAULT_RECOVERY_BATCH_LIMIT = 100;
|
|
677
1204
|
var DEFAULT_RECOVERY_SCAN_MULTIPLIER = 5;
|
|
678
1205
|
var LIVE_EXPIRED_RUN_GRACE_MS = 60000;
|
|
1206
|
+
var SCHEMA_USER_VERSION = 6;
|
|
1207
|
+
var TERMINAL_RUN_STATUSES = ["succeeded", "failed", "timed_out", "abandoned", "skipped"];
|
|
1208
|
+
var PRUNE_BATCH_SIZE = 400;
|
|
679
1209
|
var GENERATED_ROUTE_TEMPLATE_IDS = new Set(["todos-task-worker-verifier", "task-lifecycle", "event-worker-verifier"]);
|
|
680
1210
|
var GENERATED_ROUTE_KEYS = new Set(["todos-task", "generic-event"]);
|
|
681
1211
|
function rowToLoop(row) {
|
|
@@ -716,6 +1246,8 @@ function rowToRun(row) {
|
|
|
716
1246
|
claimedBy: row.claimed_by ?? undefined,
|
|
717
1247
|
leaseExpiresAt: row.lease_expires_at ?? undefined,
|
|
718
1248
|
pid: row.pid ?? undefined,
|
|
1249
|
+
pgid: row.pgid ?? undefined,
|
|
1250
|
+
processStartedAt: row.process_started_at ?? undefined,
|
|
719
1251
|
exitCode: row.exit_code ?? undefined,
|
|
720
1252
|
durationMs: row.duration_ms ?? undefined,
|
|
721
1253
|
stdout: row.stdout ?? undefined,
|
|
@@ -901,6 +1433,24 @@ function isProcessAlive(pid) {
|
|
|
901
1433
|
return false;
|
|
902
1434
|
}
|
|
903
1435
|
}
|
|
1436
|
+
function isRecordedProcessAlive(pid, processStartedAt) {
|
|
1437
|
+
if (!pid || !isProcessAlive(pid))
|
|
1438
|
+
return false;
|
|
1439
|
+
return sameProcessStart(processStartedAt ?? undefined, processStartTimeMs(pid));
|
|
1440
|
+
}
|
|
1441
|
+
function isoProcessStart(pid) {
|
|
1442
|
+
const startedMs = processStartTimeMs(pid);
|
|
1443
|
+
return startedMs === undefined ? undefined : new Date(startedMs).toISOString();
|
|
1444
|
+
}
|
|
1445
|
+
function isLiveStepProcess(pid, stepStartedAt) {
|
|
1446
|
+
if (!isProcessAlive(pid))
|
|
1447
|
+
return false;
|
|
1448
|
+
const actualMs = processStartTimeMs(pid);
|
|
1449
|
+
const stepStartMs = stepStartedAt ? Date.parse(stepStartedAt) : Number.NaN;
|
|
1450
|
+
if (actualMs === undefined || !Number.isFinite(stepStartMs))
|
|
1451
|
+
return true;
|
|
1452
|
+
return actualMs >= stepStartMs - START_TIME_TOLERANCE_MS;
|
|
1453
|
+
}
|
|
904
1454
|
function rowToLease(row) {
|
|
905
1455
|
return {
|
|
906
1456
|
id: row.id,
|
|
@@ -920,6 +1470,9 @@ function workItemStatusForLoopRun(status, attempt, maxAttempts) {
|
|
|
920
1470
|
}
|
|
921
1471
|
return;
|
|
922
1472
|
}
|
|
1473
|
+
function scrubbedOrNull(value) {
|
|
1474
|
+
return value == null ? null : scrubSecrets(value);
|
|
1475
|
+
}
|
|
923
1476
|
function chmodIfExists(path, mode) {
|
|
924
1477
|
try {
|
|
925
1478
|
if (existsSync(path))
|
|
@@ -927,7 +1480,7 @@ function chmodIfExists(path, mode) {
|
|
|
927
1480
|
} catch {}
|
|
928
1481
|
}
|
|
929
1482
|
function ensurePrivateStorePath(file) {
|
|
930
|
-
const dir =
|
|
1483
|
+
const dir = dirname2(file);
|
|
931
1484
|
mkdirSync3(dir, { recursive: true, mode: 448 });
|
|
932
1485
|
chmodIfExists(dir, 448);
|
|
933
1486
|
chmodIfExists(file, 384);
|
|
@@ -942,14 +1495,19 @@ class Store {
|
|
|
942
1495
|
const file = path ?? dbPath();
|
|
943
1496
|
if (file !== ":memory:")
|
|
944
1497
|
ensurePrivateStorePath(file);
|
|
945
|
-
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) :
|
|
1498
|
+
this.rootDir = file === ":memory:" ? mkdtempSync(join3(tmpdir(), "open-loops-store-")) : dirname2(file);
|
|
946
1499
|
this.db = new Database(file);
|
|
947
1500
|
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
948
1501
|
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
949
1502
|
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
950
1503
|
if (file !== ":memory:")
|
|
951
1504
|
ensurePrivateStorePath(file);
|
|
952
|
-
|
|
1505
|
+
try {
|
|
1506
|
+
this.migrate();
|
|
1507
|
+
} catch (error) {
|
|
1508
|
+
this.db.close();
|
|
1509
|
+
throw error;
|
|
1510
|
+
}
|
|
953
1511
|
}
|
|
954
1512
|
migrate() {
|
|
955
1513
|
this.db.exec(`
|
|
@@ -957,7 +1515,69 @@ class Store {
|
|
|
957
1515
|
id TEXT PRIMARY KEY,
|
|
958
1516
|
applied_at TEXT NOT NULL
|
|
959
1517
|
);
|
|
960
|
-
|
|
1518
|
+
`);
|
|
1519
|
+
const versionRow = this.db.query("PRAGMA user_version").get();
|
|
1520
|
+
const userVersion = versionRow?.user_version ?? 0;
|
|
1521
|
+
if (userVersion > SCHEMA_USER_VERSION) {
|
|
1522
|
+
throw new Error(`loops database schema version ${userVersion} is newer than this binary supports (${SCHEMA_USER_VERSION}); upgrade open-loops before opening this database`);
|
|
1523
|
+
}
|
|
1524
|
+
const applied = new Set(this.db.query("SELECT id FROM schema_migrations").all().map((row) => row.id));
|
|
1525
|
+
for (const migration of this.migrations()) {
|
|
1526
|
+
if (!migration.baseline && applied.has(migration.id))
|
|
1527
|
+
continue;
|
|
1528
|
+
migration.apply();
|
|
1529
|
+
if (!applied.has(migration.id)) {
|
|
1530
|
+
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run(migration.id, nowIso());
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
if (userVersion !== SCHEMA_USER_VERSION)
|
|
1534
|
+
this.db.exec(`PRAGMA user_version = ${SCHEMA_USER_VERSION}`);
|
|
1535
|
+
}
|
|
1536
|
+
migrations() {
|
|
1537
|
+
return [
|
|
1538
|
+
{ id: "0001_initial_and_workflows", baseline: true, apply: () => this.createBaseSchema() },
|
|
1539
|
+
{ id: "0002_loop_machines", baseline: true, apply: () => this.addColumnIfMissing("loops", "machine_json", "TEXT") },
|
|
1540
|
+
{
|
|
1541
|
+
id: "0003_goals",
|
|
1542
|
+
baseline: true,
|
|
1543
|
+
apply: () => {
|
|
1544
|
+
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1545
|
+
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1546
|
+
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1547
|
+
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1548
|
+
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1549
|
+
}
|
|
1550
|
+
},
|
|
1551
|
+
{
|
|
1552
|
+
id: "0004_loop_archive_metadata",
|
|
1553
|
+
baseline: true,
|
|
1554
|
+
apply: () => {
|
|
1555
|
+
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1556
|
+
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1557
|
+
}
|
|
1558
|
+
},
|
|
1559
|
+
{
|
|
1560
|
+
id: "0005_workflow_invocations_and_admission",
|
|
1561
|
+
baseline: true,
|
|
1562
|
+
apply: () => {
|
|
1563
|
+
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1564
|
+
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1565
|
+
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1566
|
+
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1567
|
+
this.createWorkflowRunBackfillIndexes();
|
|
1568
|
+
}
|
|
1569
|
+
},
|
|
1570
|
+
{
|
|
1571
|
+
id: "0006_run_process_tracking",
|
|
1572
|
+
apply: () => {
|
|
1573
|
+
this.addColumnIfMissing("loop_runs", "pgid", "INTEGER");
|
|
1574
|
+
this.addColumnIfMissing("loop_runs", "process_started_at", "TEXT");
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
];
|
|
1578
|
+
}
|
|
1579
|
+
createBaseSchema() {
|
|
1580
|
+
this.db.exec(`
|
|
961
1581
|
CREATE TABLE IF NOT EXISTS loops (
|
|
962
1582
|
id TEXT PRIMARY KEY,
|
|
963
1583
|
name TEXT NOT NULL,
|
|
@@ -996,6 +1616,8 @@ class Store {
|
|
|
996
1616
|
claimed_by TEXT,
|
|
997
1617
|
lease_expires_at TEXT,
|
|
998
1618
|
pid INTEGER,
|
|
1619
|
+
pgid INTEGER,
|
|
1620
|
+
process_started_at TEXT,
|
|
999
1621
|
exit_code INTEGER,
|
|
1000
1622
|
duration_ms INTEGER,
|
|
1001
1623
|
stdout TEXT,
|
|
@@ -1220,24 +1842,6 @@ class Store {
|
|
|
1220
1842
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_loop_run ON goal_runs(loop_run_id);
|
|
1221
1843
|
CREATE INDEX IF NOT EXISTS idx_goal_runs_workflow_run ON goal_runs(workflow_run_id);
|
|
1222
1844
|
`);
|
|
1223
|
-
this.addColumnIfMissing("loops", "machine_json", "TEXT");
|
|
1224
|
-
this.addColumnIfMissing("loops", "goal_json", "TEXT");
|
|
1225
|
-
this.addColumnIfMissing("loops", "archived_at", "TEXT");
|
|
1226
|
-
this.addColumnIfMissing("loops", "archived_from_status", "TEXT");
|
|
1227
|
-
this.addColumnIfMissing("loop_runs", "goal_run_id", "TEXT");
|
|
1228
|
-
this.addColumnIfMissing("workflow_specs", "goal_json", "TEXT");
|
|
1229
|
-
this.addColumnIfMissing("workflow_runs", "goal_run_id", "TEXT");
|
|
1230
|
-
this.addColumnIfMissing("workflow_runs", "invocation_id", "TEXT");
|
|
1231
|
-
this.addColumnIfMissing("workflow_runs", "work_item_id", "TEXT");
|
|
1232
|
-
this.addColumnIfMissing("workflow_runs", "manifest_path", "TEXT");
|
|
1233
|
-
this.addColumnIfMissing("workflow_step_runs", "pid", "INTEGER");
|
|
1234
|
-
this.addColumnIfMissing("workflow_step_runs", "goal_run_id", "TEXT");
|
|
1235
|
-
this.createWorkflowRunBackfillIndexes();
|
|
1236
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0001_initial_and_workflows", nowIso());
|
|
1237
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0002_loop_machines", nowIso());
|
|
1238
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0003_goals", nowIso());
|
|
1239
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0004_loop_archive_metadata", nowIso());
|
|
1240
|
-
this.db.query("INSERT OR IGNORE INTO schema_migrations (id, applied_at) VALUES (?, ?)").run("0005_workflow_invocations_and_admission", nowIso());
|
|
1241
1845
|
}
|
|
1242
1846
|
addColumnIfMissing(table, column, definition) {
|
|
1243
1847
|
const columns = this.db.query(`PRAGMA table_info(${table})`).all();
|
|
@@ -1251,6 +1855,9 @@ class Store {
|
|
|
1251
1855
|
CREATE INDEX IF NOT EXISTS idx_workflow_runs_work_item ON workflow_runs(work_item_id);
|
|
1252
1856
|
`);
|
|
1253
1857
|
}
|
|
1858
|
+
transact(fn) {
|
|
1859
|
+
return this.db.inTransaction ? fn() : this.writeTransaction(fn);
|
|
1860
|
+
}
|
|
1254
1861
|
assertDaemonLeaseFence(opts = {}, now = nowIso()) {
|
|
1255
1862
|
if (!opts.daemonLeaseId)
|
|
1256
1863
|
return;
|
|
@@ -1332,14 +1939,14 @@ class Store {
|
|
|
1332
1939
|
return byId;
|
|
1333
1940
|
const rows = this.db.query("SELECT * FROM loops WHERE name = ? ORDER BY created_at DESC LIMIT 2").all(idOrName);
|
|
1334
1941
|
if (rows.length === 0)
|
|
1335
|
-
throw new
|
|
1942
|
+
throw new LoopNotFoundError(idOrName);
|
|
1336
1943
|
if (rows.length > 1)
|
|
1337
|
-
throw new
|
|
1944
|
+
throw new AmbiguousNameError(idOrName);
|
|
1338
1945
|
return rowToLoop(rows[0]);
|
|
1339
1946
|
}
|
|
1340
1947
|
requireLoop(idOrName) {
|
|
1341
1948
|
return this.getLoop(idOrName) ?? this.findLoopByName(idOrName) ?? (() => {
|
|
1342
|
-
throw new
|
|
1949
|
+
throw new LoopNotFoundError(idOrName);
|
|
1343
1950
|
})();
|
|
1344
1951
|
}
|
|
1345
1952
|
listLoops(opts = {}) {
|
|
@@ -1376,7 +1983,9 @@ class Store {
|
|
|
1376
1983
|
try {
|
|
1377
1984
|
const current = this.getLoop(id);
|
|
1378
1985
|
if (!current)
|
|
1379
|
-
throw new
|
|
1986
|
+
throw new LoopNotFoundError(id);
|
|
1987
|
+
if (current.archivedAt)
|
|
1988
|
+
throw new LoopArchivedError(current.name || id);
|
|
1380
1989
|
const merged = { ...current, ...patch, updatedAt: updated };
|
|
1381
1990
|
const res = this.db.query(`UPDATE loops SET status=$status, next_run_at=$nextRun, retry_scheduled_for=$retrySlot,
|
|
1382
1991
|
expires_at=$expiresAt, updated_at=$updated
|
|
@@ -1608,10 +2217,10 @@ class Store {
|
|
|
1608
2217
|
renameLoop(id, name, opts = {}) {
|
|
1609
2218
|
const current = this.getLoop(id);
|
|
1610
2219
|
if (!current)
|
|
1611
|
-
throw new
|
|
2220
|
+
throw new LoopNotFoundError(id);
|
|
1612
2221
|
const trimmed = name.trim();
|
|
1613
2222
|
if (!trimmed)
|
|
1614
|
-
throw new
|
|
2223
|
+
throw new ValidationError("loop name must not be empty");
|
|
1615
2224
|
const updated = (opts.now ?? new Date).toISOString();
|
|
1616
2225
|
this.db.query(`UPDATE loops SET name=$name, updated_at=$updated
|
|
1617
2226
|
WHERE id=$id
|
|
@@ -1630,25 +2239,27 @@ class Store {
|
|
|
1630
2239
|
return after;
|
|
1631
2240
|
}
|
|
1632
2241
|
archiveLoop(idOrName) {
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
2242
|
+
return this.transact(() => {
|
|
2243
|
+
const loop = this.requireLoop(idOrName);
|
|
2244
|
+
if (loop.archivedAt)
|
|
2245
|
+
return loop;
|
|
2246
|
+
const updated = nowIso();
|
|
2247
|
+
const archivedStatus = loop.status === "active" ? "paused" : loop.status;
|
|
2248
|
+
this.db.query(`UPDATE loops
|
|
2249
|
+
SET status=$status, archived_at=$archivedAt, archived_from_status=$archivedFromStatus, updated_at=$updated
|
|
2250
|
+
WHERE id=$id`).run({
|
|
2251
|
+
$id: loop.id,
|
|
2252
|
+
$status: archivedStatus,
|
|
2253
|
+
$archivedAt: updated,
|
|
2254
|
+
$archivedFromStatus: loop.status,
|
|
2255
|
+
$updated: updated
|
|
2256
|
+
});
|
|
2257
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
2258
|
+
const archived = this.getLoop(loop.id);
|
|
2259
|
+
if (!archived)
|
|
2260
|
+
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
2261
|
+
return archived;
|
|
1646
2262
|
});
|
|
1647
|
-
this.setWorkflowWorkItemsForLoop(loop.id, "deferred", "loop archived", updated);
|
|
1648
|
-
const archived = this.getLoop(loop.id);
|
|
1649
|
-
if (!archived)
|
|
1650
|
-
throw new Error(`loop not found after archive: ${loop.id}`);
|
|
1651
|
-
return archived;
|
|
1652
2263
|
}
|
|
1653
2264
|
unarchiveLoop(idOrName) {
|
|
1654
2265
|
const loop = this.requireLoop(idOrName);
|
|
@@ -1669,10 +2280,12 @@ class Store {
|
|
|
1669
2280
|
return unarchived;
|
|
1670
2281
|
}
|
|
1671
2282
|
deleteLoop(idOrName) {
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
2283
|
+
return this.transact(() => {
|
|
2284
|
+
const loop = this.requireLoop(idOrName);
|
|
2285
|
+
this.setWorkflowWorkItemsForLoop(loop.id, "cancelled", "loop deleted", nowIso());
|
|
2286
|
+
const res = this.db.query("DELETE FROM loops WHERE id = ?").run(loop.id);
|
|
2287
|
+
return res.changes > 0;
|
|
2288
|
+
});
|
|
1676
2289
|
}
|
|
1677
2290
|
createWorkflow(input) {
|
|
1678
2291
|
const normalized = normalizeCreateWorkflowInput(input);
|
|
@@ -2189,26 +2802,7 @@ class Store {
|
|
|
2189
2802
|
try {
|
|
2190
2803
|
this.assertDaemonLeaseFence(opts, now);
|
|
2191
2804
|
for (const node of withReady) {
|
|
2192
|
-
this.
|
|
2193
|
-
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2194
|
-
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2195
|
-
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2196
|
-
$id: node.nodeId,
|
|
2197
|
-
$goalId: goal.goalId,
|
|
2198
|
-
$planId: goal.planId,
|
|
2199
|
-
$key: node.key,
|
|
2200
|
-
$sequence: node.sequence,
|
|
2201
|
-
$priority: node.priority,
|
|
2202
|
-
$objective: node.objective,
|
|
2203
|
-
$status: node.status,
|
|
2204
|
-
$ready: node.ready ? 1 : 0,
|
|
2205
|
-
$tokenBudget: node.tokenBudget ?? null,
|
|
2206
|
-
$tokensUsed: node.tokensUsed,
|
|
2207
|
-
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2208
|
-
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2209
|
-
$created: node.createdAt,
|
|
2210
|
-
$updated: node.updatedAt
|
|
2211
|
-
});
|
|
2805
|
+
this.insertGoalPlanNode(goal, node);
|
|
2212
2806
|
}
|
|
2213
2807
|
this.db.exec("COMMIT");
|
|
2214
2808
|
} catch (error) {
|
|
@@ -2219,6 +2813,38 @@ class Store {
|
|
|
2219
2813
|
}
|
|
2220
2814
|
return this.listGoalPlanNodes(goalId);
|
|
2221
2815
|
}
|
|
2816
|
+
insertGoalPlanNode(goal, node) {
|
|
2817
|
+
let id = node.nodeId;
|
|
2818
|
+
for (let attempt = 0;attempt < 3; attempt += 1) {
|
|
2819
|
+
const res = this.db.query(`INSERT OR IGNORE INTO goal_plan_nodes (id, goal_id, plan_id, key, sequence, priority, objective, status, ready,
|
|
2820
|
+
token_budget, tokens_used, time_used_seconds, depends_on_json, created_at, updated_at)
|
|
2821
|
+
VALUES ($id, $goalId, $planId, $key, $sequence, $priority, $objective, $status, $ready, $tokenBudget,
|
|
2822
|
+
$tokensUsed, $timeUsedSeconds, $dependsOn, $created, $updated)`).run({
|
|
2823
|
+
$id: id,
|
|
2824
|
+
$goalId: goal.goalId,
|
|
2825
|
+
$planId: goal.planId,
|
|
2826
|
+
$key: node.key,
|
|
2827
|
+
$sequence: node.sequence,
|
|
2828
|
+
$priority: node.priority,
|
|
2829
|
+
$objective: node.objective,
|
|
2830
|
+
$status: node.status,
|
|
2831
|
+
$ready: node.ready ? 1 : 0,
|
|
2832
|
+
$tokenBudget: node.tokenBudget ?? null,
|
|
2833
|
+
$tokensUsed: node.tokensUsed,
|
|
2834
|
+
$timeUsedSeconds: node.timeUsedSeconds,
|
|
2835
|
+
$dependsOn: JSON.stringify(node.dependsOn),
|
|
2836
|
+
$created: node.createdAt,
|
|
2837
|
+
$updated: node.updatedAt
|
|
2838
|
+
});
|
|
2839
|
+
if (res.changes === 1)
|
|
2840
|
+
return;
|
|
2841
|
+
const existingByKey = this.db.query("SELECT id FROM goal_plan_nodes WHERE plan_id = ? AND key = ? LIMIT 1").get(goal.planId, node.key);
|
|
2842
|
+
if (existingByKey)
|
|
2843
|
+
return;
|
|
2844
|
+
id = genId();
|
|
2845
|
+
}
|
|
2846
|
+
throw new Error(`goal plan node was not inserted after id retries: ${goal.planId}/${node.key}`);
|
|
2847
|
+
}
|
|
2222
2848
|
listGoalPlanNodes(goalIdOrPlanId) {
|
|
2223
2849
|
const rows = this.db.query("SELECT * FROM goal_plan_nodes WHERE goal_id = ? OR plan_id = ? ORDER BY sequence ASC").all(goalIdOrPlanId, goalIdOrPlanId);
|
|
2224
2850
|
return rows.map(rowToGoalPlanNode);
|
|
@@ -2310,8 +2936,8 @@ class Store {
|
|
|
2310
2936
|
$status: input.status,
|
|
2311
2937
|
$nodeKey: input.nodeKey ?? null,
|
|
2312
2938
|
$tokensUsed: input.tokensUsed ?? 0,
|
|
2313
|
-
$evidence: input.evidence ? JSON.stringify(input.evidence) : null,
|
|
2314
|
-
$rawResponse: input.rawResponse === undefined ? null : JSON.stringify(input.rawResponse),
|
|
2939
|
+
$evidence: input.evidence ? scrubSecrets(JSON.stringify(scrubSecretsDeep(input.evidence))) : null,
|
|
2940
|
+
$rawResponse: input.rawResponse === undefined ? null : scrubSecrets(JSON.stringify(scrubSecretsDeep(input.rawResponse))),
|
|
2315
2941
|
$created: now,
|
|
2316
2942
|
$updated: now
|
|
2317
2943
|
});
|
|
@@ -2349,7 +2975,6 @@ class Store {
|
|
|
2349
2975
|
const targetInput = input.loop?.target.type === "workflow" ? input.loop.target.input : undefined;
|
|
2350
2976
|
const invocationId = input.invocationId ?? targetInput?.workflowInvocationId ?? targetInput?.invocationId;
|
|
2351
2977
|
const workItemId = input.workItemId ?? targetInput?.workflowWorkItemId ?? targetInput?.workItemId;
|
|
2352
|
-
let manifestPath;
|
|
2353
2978
|
if (input.idempotencyKey) {
|
|
2354
2979
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2355
2980
|
if (existing) {
|
|
@@ -2357,6 +2982,28 @@ class Store {
|
|
|
2357
2982
|
return rowToWorkflowRun(existing);
|
|
2358
2983
|
}
|
|
2359
2984
|
}
|
|
2985
|
+
const runId = genId();
|
|
2986
|
+
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2987
|
+
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2988
|
+
const staged = stageWorkflowRunManifest({
|
|
2989
|
+
loopsDataDir: this.rootDir,
|
|
2990
|
+
workflowRunId: runId,
|
|
2991
|
+
workflowId: input.workflow.id,
|
|
2992
|
+
workflowName: input.workflow.name,
|
|
2993
|
+
invocationId,
|
|
2994
|
+
workItemId,
|
|
2995
|
+
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2996
|
+
subjectKind: invocation?.subjectRef.kind ?? (input.loop ? "loop" : "workflow"),
|
|
2997
|
+
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url ?? input.loop?.name ?? input.workflow.name,
|
|
2998
|
+
payload: {
|
|
2999
|
+
workflowInvocation: invocation,
|
|
3000
|
+
workflowWorkItem: workItem,
|
|
3001
|
+
loopId: input.loop?.id,
|
|
3002
|
+
loopRunId: input.loopRun?.id,
|
|
3003
|
+
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
3004
|
+
}
|
|
3005
|
+
});
|
|
3006
|
+
const manifestPath = staged.manifestPath;
|
|
2360
3007
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2361
3008
|
try {
|
|
2362
3009
|
this.assertDaemonLeaseFence(input, now);
|
|
@@ -2364,30 +3011,10 @@ class Store {
|
|
|
2364
3011
|
const existing = this.db.query("SELECT * FROM workflow_runs WHERE workflow_id = ? AND idempotency_key = ? LIMIT 1").get(input.workflow.id, input.idempotencyKey);
|
|
2365
3012
|
if (existing) {
|
|
2366
3013
|
this.db.exec("COMMIT");
|
|
3014
|
+
discardWorkflowRunManifest(staged);
|
|
2367
3015
|
return rowToWorkflowRun(existing);
|
|
2368
3016
|
}
|
|
2369
3017
|
}
|
|
2370
|
-
const runId = genId();
|
|
2371
|
-
const workItem = workItemId ? this.getWorkflowWorkItem(workItemId) : undefined;
|
|
2372
|
-
const invocation = invocationId ? this.getWorkflowInvocation(invocationId) : undefined;
|
|
2373
|
-
manifestPath = invocation || workItem ? writeWorkflowRunManifest({
|
|
2374
|
-
loopsDataDir: this.rootDir,
|
|
2375
|
-
workflowRunId: runId,
|
|
2376
|
-
workflowId: input.workflow.id,
|
|
2377
|
-
workflowName: input.workflow.name,
|
|
2378
|
-
invocationId,
|
|
2379
|
-
workItemId,
|
|
2380
|
-
projectKey: workItem?.projectKey ?? invocation?.scope?.projectPath,
|
|
2381
|
-
subjectKind: invocation?.subjectRef.kind,
|
|
2382
|
-
rawSubjectRef: workItem?.subjectRef ?? invocation?.subjectRef.path ?? invocation?.subjectRef.id ?? invocation?.subjectRef.url,
|
|
2383
|
-
payload: {
|
|
2384
|
-
workflowInvocation: invocation,
|
|
2385
|
-
workflowWorkItem: workItem,
|
|
2386
|
-
loopId: input.loop?.id,
|
|
2387
|
-
loopRunId: input.loopRun?.id,
|
|
2388
|
-
scheduledFor: input.scheduledFor ?? input.loopRun?.scheduledFor
|
|
2389
|
-
}
|
|
2390
|
-
}) : undefined;
|
|
2391
3018
|
this.db.query(`INSERT INTO workflow_runs (id, workflow_id, workflow_name, loop_id, loop_run_id, invocation_id, work_item_id,
|
|
2392
3019
|
scheduled_for, idempotency_key, manifest_path, status, started_at, finished_at, duration_ms, error,
|
|
2393
3020
|
created_at, updated_at)
|
|
@@ -2454,6 +3081,7 @@ class Store {
|
|
|
2454
3081
|
$created: now
|
|
2455
3082
|
});
|
|
2456
3083
|
this.db.exec("COMMIT");
|
|
3084
|
+
commitWorkflowRunManifest(staged);
|
|
2457
3085
|
const run = this.getWorkflowRun(runId);
|
|
2458
3086
|
if (!run)
|
|
2459
3087
|
throw new Error(`workflow run not found after create: ${runId}`);
|
|
@@ -2462,8 +3090,7 @@ class Store {
|
|
|
2462
3090
|
try {
|
|
2463
3091
|
this.db.exec("ROLLBACK");
|
|
2464
3092
|
} catch {}
|
|
2465
|
-
|
|
2466
|
-
rmSync(manifestPath, { force: true });
|
|
3093
|
+
discardWorkflowRunManifest(staged);
|
|
2467
3094
|
throw error;
|
|
2468
3095
|
}
|
|
2469
3096
|
}
|
|
@@ -2561,29 +3188,32 @@ class Store {
|
|
|
2561
3188
|
return run;
|
|
2562
3189
|
}
|
|
2563
3190
|
recoverWorkflowRun(workflowRunId, reason = "workflow run recovered for retry") {
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
3191
|
+
return this.transact(() => {
|
|
3192
|
+
const now = nowIso();
|
|
3193
|
+
const before = this.listWorkflowStepRuns(workflowRunId).filter((step) => step.status === "running");
|
|
3194
|
+
const live = before.filter((step) => step.pid !== undefined && isLiveStepProcess(step.pid, step.startedAt));
|
|
3195
|
+
if (live.length > 0) {
|
|
3196
|
+
throw new Error(`cannot recover workflow run while step processes are still alive: ${live.map((step) => `${step.stepId} pid=${step.pid}`).join(", ")}`);
|
|
3197
|
+
}
|
|
3198
|
+
this.db.query(`UPDATE workflow_step_runs
|
|
3199
|
+
SET status='pending', started_at=NULL, finished_at=NULL, exit_code=NULL, pid=NULL, duration_ms=NULL,
|
|
3200
|
+
stdout=NULL, stderr=NULL, error=$reason, updated_at=$updated
|
|
3201
|
+
WHERE workflow_run_id=$workflowRunId AND status='running'`).run({ $workflowRunId: workflowRunId, $reason: reason, $updated: now });
|
|
3202
|
+
if (before.length > 0) {
|
|
3203
|
+
this.appendWorkflowEvent(workflowRunId, "recovered", undefined, {
|
|
3204
|
+
reason,
|
|
3205
|
+
recoveredSteps: before.map((step) => step.stepId)
|
|
3206
|
+
});
|
|
3207
|
+
}
|
|
3208
|
+
return {
|
|
3209
|
+
run: this.requireWorkflowRun(workflowRunId),
|
|
3210
|
+
recoveredSteps: before.map((step) => this.getWorkflowStepRun(workflowRunId, step.stepId)).filter(Boolean)
|
|
3211
|
+
};
|
|
3212
|
+
});
|
|
2584
3213
|
}
|
|
2585
3214
|
finalizeWorkflowStepRun(workflowRunId, stepId, patch, opts = {}) {
|
|
2586
3215
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3216
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2587
3217
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2588
3218
|
try {
|
|
2589
3219
|
const res = this.db.query(`UPDATE workflow_step_runs SET status=$status, finished_at=$finished, exit_code=$exitCode, duration_ms=$durationMs,
|
|
@@ -2598,9 +3228,9 @@ class Store {
|
|
|
2598
3228
|
$finished: finishedAt,
|
|
2599
3229
|
$exitCode: patch.exitCode ?? null,
|
|
2600
3230
|
$durationMs: patch.durationMs ?? null,
|
|
2601
|
-
$stdout: patch.stdout
|
|
2602
|
-
$stderr: patch.stderr
|
|
2603
|
-
$error:
|
|
3231
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3232
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3233
|
+
$error: error ?? null,
|
|
2604
3234
|
$updated: finishedAt,
|
|
2605
3235
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2606
3236
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
@@ -2608,15 +3238,15 @@ class Store {
|
|
|
2608
3238
|
if (res.changes === 1) {
|
|
2609
3239
|
this.appendWorkflowEvent(workflowRunId, `step_${patch.status}`, stepId, {
|
|
2610
3240
|
exitCode: patch.exitCode,
|
|
2611
|
-
error
|
|
3241
|
+
error
|
|
2612
3242
|
});
|
|
2613
3243
|
}
|
|
2614
3244
|
this.db.exec("COMMIT");
|
|
2615
|
-
} catch (
|
|
3245
|
+
} catch (error2) {
|
|
2616
3246
|
try {
|
|
2617
3247
|
this.db.exec("ROLLBACK");
|
|
2618
3248
|
} catch {}
|
|
2619
|
-
throw
|
|
3249
|
+
throw error2;
|
|
2620
3250
|
}
|
|
2621
3251
|
const run = this.getWorkflowStepRun(workflowRunId, stepId);
|
|
2622
3252
|
if (!run)
|
|
@@ -2656,6 +3286,7 @@ class Store {
|
|
|
2656
3286
|
}
|
|
2657
3287
|
finalizeWorkflowRun(workflowRunId, status, patch = {}, opts = {}) {
|
|
2658
3288
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3289
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2659
3290
|
let changed = false;
|
|
2660
3291
|
this.db.exec("BEGIN IMMEDIATE");
|
|
2661
3292
|
try {
|
|
@@ -2668,25 +3299,25 @@ class Store {
|
|
|
2668
3299
|
$status: status,
|
|
2669
3300
|
$finished: finishedAt,
|
|
2670
3301
|
$durationMs: patch.durationMs ?? null,
|
|
2671
|
-
$error:
|
|
3302
|
+
$error: error ?? null,
|
|
2672
3303
|
$updated: finishedAt,
|
|
2673
3304
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2674
3305
|
$now: (opts.now ?? new Date(finishedAt)).toISOString()
|
|
2675
3306
|
});
|
|
2676
3307
|
changed = res.changes === 1;
|
|
2677
3308
|
if (changed)
|
|
2678
|
-
this.appendWorkflowEvent(workflowRunId, status, undefined, { error
|
|
3309
|
+
this.appendWorkflowEvent(workflowRunId, status, undefined, { error });
|
|
2679
3310
|
if (changed) {
|
|
2680
3311
|
const itemStatus = status === "succeeded" ? "succeeded" : status === "cancelled" ? "cancelled" : "failed";
|
|
2681
|
-
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus,
|
|
3312
|
+
this.setWorkflowWorkItemsForWorkflowRun(workflowRunId, itemStatus, error, finishedAt);
|
|
2682
3313
|
this.maybeArchiveTerminalGeneratedRouteWorkflow(workflowRunId, finishedAt);
|
|
2683
3314
|
}
|
|
2684
3315
|
this.db.exec("COMMIT");
|
|
2685
|
-
} catch (
|
|
3316
|
+
} catch (error2) {
|
|
2686
3317
|
try {
|
|
2687
3318
|
this.db.exec("ROLLBACK");
|
|
2688
3319
|
} catch {}
|
|
2689
|
-
throw
|
|
3320
|
+
throw error2;
|
|
2690
3321
|
}
|
|
2691
3322
|
const run = this.getWorkflowRun(workflowRunId);
|
|
2692
3323
|
if (!run)
|
|
@@ -2719,24 +3350,26 @@ class Store {
|
|
|
2719
3350
|
}
|
|
2720
3351
|
}
|
|
2721
3352
|
appendWorkflowEvent(workflowRunId, eventType, stepId, payload) {
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
3353
|
+
return this.transact(() => {
|
|
3354
|
+
const now = nowIso();
|
|
3355
|
+
const current = this.db.query("SELECT MAX(sequence) AS sequence FROM workflow_events WHERE workflow_run_id = ?").get(workflowRunId);
|
|
3356
|
+
const sequence = (current?.sequence ?? 0) + 1;
|
|
3357
|
+
const id = genId();
|
|
3358
|
+
this.db.query(`INSERT INTO workflow_events (id, workflow_run_id, sequence, event_type, step_id, payload_json, created_at)
|
|
3359
|
+
VALUES ($id, $workflowRunId, $sequence, $eventType, $stepId, $payload, $created)`).run({
|
|
3360
|
+
$id: id,
|
|
3361
|
+
$workflowRunId: workflowRunId,
|
|
3362
|
+
$sequence: sequence,
|
|
3363
|
+
$eventType: eventType,
|
|
3364
|
+
$stepId: stepId ?? null,
|
|
3365
|
+
$payload: payload ? JSON.stringify(payload) : null,
|
|
3366
|
+
$created: now
|
|
3367
|
+
});
|
|
3368
|
+
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
3369
|
+
if (!event)
|
|
3370
|
+
throw new Error(`workflow event not found after append: ${id}`);
|
|
3371
|
+
return rowToWorkflowEvent(event);
|
|
2735
3372
|
});
|
|
2736
|
-
const event = this.db.query("SELECT * FROM workflow_events WHERE id = ?").get(id);
|
|
2737
|
-
if (!event)
|
|
2738
|
-
throw new Error(`workflow event not found after append: ${id}`);
|
|
2739
|
-
return rowToWorkflowEvent(event);
|
|
2740
3373
|
}
|
|
2741
3374
|
listWorkflowEvents(workflowRunId, limit = 200) {
|
|
2742
3375
|
const rows = this.db.query("SELECT * FROM workflow_events WHERE workflow_run_id = ? ORDER BY sequence ASC LIMIT ?").all(workflowRunId, limit);
|
|
@@ -2752,35 +3385,64 @@ class Store {
|
|
|
2752
3385
|
}
|
|
2753
3386
|
markRunPid(id, pid, claimedBy, opts = {}) {
|
|
2754
3387
|
const now = (opts.now ?? new Date).toISOString();
|
|
2755
|
-
const
|
|
3388
|
+
const startedMs = processStartTimeMs(pid);
|
|
3389
|
+
const processStartedAt = startedMs === undefined ? null : new Date(startedMs).toISOString();
|
|
3390
|
+
const res = claimedBy ? this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2756
3391
|
WHERE id=$id AND status='running' AND claimed_by=$claimedBy
|
|
2757
3392
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2758
3393
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2759
3394
|
))`).run({
|
|
2760
3395
|
$id: id,
|
|
2761
3396
|
$pid: pid,
|
|
3397
|
+
$processStartedAt: processStartedAt,
|
|
2762
3398
|
$updated: now,
|
|
2763
3399
|
$claimedBy: claimedBy,
|
|
2764
3400
|
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
2765
3401
|
$now: now
|
|
2766
|
-
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, updated_at=$updated
|
|
3402
|
+
}) : this.db.query(`UPDATE loop_runs SET pid=$pid, process_started_at=$processStartedAt, updated_at=$updated
|
|
2767
3403
|
WHERE id=$id AND status='running'
|
|
2768
3404
|
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
2769
3405
|
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
2770
|
-
))`).run({
|
|
3406
|
+
))`).run({
|
|
3407
|
+
$id: id,
|
|
3408
|
+
$pid: pid,
|
|
3409
|
+
$processStartedAt: processStartedAt,
|
|
3410
|
+
$updated: now,
|
|
3411
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3412
|
+
$now: now
|
|
3413
|
+
});
|
|
2771
3414
|
if (res.changes !== 1)
|
|
2772
3415
|
return;
|
|
2773
3416
|
return this.getRun(id);
|
|
2774
3417
|
}
|
|
3418
|
+
recordRunProcess(runId, info, opts = {}) {
|
|
3419
|
+
const now = (opts.now ?? new Date).toISOString();
|
|
3420
|
+
const res = this.db.query(`UPDATE loop_runs SET pid=$pid, pgid=$pgid, process_started_at=$processStartedAt, updated_at=$updated
|
|
3421
|
+
WHERE id=$id AND status='running'
|
|
3422
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3423
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3424
|
+
))`).run({
|
|
3425
|
+
$id: runId,
|
|
3426
|
+
$pid: info.pid,
|
|
3427
|
+
$pgid: info.pgid ?? null,
|
|
3428
|
+
$processStartedAt: info.processStartedAt ?? isoProcessStart(info.pid) ?? now,
|
|
3429
|
+
$updated: now,
|
|
3430
|
+
$daemonLeaseId: opts.daemonLeaseId ?? null,
|
|
3431
|
+
$now: now
|
|
3432
|
+
});
|
|
3433
|
+
if (res.changes !== 1)
|
|
3434
|
+
return;
|
|
3435
|
+
return this.getRun(runId);
|
|
3436
|
+
}
|
|
2775
3437
|
hasLiveWorkflowStepProcesses(loopRunId) {
|
|
2776
|
-
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid
|
|
3438
|
+
const liveWorkflowSteps = this.db.query(`SELECT wr.id AS workflow_run_id, wsr.step_id AS step_id, wsr.pid AS pid, wsr.started_at AS started_at
|
|
2777
3439
|
FROM workflow_runs wr
|
|
2778
3440
|
JOIN workflow_step_runs wsr ON wsr.workflow_run_id = wr.id
|
|
2779
3441
|
WHERE wr.loop_run_id = ?
|
|
2780
3442
|
AND wr.status NOT IN ('succeeded', 'failed', 'timed_out', 'cancelled')
|
|
2781
3443
|
AND wsr.status = 'running'
|
|
2782
3444
|
AND wsr.pid IS NOT NULL`).all(loopRunId);
|
|
2783
|
-
return liveWorkflowSteps.some((step) =>
|
|
3445
|
+
return liveWorkflowSteps.some((step) => isLiveStepProcess(step.pid, step.started_at));
|
|
2784
3446
|
}
|
|
2785
3447
|
createSkippedRun(loop, scheduledFor, reason, opts = {}) {
|
|
2786
3448
|
const now = nowIso();
|
|
@@ -2854,7 +3516,7 @@ class Store {
|
|
|
2854
3516
|
const existing = this.getRunBySlot(loop.id, scheduledFor);
|
|
2855
3517
|
if (existing) {
|
|
2856
3518
|
if (existing.status === "running") {
|
|
2857
|
-
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && existing.pid
|
|
3519
|
+
if (existing.leaseExpiresAt && existing.leaseExpiresAt <= startedAt && isRecordedProcessAlive(existing.pid, existing.processStartedAt)) {
|
|
2858
3520
|
this.db.exec("COMMIT");
|
|
2859
3521
|
return;
|
|
2860
3522
|
}
|
|
@@ -2863,7 +3525,7 @@ class Store {
|
|
|
2863
3525
|
return;
|
|
2864
3526
|
}
|
|
2865
3527
|
const res3 = this.db.query(`UPDATE loop_runs SET status='running', started_at=$started, finished_at=NULL,
|
|
2866
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3528
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2867
3529
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2868
3530
|
WHERE id=$id AND status='running' AND lease_expires_at <= $now`).run({
|
|
2869
3531
|
$id: existing.id,
|
|
@@ -2885,7 +3547,7 @@ class Store {
|
|
|
2885
3547
|
}
|
|
2886
3548
|
const attempt = existing.attempt + 1;
|
|
2887
3549
|
const res2 = this.db.query(`UPDATE loop_runs SET attempt=$attempt, status='running', started_at=$started, finished_at=NULL,
|
|
2888
|
-
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, exit_code=NULL,
|
|
3550
|
+
claimed_by=$claimedBy, lease_expires_at=$lease, pid=NULL, pgid=NULL, process_started_at=NULL, exit_code=NULL,
|
|
2889
3551
|
duration_ms=NULL, stdout=NULL, stderr=NULL, error=NULL, updated_at=$updated
|
|
2890
3552
|
WHERE id=$id
|
|
2891
3553
|
AND status IN ('failed', 'timed_out', 'abandoned')
|
|
@@ -2933,6 +3595,7 @@ class Store {
|
|
|
2933
3595
|
}
|
|
2934
3596
|
finalizeRun(id, patch, opts = {}) {
|
|
2935
3597
|
const finishedAt = patch.finishedAt ?? nowIso();
|
|
3598
|
+
const error = patch.error === undefined ? undefined : scrubSecrets(patch.error);
|
|
2936
3599
|
const params = {
|
|
2937
3600
|
$id: id,
|
|
2938
3601
|
$status: patch.status,
|
|
@@ -2940,41 +3603,43 @@ class Store {
|
|
|
2940
3603
|
$pid: patch.pid ?? null,
|
|
2941
3604
|
$exitCode: patch.exitCode ?? null,
|
|
2942
3605
|
$durationMs: patch.durationMs ?? null,
|
|
2943
|
-
$stdout: patch.stdout
|
|
2944
|
-
$stderr: patch.stderr
|
|
2945
|
-
$error:
|
|
3606
|
+
$stdout: scrubbedOrNull(patch.stdout),
|
|
3607
|
+
$stderr: scrubbedOrNull(patch.stderr),
|
|
3608
|
+
$error: error ?? null,
|
|
2946
3609
|
$updated: finishedAt,
|
|
2947
3610
|
$claimedBy: opts.claimedBy ?? null,
|
|
2948
3611
|
$now: (opts.now ?? new Date).toISOString(),
|
|
2949
3612
|
$daemonLeaseId: opts.daemonLeaseId ?? null
|
|
2950
3613
|
};
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
AND
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
3614
|
+
return this.transact(() => {
|
|
3615
|
+
const res = opts.claimedBy ? this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3616
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated
|
|
3617
|
+
WHERE id=$id AND status='running' AND claimed_by=$claimedBy AND lease_expires_at > $now
|
|
3618
|
+
AND ($daemonLeaseId IS NULL OR EXISTS (
|
|
3619
|
+
SELECT 1 FROM daemon_lease WHERE id=$daemonLeaseId AND expires_at > $now
|
|
3620
|
+
))`).run(params) : this.db.query(`UPDATE loop_runs SET status=$status, finished_at=$finished, lease_expires_at=NULL, pid=$pid, exit_code=$exitCode,
|
|
3621
|
+
duration_ms=$durationMs, stdout=$stdout, stderr=$stderr, error=$error, updated_at=$updated WHERE id=$id`).run(params);
|
|
3622
|
+
const run = this.getRun(id);
|
|
3623
|
+
if (!run)
|
|
3624
|
+
throw new Error(`run not found after finalize: ${id}`);
|
|
3625
|
+
if (opts.claimedBy && res.changes !== 1)
|
|
3626
|
+
return run;
|
|
3627
|
+
if (res.changes === 1) {
|
|
3628
|
+
this.setWorkflowWorkItemsForLoopRun(run, error, finishedAt);
|
|
3629
|
+
const loop = this.getLoop(run.loopId);
|
|
3630
|
+
const itemStatus = workItemStatusForLoopRun(run.status, run.attempt, loop?.maxAttempts);
|
|
3631
|
+
if (loop?.target.type === "workflow" && itemStatus && itemStatus !== "admitted") {
|
|
3632
|
+
const workItemId = loop.target.input?.workflowWorkItemId ?? loop.target.input?.workItemId;
|
|
3633
|
+
this.maybeArchiveGeneratedRouteWorkflow({
|
|
3634
|
+
workflowId: loop.target.workflowId,
|
|
3635
|
+
loopId: loop.id,
|
|
3636
|
+
workItemId,
|
|
3637
|
+
updated: finishedAt
|
|
3638
|
+
});
|
|
3639
|
+
}
|
|
2975
3640
|
}
|
|
2976
|
-
|
|
2977
|
-
|
|
3641
|
+
return run;
|
|
3642
|
+
});
|
|
2978
3643
|
}
|
|
2979
3644
|
heartbeatRunLease(id, claimedBy, leaseMs, now = new Date, opts = {}) {
|
|
2980
3645
|
const expiresAt = new Date(now.getTime() + leaseMs).toISOString();
|
|
@@ -3024,6 +3689,9 @@ class Store {
|
|
|
3024
3689
|
});
|
|
3025
3690
|
}
|
|
3026
3691
|
recoverExpiredRunLeases(now = new Date, opts = {}) {
|
|
3692
|
+
return this.recoverExpiredRunLeasesDetailed(now, opts).abandoned;
|
|
3693
|
+
}
|
|
3694
|
+
recoverExpiredRunLeasesDetailed(now = new Date, opts = {}) {
|
|
3027
3695
|
const limit = Math.max(1, Math.min(1000, Math.floor(opts.limit ?? DEFAULT_RECOVERY_BATCH_LIMIT)));
|
|
3028
3696
|
const scanLimit = Math.max(limit, Math.min(5000, Math.floor(opts.scanLimit ?? limit * DEFAULT_RECOVERY_SCAN_MULTIPLIER)));
|
|
3029
3697
|
const rows = this.db.query(`SELECT * FROM loop_runs
|
|
@@ -3031,15 +3699,15 @@ class Store {
|
|
|
3031
3699
|
ORDER BY lease_expires_at ASC
|
|
3032
3700
|
LIMIT ?`).all(now.toISOString(), scanLimit);
|
|
3033
3701
|
const recovered = [];
|
|
3702
|
+
const deferred = [];
|
|
3034
3703
|
for (const row of rows) {
|
|
3035
3704
|
if (recovered.length >= limit)
|
|
3036
3705
|
break;
|
|
3037
|
-
if (row.pid
|
|
3038
|
-
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3039
|
-
continue;
|
|
3040
|
-
}
|
|
3041
|
-
if (this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3706
|
+
if (isRecordedProcessAlive(row.pid, row.process_started_at) || this.hasLiveWorkflowStepProcesses(row.id)) {
|
|
3042
3707
|
this.deferLiveExpiredRun(row.id, now, opts);
|
|
3708
|
+
const deferredRun = this.getRun(row.id);
|
|
3709
|
+
if (deferredRun)
|
|
3710
|
+
deferred.push(deferredRun);
|
|
3043
3711
|
continue;
|
|
3044
3712
|
}
|
|
3045
3713
|
const finished = now.toISOString();
|
|
@@ -3123,7 +3791,7 @@ class Store {
|
|
|
3123
3791
|
if (run)
|
|
3124
3792
|
recovered.push(run);
|
|
3125
3793
|
}
|
|
3126
|
-
return recovered;
|
|
3794
|
+
return { abandoned: recovered, deferred };
|
|
3127
3795
|
}
|
|
3128
3796
|
expireLoops(now = new Date, opts = {}) {
|
|
3129
3797
|
const rows = this.db.query("SELECT * FROM loops WHERE status = 'active' AND archived_at IS NULL AND expires_at IS NOT NULL AND expires_at <= ?").all(now.toISOString());
|
|
@@ -3156,6 +3824,83 @@ class Store {
|
|
|
3156
3824
|
const row = status ? this.db.query("SELECT COUNT(*) AS count FROM loop_runs WHERE status = ?").get(status) : this.db.query("SELECT COUNT(*) AS count FROM loop_runs").get();
|
|
3157
3825
|
return row?.count ?? 0;
|
|
3158
3826
|
}
|
|
3827
|
+
pruneHistory(opts) {
|
|
3828
|
+
const { maxAgeDays, keepPerLoop } = opts;
|
|
3829
|
+
if (maxAgeDays === undefined && keepPerLoop === undefined) {
|
|
3830
|
+
throw new ValidationError("pruneHistory requires maxAgeDays and/or keepPerLoop");
|
|
3831
|
+
}
|
|
3832
|
+
if (maxAgeDays !== undefined && (!Number.isFinite(maxAgeDays) || maxAgeDays < 0)) {
|
|
3833
|
+
throw new ValidationError(`pruneHistory maxAgeDays must be a non-negative number: ${maxAgeDays}`);
|
|
3834
|
+
}
|
|
3835
|
+
if (keepPerLoop !== undefined && (!Number.isInteger(keepPerLoop) || keepPerLoop < 0)) {
|
|
3836
|
+
throw new ValidationError(`pruneHistory keepPerLoop must be a non-negative integer: ${keepPerLoop}`);
|
|
3837
|
+
}
|
|
3838
|
+
const now = opts.now ?? new Date;
|
|
3839
|
+
const dryRun = opts.dryRun ?? false;
|
|
3840
|
+
const cutoff = maxAgeDays === undefined ? undefined : new Date(now.getTime() - maxAgeDays * 86400000).toISOString();
|
|
3841
|
+
const terminal = TERMINAL_RUN_STATUSES.map((status) => `'${status}'`).join(",");
|
|
3842
|
+
const candidateIds = this.db.query(`WITH ranked AS (
|
|
3843
|
+
SELECT id, status, created_at,
|
|
3844
|
+
ROW_NUMBER() OVER (PARTITION BY loop_id ORDER BY created_at DESC, id DESC) AS recency
|
|
3845
|
+
FROM loop_runs
|
|
3846
|
+
)
|
|
3847
|
+
SELECT id FROM ranked
|
|
3848
|
+
WHERE status IN (${terminal})
|
|
3849
|
+
AND ($cutoff IS NULL OR created_at < $cutoff)
|
|
3850
|
+
AND ($keep IS NULL OR recency > $keep)`).all({ $cutoff: cutoff ?? null, $keep: keepPerLoop ?? null }).map((row) => row.id);
|
|
3851
|
+
const summary = {
|
|
3852
|
+
dryRun,
|
|
3853
|
+
cutoff,
|
|
3854
|
+
keepPerLoop,
|
|
3855
|
+
loopRuns: dryRun ? candidateIds.length : 0,
|
|
3856
|
+
workflowRuns: 0,
|
|
3857
|
+
goalRuns: 0
|
|
3858
|
+
};
|
|
3859
|
+
const manifestPaths = [];
|
|
3860
|
+
for (let offset = 0;offset < candidateIds.length; offset += PRUNE_BATCH_SIZE) {
|
|
3861
|
+
const batch = candidateIds.slice(offset, offset + PRUNE_BATCH_SIZE);
|
|
3862
|
+
const batchPlaceholders = batch.map(() => "?").join(",");
|
|
3863
|
+
if (dryRun) {
|
|
3864
|
+
const workflowRunIds = this.db.query(`SELECT id FROM workflow_runs WHERE loop_run_id IN (${batchPlaceholders})`).all(...batch).map((row) => row.id);
|
|
3865
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3866
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3867
|
+
summary.goalRuns += this.db.query(`SELECT COUNT(*) AS count FROM goal_runs
|
|
3868
|
+
WHERE loop_run_id IN (${batchPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).get(...batch, ...workflowRunIds)?.count ?? 0;
|
|
3869
|
+
continue;
|
|
3870
|
+
}
|
|
3871
|
+
this.transact(() => {
|
|
3872
|
+
const confirmed = this.db.query(`SELECT id FROM loop_runs WHERE id IN (${batchPlaceholders}) AND status IN (${terminal})`).all(...batch).map((row) => row.id);
|
|
3873
|
+
if (confirmed.length === 0)
|
|
3874
|
+
return;
|
|
3875
|
+
const runPlaceholders = confirmed.map(() => "?").join(",");
|
|
3876
|
+
const workflowRuns = this.db.query(`SELECT id, manifest_path FROM workflow_runs WHERE loop_run_id IN (${runPlaceholders})`).all(...confirmed);
|
|
3877
|
+
const workflowRunIds = workflowRuns.map((row) => row.id);
|
|
3878
|
+
const workflowPlaceholders = workflowRunIds.map(() => "?").join(",") || "''";
|
|
3879
|
+
summary.loopRuns += confirmed.length;
|
|
3880
|
+
summary.workflowRuns += workflowRunIds.length;
|
|
3881
|
+
summary.goalRuns += this.db.query(`DELETE FROM goal_runs WHERE loop_run_id IN (${runPlaceholders}) OR workflow_run_id IN (${workflowPlaceholders})`).run(...confirmed, ...workflowRunIds).changes;
|
|
3882
|
+
if (workflowRunIds.length > 0) {
|
|
3883
|
+
this.db.query(`DELETE FROM workflow_runs WHERE id IN (${workflowPlaceholders})`).run(...workflowRunIds);
|
|
3884
|
+
}
|
|
3885
|
+
this.db.query(`DELETE FROM loop_runs WHERE id IN (${runPlaceholders}) AND status IN (${terminal})`).run(...confirmed);
|
|
3886
|
+
for (const row of workflowRuns) {
|
|
3887
|
+
if (row.manifest_path)
|
|
3888
|
+
manifestPaths.push(row.manifest_path);
|
|
3889
|
+
}
|
|
3890
|
+
});
|
|
3891
|
+
}
|
|
3892
|
+
for (const manifestPath of manifestPaths) {
|
|
3893
|
+
const runDir = dirname2(manifestPath);
|
|
3894
|
+
try {
|
|
3895
|
+
if (/^[0-9a-f]{12,64}$/.test(basename2(runDir))) {
|
|
3896
|
+
rmSync2(runDir, { recursive: true, force: true });
|
|
3897
|
+
} else {
|
|
3898
|
+
rmSync2(manifestPath, { force: true });
|
|
3899
|
+
}
|
|
3900
|
+
} catch {}
|
|
3901
|
+
}
|
|
3902
|
+
return summary;
|
|
3903
|
+
}
|
|
3159
3904
|
acquireDaemonLease(input) {
|
|
3160
3905
|
const now = input.now ?? new Date;
|
|
3161
3906
|
const expiresAt = new Date(now.getTime() + input.ttlMs).toISOString();
|