@sweny-ai/core 0.1.172 → 0.1.173
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-env.js +6 -1
- package/dist/budget.d.ts +8 -0
- package/dist/budget.js +11 -0
- package/dist/cli/final-output.d.ts +26 -6
- package/dist/cli/final-output.js +64 -15
- package/dist/cli/main.js +12 -4
- package/dist/cli/resume.js +10 -2
- package/dist/executor.js +42 -4
- package/dist/harness/sandbox-wrapper.js +9 -1
- package/dist/journal.d.ts +114 -17
- package/dist/journal.js +463 -36
- package/dist/schema.d.ts +97 -97
- package/package.json +1 -1
package/dist/journal.js
CHANGED
|
@@ -2,10 +2,23 @@
|
|
|
2
2
|
* Run journal (#363): an append-only log of one workflow run, enough to resume
|
|
3
3
|
* a killed or failed run without re-running the nodes that already finished.
|
|
4
4
|
*
|
|
5
|
-
* File: `.sweny/runs/<run-id>/journal.ndjson
|
|
6
|
-
* record carries the format version (`v`), a sequence number (`seq`)
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* File: `.sweny/runs/<run-id>/journal.ndjson` (mode 0600), one JSON record per
|
|
6
|
+
* line. Every record carries the format version (`v`), a sequence number (`seq`)
|
|
7
|
+
* and an authentication code (`h`, HMAC-SHA256 of the record without `h`, under
|
|
8
|
+
* a per-run key). Each append is written and fsync'd before the executor moves
|
|
9
|
+
* on, so a record on disk is a fact.
|
|
10
|
+
*
|
|
11
|
+
* Integrity: the key is 32 random bytes made when the run starts and kept
|
|
12
|
+
* OUTSIDE the workspace, in the user's sweny state dir
|
|
13
|
+
* (`$SWENY_STATE_DIR`, else `$XDG_STATE_HOME/sweny`, else
|
|
14
|
+
* `~/.local/state/sweny`), under `run-keys/` (dir 0700, file 0600). Anything
|
|
15
|
+
* that can write the workspace but not that dir (an agent confined to the
|
|
16
|
+
* workspace, a checked-out branch, an artifact) cannot mint a record resume
|
|
17
|
+
* accepts. Both sandboxes deny reading the key dir; an unsandboxed agent
|
|
18
|
+
* running as the same OS user can still read it, which is why resume also
|
|
19
|
+
* checks the record sequence (see {@link buildResumePlan}) and every replayed
|
|
20
|
+
* route against the workflow's real edges. A resume on another machine needs
|
|
21
|
+
* the same state dir.
|
|
9
22
|
*
|
|
10
23
|
* Records, in run order:
|
|
11
24
|
* run:start workflow / instruction / input / tool hashes, harness id, input (redacted)
|
|
@@ -17,8 +30,12 @@
|
|
|
17
30
|
* output:applied the write returned (key + the ids it produced)
|
|
18
31
|
* node:end final result of the visit and the write-stage counters
|
|
19
32
|
* route the edge taken after the visit (null = the run ended)
|
|
33
|
+
* usage spend of one agent attempt (cumulative; live while it runs, final when it returns)
|
|
20
34
|
* run:end success | failed | crashed
|
|
21
35
|
*
|
|
36
|
+
* Spend: a resume seeds the run budget with every attempt's journaled usage,
|
|
37
|
+
* so a crash never resets the whole-run ceiling.
|
|
38
|
+
*
|
|
22
39
|
* Recovery: a torn or corrupt tail (power loss mid-append) is truncated back to
|
|
23
40
|
* the last valid record. Corruption in the middle of the file is refused.
|
|
24
41
|
*
|
|
@@ -37,10 +54,13 @@
|
|
|
37
54
|
*/
|
|
38
55
|
import crypto from "node:crypto";
|
|
39
56
|
import fs from "node:fs";
|
|
57
|
+
import os from "node:os";
|
|
40
58
|
import path from "node:path";
|
|
41
59
|
import { resolveNodePermissions } from "./node-policy.js";
|
|
42
60
|
import { CURRENT_SPEC_VERSION } from "./migrations.js";
|
|
43
|
-
|
|
61
|
+
import { spendOf } from "./budget.js";
|
|
62
|
+
/** v2: records are authenticated with a per-run HMAC key (v1 used a public checksum and is refused). */
|
|
63
|
+
export const JOURNAL_SCHEMA_VERSION = 2;
|
|
44
64
|
export const JOURNAL_DIR = path.join(".sweny", "runs");
|
|
45
65
|
export const JOURNAL_FILE = "journal.ndjson";
|
|
46
66
|
const LOCK_FILE = "journal.lock";
|
|
@@ -82,19 +102,109 @@ export function workflowHashOf(workflow) {
|
|
|
82
102
|
export function instructionHash(sources) {
|
|
83
103
|
return canonicalHash(Object.fromEntries(Object.entries(sources).map(([k, v]) => [k, v?.content ?? ""])));
|
|
84
104
|
}
|
|
85
|
-
|
|
86
|
-
|
|
105
|
+
const sha256 = (s) => crypto.createHash("sha256").update(s).digest("hex");
|
|
106
|
+
/**
|
|
107
|
+
* Fingerprint of everything that decides what the run's tools do: per skill its
|
|
108
|
+
* id, name, description, category, instruction, config field names and env var
|
|
109
|
+
* NAMES, every tool's name, description, input schema, access class and a
|
|
110
|
+
* digest of its handler source, the MCP server's type, command, args, url,
|
|
111
|
+
* env var NAMES and header NAMES, and its tool aliases; plus the sweny version
|
|
112
|
+
* (which pins built-in skill code) and the harness. Secret values never enter
|
|
113
|
+
* it: MCP env and header values are dropped, and args are redacted first, so
|
|
114
|
+
* rotating a token neither changes the fingerprint nor leaves a hash of it.
|
|
115
|
+
*/
|
|
116
|
+
export function toolsHash(skills, swenyVersion, harnessId, secrets = []) {
|
|
117
|
+
const names = (m) => (m ? Object.keys(m).sort() : null);
|
|
87
118
|
const shape = [...skills.values()]
|
|
88
119
|
.map((s) => ({
|
|
89
120
|
id: s.id,
|
|
90
|
-
|
|
91
|
-
|
|
121
|
+
name: s.name ?? null,
|
|
122
|
+
description: s.description ?? null,
|
|
123
|
+
category: s.category ?? null,
|
|
124
|
+
instruction: s.instruction ?? null,
|
|
125
|
+
config: Object.entries(s.config ?? {})
|
|
126
|
+
.map(([k, f]) => ({ key: k, env: f?.env ?? null, required: f?.required === true }))
|
|
127
|
+
.sort((a, b) => a.key.localeCompare(b.key)),
|
|
128
|
+
tools: (s.tools ?? [])
|
|
129
|
+
.map((t) => ({
|
|
130
|
+
name: t.name,
|
|
131
|
+
description: t.description ?? null,
|
|
132
|
+
schema: t.input_schema ?? null,
|
|
133
|
+
access: t.access ?? "unclassified",
|
|
134
|
+
impl: typeof t.handler === "function" ? sha256(t.handler.toString()) : null,
|
|
135
|
+
}))
|
|
136
|
+
.sort((a, b) => a.name.localeCompare(b.name)),
|
|
137
|
+
mcp: s.mcp
|
|
138
|
+
? {
|
|
139
|
+
type: s.mcp.type ?? null,
|
|
140
|
+
command: s.mcp.command ?? null,
|
|
141
|
+
args: s.mcp.args ? redact(s.mcp.args, secrets).value : null,
|
|
142
|
+
url: s.mcp.url ? redact(s.mcp.url, secrets).value : null,
|
|
143
|
+
env: names(s.mcp.env),
|
|
144
|
+
headers: names(s.mcp.headers),
|
|
145
|
+
}
|
|
146
|
+
: null,
|
|
147
|
+
aliases: s.mcpAliases ?? null,
|
|
92
148
|
}))
|
|
93
149
|
.sort((a, b) => a.id.localeCompare(b.id));
|
|
94
150
|
return canonicalHash({ skills: shape, sweny: swenyVersion ?? null, harness: harnessId ?? null });
|
|
95
151
|
}
|
|
96
|
-
function
|
|
97
|
-
return crypto.
|
|
152
|
+
function recordMac(key, body) {
|
|
153
|
+
return crypto.createHmac("sha256", key).update(body).digest("hex");
|
|
154
|
+
}
|
|
155
|
+
// ─── Run keys ─────────────────────────────────────────────────────
|
|
156
|
+
/** sweny's per-user state dir: `$SWENY_STATE_DIR`, else `$XDG_STATE_HOME/sweny`, else `~/.local/state/sweny`. */
|
|
157
|
+
export function swenyStateDir(env = process.env) {
|
|
158
|
+
if (env.SWENY_STATE_DIR && env.SWENY_STATE_DIR.trim() !== "")
|
|
159
|
+
return path.resolve(env.SWENY_STATE_DIR);
|
|
160
|
+
const xdg = env.XDG_STATE_HOME;
|
|
161
|
+
const base = xdg && path.isAbsolute(xdg) ? xdg : path.join(os.homedir(), ".local", "state");
|
|
162
|
+
return path.join(base, "sweny");
|
|
163
|
+
}
|
|
164
|
+
/** Where run keys live. Never inside the workspace. */
|
|
165
|
+
export function runKeyDir(env = process.env) {
|
|
166
|
+
return path.join(swenyStateDir(env), "run-keys");
|
|
167
|
+
}
|
|
168
|
+
function realOr(p) {
|
|
169
|
+
try {
|
|
170
|
+
return fs.realpathSync(p);
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
return path.resolve(p);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/** The key file for one run in one workspace (the workspace path is hashed, not stored). */
|
|
177
|
+
export function runKeyFile(cwd, runId, keyDir = runKeyDir()) {
|
|
178
|
+
const scope = sha256(realOr(cwd)).slice(0, 16);
|
|
179
|
+
return path.join(keyDir, `${scope}-${runId}.key`);
|
|
180
|
+
}
|
|
181
|
+
/** Thrown when a journal's key is missing or unreadable: its records cannot be checked. */
|
|
182
|
+
export class JournalKeyError extends Error {
|
|
183
|
+
constructor(file, why) {
|
|
184
|
+
super(`cannot verify the run journal: its key ${file} is ${why}. A journal can only be resumed by the user ` +
|
|
185
|
+
`(and state dir) that started the run; set SWENY_STATE_DIR to that dir, or start a new run.`);
|
|
186
|
+
this.name = "JournalKeyError";
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
function createRunKey(file) {
|
|
190
|
+
const key = crypto.randomBytes(32);
|
|
191
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
192
|
+
fs.writeFileSync(file, key.toString("hex") + "\n", { mode: 0o600 });
|
|
193
|
+
fs.chmodSync(file, 0o600);
|
|
194
|
+
return key;
|
|
195
|
+
}
|
|
196
|
+
/** Read a run key. Throws {@link JournalKeyError}. */
|
|
197
|
+
export function loadRunKey(file) {
|
|
198
|
+
let text;
|
|
199
|
+
try {
|
|
200
|
+
text = fs.readFileSync(file, "utf-8").trim();
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
throw new JournalKeyError(file, "missing");
|
|
204
|
+
}
|
|
205
|
+
if (!/^[0-9a-f]{64}$/.test(text))
|
|
206
|
+
throw new JournalKeyError(file, "not a valid key");
|
|
207
|
+
return Buffer.from(text, "hex");
|
|
98
208
|
}
|
|
99
209
|
// ─── Redaction ────────────────────────────────────────────────────
|
|
100
210
|
const SECRET_KEY_PARTS = [
|
|
@@ -134,6 +244,20 @@ export function collectSecretValues(...maps) {
|
|
|
134
244
|
}
|
|
135
245
|
return [...out].sort((a, b) => b.length - a.length);
|
|
136
246
|
}
|
|
247
|
+
/**
|
|
248
|
+
* The secret values a run can see: secret-named env vars, plus skill config
|
|
249
|
+
* fields (resolved from their env vars) whose config key looks secret. The
|
|
250
|
+
* same set the journal redacts, for anything else that prints node output.
|
|
251
|
+
*/
|
|
252
|
+
export function runSecretValues(skills, env = process.env) {
|
|
253
|
+
const config = {};
|
|
254
|
+
for (const s of skills ?? []) {
|
|
255
|
+
for (const [k, f] of Object.entries(s.config ?? {}))
|
|
256
|
+
if (f?.env)
|
|
257
|
+
config[k] = env[f.env];
|
|
258
|
+
}
|
|
259
|
+
return collectSecretValues(env, config);
|
|
260
|
+
}
|
|
137
261
|
/**
|
|
138
262
|
* Deep copy of `value` with secrets replaced by `[redacted]`: string values
|
|
139
263
|
* under secret-looking keys, known token shapes, and any occurrence of a value
|
|
@@ -183,13 +307,23 @@ export function redact(value, secrets = []) {
|
|
|
183
307
|
};
|
|
184
308
|
return { value: walk(value, 0), redacted };
|
|
185
309
|
}
|
|
186
|
-
/** Thrown when the journal was written by a newer sweny. */
|
|
310
|
+
/** Thrown when the journal was written by a newer sweny, or by an older one without record authentication. */
|
|
187
311
|
export class JournalVersionError extends Error {
|
|
188
312
|
constructor(version) {
|
|
189
|
-
super(
|
|
313
|
+
super(typeof version === "number" && version < JOURNAL_SCHEMA_VERSION
|
|
314
|
+
? `run journal format v${version} predates authenticated records (v${JOURNAL_SCHEMA_VERSION}); it cannot be ` +
|
|
315
|
+
`resumed safely. Start a new run.`
|
|
316
|
+
: `run journal format v${String(version)} is newer than this sweny understands (v${JOURNAL_SCHEMA_VERSION}); upgrade sweny to resume it`);
|
|
190
317
|
this.name = "JournalVersionError";
|
|
191
318
|
}
|
|
192
319
|
}
|
|
320
|
+
/** Thrown when the journal's records describe something the executor never writes: edited, spliced or forged. */
|
|
321
|
+
export class JournalIntegrityError extends Error {
|
|
322
|
+
constructor(message) {
|
|
323
|
+
super(`${message}. The journal was edited or does not come from this run; start a new run.`);
|
|
324
|
+
this.name = "JournalIntegrityError";
|
|
325
|
+
}
|
|
326
|
+
}
|
|
193
327
|
/** Thrown at resume when what the run depends on changed since it was journaled (and `force` is off). */
|
|
194
328
|
export class JournalMismatchError extends Error {
|
|
195
329
|
what;
|
|
@@ -207,7 +341,7 @@ export class JournalLockedError extends Error {
|
|
|
207
341
|
this.name = "JournalLockedError";
|
|
208
342
|
}
|
|
209
343
|
}
|
|
210
|
-
function verifyLine(line, expectedSeq) {
|
|
344
|
+
function verifyLine(line, expectedSeq, key) {
|
|
211
345
|
let parsed;
|
|
212
346
|
try {
|
|
213
347
|
parsed = JSON.parse(line);
|
|
@@ -218,15 +352,27 @@ function verifyLine(line, expectedSeq) {
|
|
|
218
352
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
219
353
|
return undefined;
|
|
220
354
|
const rec = parsed;
|
|
355
|
+
if (typeof rec.v === "number" && rec.v !== JOURNAL_SCHEMA_VERSION && expectedSeq === 1)
|
|
356
|
+
return "version";
|
|
221
357
|
if (typeof rec.v === "number" && rec.v > JOURNAL_SCHEMA_VERSION)
|
|
222
358
|
return "version";
|
|
223
|
-
if (rec.v !== JOURNAL_SCHEMA_VERSION || rec.
|
|
359
|
+
if (rec.v !== JOURNAL_SCHEMA_VERSION || typeof rec.type !== "string" || typeof rec.h !== "string")
|
|
224
360
|
return undefined;
|
|
225
361
|
const { h, ...body } = rec;
|
|
226
|
-
|
|
227
|
-
|
|
362
|
+
const expected = Buffer.from(recordMac(key(), JSON.stringify(body)), "hex");
|
|
363
|
+
const given = /^[0-9a-f]{64}$/.test(h) ? Buffer.from(h, "hex") : Buffer.alloc(0);
|
|
364
|
+
// A whole, well-formed line that fails its MAC is tampering, not a torn append.
|
|
365
|
+
if (given.length !== expected.length || !crypto.timingSafeEqual(given, expected))
|
|
366
|
+
return "forged";
|
|
367
|
+
if (rec.seq !== expectedSeq)
|
|
368
|
+
return "forged";
|
|
228
369
|
return rec;
|
|
229
370
|
}
|
|
371
|
+
/** `<cwd>/.sweny/runs/<run-id>/journal.ndjson` -> cwd and run id. */
|
|
372
|
+
function journalLocation(file) {
|
|
373
|
+
const runDir = path.dirname(path.resolve(file));
|
|
374
|
+
return { runId: path.basename(runDir), cwd: path.dirname(path.dirname(path.dirname(runDir))) };
|
|
375
|
+
}
|
|
230
376
|
function looksLikeRecord(text) {
|
|
231
377
|
try {
|
|
232
378
|
const p = JSON.parse(text);
|
|
@@ -237,18 +383,29 @@ function looksLikeRecord(text) {
|
|
|
237
383
|
}
|
|
238
384
|
}
|
|
239
385
|
/**
|
|
240
|
-
* Read a journal. With `repair`, a torn
|
|
241
|
-
*
|
|
242
|
-
*
|
|
386
|
+
* Read a journal. With `repair`, a torn tail is truncated on disk back to the
|
|
387
|
+
* last valid record. Records are authenticated with the run's key (`key`, else
|
|
388
|
+
* loaded from the state dir for the run the path names). Throws
|
|
389
|
+
* {@link JournalVersionError} for another format and {@link JournalKeyError}
|
|
390
|
+
* when the key is missing.
|
|
243
391
|
*/
|
|
244
392
|
export function readJournal(file, opts = {}) {
|
|
245
393
|
const buf = fs.readFileSync(file);
|
|
394
|
+
let key = opts.key;
|
|
395
|
+
const getKey = () => {
|
|
396
|
+
if (!key) {
|
|
397
|
+
const { cwd, runId } = journalLocation(file);
|
|
398
|
+
key = loadRunKey(runKeyFile(cwd, runId, opts.keyDir ?? runKeyDir()));
|
|
399
|
+
}
|
|
400
|
+
return key;
|
|
401
|
+
};
|
|
246
402
|
const records = [];
|
|
247
403
|
let offset = 0;
|
|
248
404
|
let validBytes = 0;
|
|
249
405
|
let line = 0;
|
|
250
406
|
let firstBad;
|
|
251
407
|
let corruptAtLine;
|
|
408
|
+
let forgedAtLine;
|
|
252
409
|
while (offset < buf.length) {
|
|
253
410
|
const nl = buf.indexOf(0x0a, offset);
|
|
254
411
|
line++;
|
|
@@ -260,10 +417,15 @@ export function readJournal(file, opts = {}) {
|
|
|
260
417
|
const text = buf.subarray(offset, nl).toString("utf-8");
|
|
261
418
|
offset = nl + 1;
|
|
262
419
|
if (firstBad === undefined) {
|
|
263
|
-
const rec = verifyLine(text, records.length + 1);
|
|
420
|
+
const rec = verifyLine(text, records.length + 1, getKey);
|
|
264
421
|
if (rec === "version")
|
|
265
422
|
throw new JournalVersionError(JSON.parse(text).v);
|
|
266
|
-
if (rec) {
|
|
423
|
+
if (rec === "forged") {
|
|
424
|
+
firstBad = line;
|
|
425
|
+
corruptAtLine = line;
|
|
426
|
+
forgedAtLine = line;
|
|
427
|
+
}
|
|
428
|
+
else if (rec) {
|
|
267
429
|
records.push(rec);
|
|
268
430
|
validBytes = offset;
|
|
269
431
|
}
|
|
@@ -280,14 +442,220 @@ export function readJournal(file, opts = {}) {
|
|
|
280
442
|
if (opts.repair && truncatedBytes > 0 && corruptAtLine === undefined) {
|
|
281
443
|
fs.truncateSync(file, validBytes);
|
|
282
444
|
}
|
|
283
|
-
return {
|
|
445
|
+
return {
|
|
446
|
+
file,
|
|
447
|
+
records,
|
|
448
|
+
truncatedBytes,
|
|
449
|
+
...(corruptAtLine !== undefined ? { corruptAtLine } : {}),
|
|
450
|
+
...(forgedAtLine !== undefined ? { forgedAtLine } : {}),
|
|
451
|
+
};
|
|
284
452
|
}
|
|
285
453
|
const visitKey = (node, iteration) => `${node}#${iteration}`;
|
|
286
|
-
|
|
454
|
+
const RECORD_TYPES = new Set([
|
|
455
|
+
"run:start",
|
|
456
|
+
"run:resume",
|
|
457
|
+
"node:start",
|
|
458
|
+
"node:checkpoint",
|
|
459
|
+
"output:intent",
|
|
460
|
+
"output:applied",
|
|
461
|
+
"node:end",
|
|
462
|
+
"route",
|
|
463
|
+
"usage",
|
|
464
|
+
"run:end",
|
|
465
|
+
]);
|
|
466
|
+
const isPosInt = (v) => typeof v === "number" && Number.isInteger(v) && v > 0;
|
|
467
|
+
const isName = (v) => typeof v === "string" && v.length > 0;
|
|
468
|
+
const isAmount = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0;
|
|
469
|
+
/**
|
|
470
|
+
* Check that the records are a sequence the executor can write: one run:start
|
|
471
|
+
* first; nothing but run:resume after run:end; a visit starts only where the
|
|
472
|
+
* last route pointed (or the entry), or re-starts an unfinished or failed
|
|
473
|
+
* visit after a resume; checkpoints, writes, usage and node:end only inside
|
|
474
|
+
* the open visit; one route per finished visit, from that visit's node.
|
|
475
|
+
* Throws {@link JournalIntegrityError}.
|
|
476
|
+
*/
|
|
477
|
+
export function checkRecordSequence(records) {
|
|
478
|
+
const bad = (r, why) => {
|
|
479
|
+
throw new JournalIntegrityError(`run journal record ${r.seq} (${r.type}) ${why}`);
|
|
480
|
+
};
|
|
481
|
+
let cur;
|
|
482
|
+
let next;
|
|
483
|
+
let closed = false;
|
|
484
|
+
let resumedSinceStart = false;
|
|
485
|
+
const visits = new Map();
|
|
486
|
+
const checkpointed = new Set();
|
|
487
|
+
const intentKeys = new Set();
|
|
488
|
+
const inVisit = (r) => {
|
|
489
|
+
if (!cur || cur.ended)
|
|
490
|
+
bad(r, "is outside any open node visit");
|
|
491
|
+
if (r.node !== cur.node || r.iteration !== cur.iteration) {
|
|
492
|
+
bad(r, `names ${String(r.node)}#${String(r.iteration)}, but the open visit is ${cur.node}#${cur.iteration}`);
|
|
493
|
+
}
|
|
494
|
+
};
|
|
495
|
+
records.forEach((r, i) => {
|
|
496
|
+
if (!RECORD_TYPES.has(r.type))
|
|
497
|
+
bad(r, "has an unknown type");
|
|
498
|
+
if (i === 0) {
|
|
499
|
+
if (r.type !== "run:start")
|
|
500
|
+
bad(r, "is not run:start");
|
|
501
|
+
if (!isName(r.run_id) || !isName(r.workflow_entry))
|
|
502
|
+
bad(r, "is missing the run id or entry node");
|
|
503
|
+
next = r.workflow_entry;
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
if (r.type === "run:start")
|
|
507
|
+
bad(r, "is a second run:start");
|
|
508
|
+
if (closed && r.type !== "run:resume")
|
|
509
|
+
bad(r, "comes after run:end");
|
|
510
|
+
switch (r.type) {
|
|
511
|
+
case "run:resume":
|
|
512
|
+
closed = false;
|
|
513
|
+
resumedSinceStart = true;
|
|
514
|
+
break;
|
|
515
|
+
case "node:start": {
|
|
516
|
+
if (!isName(r.node) || !isPosInt(r.iteration))
|
|
517
|
+
bad(r, "has no valid node or iteration");
|
|
518
|
+
const node = r.node;
|
|
519
|
+
const restart = resumedSinceStart &&
|
|
520
|
+
!!cur &&
|
|
521
|
+
cur.node === node &&
|
|
522
|
+
cur.iteration === r.iteration &&
|
|
523
|
+
(!cur.ended || (cur.failed && !cur.routed));
|
|
524
|
+
if (!restart) {
|
|
525
|
+
if (cur && !cur.ended)
|
|
526
|
+
bad(r, `starts ${node} while ${cur.node} is still running`);
|
|
527
|
+
if (cur && !cur.routed)
|
|
528
|
+
bad(r, `starts ${node} before ${cur.node} was routed`);
|
|
529
|
+
if (typeof next !== "string" || node !== next) {
|
|
530
|
+
bad(r, `starts ${node}, but the run was routed to ${next === null ? "the end" : String(next)}`);
|
|
531
|
+
}
|
|
532
|
+
if (r.iteration !== (visits.get(node) ?? 0) + 1)
|
|
533
|
+
bad(r, `starts ${node} at the wrong iteration`);
|
|
534
|
+
visits.set(node, r.iteration);
|
|
535
|
+
}
|
|
536
|
+
cur = { node, iteration: r.iteration, ended: false, failed: false, routed: false };
|
|
537
|
+
resumedSinceStart = false;
|
|
538
|
+
break;
|
|
539
|
+
}
|
|
540
|
+
case "node:checkpoint": {
|
|
541
|
+
inVisit(r);
|
|
542
|
+
const k = visitKey(r.node, r.iteration);
|
|
543
|
+
if (checkpointed.has(k))
|
|
544
|
+
bad(r, "checkpoints a visit twice");
|
|
545
|
+
checkpointed.add(k);
|
|
546
|
+
break;
|
|
547
|
+
}
|
|
548
|
+
case "output:intent":
|
|
549
|
+
case "output:applied":
|
|
550
|
+
inVisit(r);
|
|
551
|
+
if (!checkpointed.has(visitKey(r.node, r.iteration))) {
|
|
552
|
+
bad(r, "records a write before the visit's checkpoint");
|
|
553
|
+
}
|
|
554
|
+
if (!isName(r.key))
|
|
555
|
+
bad(r, "has no idempotency key");
|
|
556
|
+
if (r.type === "output:intent")
|
|
557
|
+
intentKeys.add(r.key);
|
|
558
|
+
else if (!intentKeys.has(r.key) && r.recovered !== true)
|
|
559
|
+
bad(r, "has no matching intent");
|
|
560
|
+
break;
|
|
561
|
+
case "usage":
|
|
562
|
+
inVisit(r);
|
|
563
|
+
if (!isAmount(r.tokens) || !isAmount(r.cost_usd) || !isPosInt(r.attempt + 1)) {
|
|
564
|
+
bad(r, "has invalid amounts");
|
|
565
|
+
}
|
|
566
|
+
break;
|
|
567
|
+
case "node:end": {
|
|
568
|
+
inVisit(r);
|
|
569
|
+
const result = r.result;
|
|
570
|
+
if (!result || typeof result !== "object" || typeof result.status !== "string")
|
|
571
|
+
bad(r, "has no result");
|
|
572
|
+
cur.ended = true;
|
|
573
|
+
cur.failed = result.status === "failed";
|
|
574
|
+
break;
|
|
575
|
+
}
|
|
576
|
+
case "route":
|
|
577
|
+
if (!cur || !cur.ended)
|
|
578
|
+
bad(r, "routes before any visit ended");
|
|
579
|
+
if (cur.routed)
|
|
580
|
+
bad(r, `routes ${cur.node} a second time`);
|
|
581
|
+
if (r.from !== cur.node)
|
|
582
|
+
bad(r, `routes from ${String(r.from)}, but the visit that ended is ${cur.node}`);
|
|
583
|
+
if (r.to !== null && !isName(r.to))
|
|
584
|
+
bad(r, "has no valid target");
|
|
585
|
+
cur.routed = true;
|
|
586
|
+
next = r.to;
|
|
587
|
+
break;
|
|
588
|
+
case "run:end":
|
|
589
|
+
if (!["success", "failed", "crashed"].includes(r.status))
|
|
590
|
+
bad(r, "has an unknown status");
|
|
591
|
+
closed = true;
|
|
592
|
+
break;
|
|
593
|
+
}
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
/**
|
|
597
|
+
* Check a journal against the workflow it is resumed with: every visited node
|
|
598
|
+
* exists, and every journaled route is a real edge that its `max_iterations`
|
|
599
|
+
* still allowed. Returns the first problem, or undefined.
|
|
600
|
+
*/
|
|
601
|
+
export function checkJournalAgainstWorkflow(records, workflow) {
|
|
602
|
+
const start = records[0];
|
|
603
|
+
if (start?.type === "run:start" && start.workflow_entry !== workflow.entry) {
|
|
604
|
+
return `the run started at node ${String(start.workflow_entry)}, but the workflow's entry is ${workflow.entry}`;
|
|
605
|
+
}
|
|
606
|
+
const taken = new Map();
|
|
607
|
+
for (const r of records) {
|
|
608
|
+
if (r.type === "node:start" && !workflow.nodes[r.node]) {
|
|
609
|
+
return `the journal visits node ${String(r.node)}, which this workflow does not have`;
|
|
610
|
+
}
|
|
611
|
+
if (r.type !== "route" || r.to === null)
|
|
612
|
+
continue;
|
|
613
|
+
const from = r.from;
|
|
614
|
+
const to = r.to;
|
|
615
|
+
const edge = workflow.edges.find((e) => e.from === from && e.to === to);
|
|
616
|
+
if (!edge)
|
|
617
|
+
return `the journal routes ${from} -> ${to}, which is not an edge of this workflow`;
|
|
618
|
+
const k = `${from}->${to}`;
|
|
619
|
+
const n = (taken.get(k) ?? 0) + 1;
|
|
620
|
+
taken.set(k, n);
|
|
621
|
+
if (edge.max_iterations !== undefined && n > edge.max_iterations) {
|
|
622
|
+
return `the journal routes ${from} -> ${to} ${n} times; the edge allows ${edge.max_iterations}`;
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
return undefined;
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Spend the journal shows: per agent attempt (of each run segment), the
|
|
629
|
+
* largest cumulative usage it reported, live or final, summed.
|
|
630
|
+
*/
|
|
631
|
+
function journaledSpend(records) {
|
|
632
|
+
const perAttempt = new Map();
|
|
633
|
+
let segment = 1;
|
|
634
|
+
for (const r of records) {
|
|
635
|
+
if (r.type === "run:resume")
|
|
636
|
+
segment++;
|
|
637
|
+
if (r.type !== "usage")
|
|
638
|
+
continue;
|
|
639
|
+
const k = `${segment}|${visitKey(r.node, r.iteration)}|${String(r.attempt)}`;
|
|
640
|
+
const prev = perAttempt.get(k) ?? { tokens: 0, costUsd: 0 };
|
|
641
|
+
perAttempt.set(k, {
|
|
642
|
+
tokens: Math.max(prev.tokens, r.tokens),
|
|
643
|
+
costUsd: Math.max(prev.costUsd, r.cost_usd),
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
const total = { tokens: 0, costUsd: 0 };
|
|
647
|
+
for (const s of perAttempt.values()) {
|
|
648
|
+
total.tokens += s.tokens;
|
|
649
|
+
total.costUsd += s.costUsd;
|
|
650
|
+
}
|
|
651
|
+
return total;
|
|
652
|
+
}
|
|
653
|
+
/** Work out what a resume replays and where it picks up. Throws {@link JournalIntegrityError} for an impossible sequence. */
|
|
287
654
|
export function buildResumePlan(records) {
|
|
288
655
|
const start = records.find((r) => r.type === "run:start");
|
|
289
656
|
if (!start)
|
|
290
657
|
throw new Error("run journal has no run:start record; nothing to resume");
|
|
658
|
+
checkRecordSequence(records);
|
|
291
659
|
const order = [];
|
|
292
660
|
const byKey = new Map();
|
|
293
661
|
const intents = new Map();
|
|
@@ -406,6 +774,7 @@ export function buildResumePlan(records) {
|
|
|
406
774
|
ends,
|
|
407
775
|
intents,
|
|
408
776
|
receipts,
|
|
777
|
+
priorSpend: journaledSpend(records),
|
|
409
778
|
};
|
|
410
779
|
}
|
|
411
780
|
/**
|
|
@@ -520,6 +889,8 @@ function slimOutput(o, depth = 0) {
|
|
|
520
889
|
}
|
|
521
890
|
return out;
|
|
522
891
|
}
|
|
892
|
+
/** Minimum gap between two live usage records of one attempt. Final reports are always written. */
|
|
893
|
+
export const USAGE_JOURNAL_INTERVAL_MS = 2000;
|
|
523
894
|
export function journalDir(cwd, runId) {
|
|
524
895
|
return path.join(cwd, JOURNAL_DIR, runId);
|
|
525
896
|
}
|
|
@@ -543,14 +914,15 @@ export function findJournalRun(ref, cwd = process.cwd()) {
|
|
|
543
914
|
const matches = runs.filter((r) => r.startsWith(ref));
|
|
544
915
|
return matches.length === 1 ? matches[0] : undefined;
|
|
545
916
|
}
|
|
546
|
-
/** Delete the oldest journals beyond `keep`, never `except`. Never throws. */
|
|
547
|
-
export function pruneJournals(cwd, keep = JOURNAL_KEEP, except) {
|
|
917
|
+
/** Delete the oldest journals (and their keys) beyond `keep`, never `except`. Never throws. */
|
|
918
|
+
export function pruneJournals(cwd, keep = JOURNAL_KEEP, except, keyDir = runKeyDir()) {
|
|
548
919
|
const runs = listJournalRuns(cwd).filter((r) => r !== except);
|
|
549
920
|
const extra = runs.slice(0, Math.max(0, runs.length - Math.max(0, keep - (except ? 1 : 0))));
|
|
550
921
|
let removed = 0;
|
|
551
922
|
for (const r of extra) {
|
|
552
923
|
try {
|
|
553
924
|
fs.rmSync(journalDir(cwd, r), { recursive: true, force: true });
|
|
925
|
+
fs.rmSync(runKeyFile(cwd, r, keyDir), { force: true });
|
|
554
926
|
removed++;
|
|
555
927
|
}
|
|
556
928
|
catch {
|
|
@@ -581,8 +953,14 @@ export class RunJournal {
|
|
|
581
953
|
seq = 0;
|
|
582
954
|
dead = false;
|
|
583
955
|
began = false;
|
|
956
|
+
closed = false;
|
|
584
957
|
secrets = [];
|
|
585
958
|
consumed = new Set();
|
|
959
|
+
/** Per-run HMAC key; lives only here and in the key file outside the workspace. */
|
|
960
|
+
key;
|
|
961
|
+
keyFile;
|
|
962
|
+
/** Last live usage record per attempt: time and spend, for throttling. */
|
|
963
|
+
lastUsage = new Map();
|
|
586
964
|
constructor(opts, resume) {
|
|
587
965
|
this.opts = opts;
|
|
588
966
|
this.runId = opts.runId;
|
|
@@ -590,6 +968,7 @@ export class RunJournal {
|
|
|
590
968
|
this.dir = journalDir(this.cwd, opts.runId);
|
|
591
969
|
this.file = path.join(this.dir, JOURNAL_FILE);
|
|
592
970
|
this.logger = opts.logger ?? quietLogger;
|
|
971
|
+
this.keyFile = runKeyFile(this.cwd, opts.runId, opts.keyDir ?? runKeyDir());
|
|
593
972
|
if (resume) {
|
|
594
973
|
this.resume = {
|
|
595
974
|
plan: resume.plan,
|
|
@@ -597,6 +976,7 @@ export class RunJournal {
|
|
|
597
976
|
allowRepeatWrites: resume.allowRepeatWrites === true,
|
|
598
977
|
};
|
|
599
978
|
this.seq = resume.read.records.length;
|
|
979
|
+
this.key = loadRunKey(this.keyFile);
|
|
600
980
|
}
|
|
601
981
|
}
|
|
602
982
|
/** A journal for a new run. Nothing touches disk until the executor calls `begin`. */
|
|
@@ -624,8 +1004,13 @@ export class RunJournal {
|
|
|
624
1004
|
if (err instanceof JournalLockedError)
|
|
625
1005
|
throw err;
|
|
626
1006
|
}
|
|
627
|
-
|
|
628
|
-
fs.writeFileSync(lock, String(process.pid));
|
|
1007
|
+
this.makeDir();
|
|
1008
|
+
fs.writeFileSync(lock, String(process.pid), { mode: 0o600 });
|
|
1009
|
+
}
|
|
1010
|
+
/** The run dir is private (0700); its parents are created as usual. */
|
|
1011
|
+
makeDir() {
|
|
1012
|
+
fs.mkdirSync(path.dirname(this.dir), { recursive: true });
|
|
1013
|
+
fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
|
|
629
1014
|
}
|
|
630
1015
|
releaseLock() {
|
|
631
1016
|
try {
|
|
@@ -638,6 +1023,12 @@ export class RunJournal {
|
|
|
638
1023
|
append(type, fields) {
|
|
639
1024
|
if (this.dead)
|
|
640
1025
|
return;
|
|
1026
|
+
if (!this.key) {
|
|
1027
|
+
// No key, no record resume could trust: stop journaling rather than write unauthenticated records.
|
|
1028
|
+
this.dead = true;
|
|
1029
|
+
this.logger.warn(` run journal: no signing key for this run; this run cannot be resumed`);
|
|
1030
|
+
return;
|
|
1031
|
+
}
|
|
641
1032
|
const body = {
|
|
642
1033
|
v: JOURNAL_SCHEMA_VERSION,
|
|
643
1034
|
seq: this.seq + 1,
|
|
@@ -645,7 +1036,7 @@ export class RunJournal {
|
|
|
645
1036
|
at: new Date().toISOString(),
|
|
646
1037
|
...fields,
|
|
647
1038
|
};
|
|
648
|
-
const record = { ...body, h:
|
|
1039
|
+
const record = { ...body, h: recordMac(this.key, JSON.stringify(body)) };
|
|
649
1040
|
if (this.opts.faults?.beforeAppend) {
|
|
650
1041
|
try {
|
|
651
1042
|
this.opts.faults.beforeAppend(record);
|
|
@@ -660,8 +1051,9 @@ export class RunJournal {
|
|
|
660
1051
|
}
|
|
661
1052
|
try {
|
|
662
1053
|
if (this.fd === undefined) {
|
|
663
|
-
|
|
664
|
-
this.fd = fs.openSync(this.file, "a");
|
|
1054
|
+
this.makeDir();
|
|
1055
|
+
this.fd = fs.openSync(this.file, "a", 0o600);
|
|
1056
|
+
fs.fchmodSync(this.fd, 0o600);
|
|
665
1057
|
}
|
|
666
1058
|
fs.writeSync(this.fd, JSON.stringify(record) + "\n");
|
|
667
1059
|
fs.fsyncSync(this.fd);
|
|
@@ -695,7 +1087,7 @@ export class RunJournal {
|
|
|
695
1087
|
this.secrets = collectSecretValues(this.opts.env, info.config);
|
|
696
1088
|
const workflowHash = workflowHashOf(info.workflow);
|
|
697
1089
|
const instrHash = instructionHash(info.sources);
|
|
698
|
-
const toolHash = toolsHash(info.skills, this.opts.swenyVersion, info.harnessId);
|
|
1090
|
+
const toolHash = toolsHash(info.skills, this.opts.swenyVersion, info.harnessId, this.secrets);
|
|
699
1091
|
if (this.resume) {
|
|
700
1092
|
const start = this.resume.plan.start;
|
|
701
1093
|
const changed = [];
|
|
@@ -732,7 +1124,7 @@ export class RunJournal {
|
|
|
732
1124
|
return;
|
|
733
1125
|
}
|
|
734
1126
|
try {
|
|
735
|
-
|
|
1127
|
+
this.makeDir();
|
|
736
1128
|
// Journals hold node output: keep them out of commits (an agent's `git add -A` included).
|
|
737
1129
|
fs.writeFileSync(path.join(this.dir, ".gitignore"), "*\n");
|
|
738
1130
|
this.takeLock();
|
|
@@ -740,7 +1132,14 @@ export class RunJournal {
|
|
|
740
1132
|
catch {
|
|
741
1133
|
// append() reports the failure
|
|
742
1134
|
}
|
|
743
|
-
|
|
1135
|
+
try {
|
|
1136
|
+
this.key = createRunKey(this.keyFile);
|
|
1137
|
+
}
|
|
1138
|
+
catch (err) {
|
|
1139
|
+
this.logger.warn(` run journal: could not write the run key under ${path.dirname(this.keyFile)} ` +
|
|
1140
|
+
`(${err instanceof Error ? err.message : String(err)}); set SWENY_STATE_DIR to a writable dir`);
|
|
1141
|
+
}
|
|
1142
|
+
pruneJournals(this.cwd, this.opts.keep ?? JOURNAL_KEEP, this.runId, this.opts.keyDir ?? runKeyDir());
|
|
744
1143
|
const input = redact(info.input, this.secrets);
|
|
745
1144
|
this.append("run:start", {
|
|
746
1145
|
run_id: this.runId,
|
|
@@ -814,10 +1213,38 @@ export class RunJournal {
|
|
|
814
1213
|
route(from, to) {
|
|
815
1214
|
this.append("route", { from, to });
|
|
816
1215
|
}
|
|
817
|
-
|
|
1216
|
+
usage(node, iteration, attempt, usage, final) {
|
|
1217
|
+
const s = spendOf(usage);
|
|
1218
|
+
if (s.tokens === undefined && s.costUsd === undefined)
|
|
1219
|
+
return;
|
|
1220
|
+
const tokens = s.tokens ?? 0;
|
|
1221
|
+
const costUsd = s.costUsd ?? 0;
|
|
1222
|
+
const k = `${visitKey(node, iteration)}|${attempt}`;
|
|
1223
|
+
const last = this.lastUsage.get(k);
|
|
1224
|
+
const now = Date.now();
|
|
1225
|
+
if (!final && last) {
|
|
1226
|
+
if (tokens <= last.tokens && costUsd <= last.costUsd)
|
|
1227
|
+
return;
|
|
1228
|
+
if (now - last.at < USAGE_JOURNAL_INTERVAL_MS)
|
|
1229
|
+
return;
|
|
1230
|
+
}
|
|
1231
|
+
if (final && last && tokens <= last.tokens && costUsd <= last.costUsd)
|
|
1232
|
+
return;
|
|
1233
|
+
this.lastUsage.set(k, {
|
|
1234
|
+
at: now,
|
|
1235
|
+
tokens: Math.max(tokens, last?.tokens ?? 0),
|
|
1236
|
+
costUsd: Math.max(costUsd, last?.costUsd ?? 0),
|
|
1237
|
+
});
|
|
1238
|
+
this.append("usage", { node, iteration, attempt, tokens, cost_usd: costUsd, final });
|
|
1239
|
+
}
|
|
1240
|
+
priorSpend() {
|
|
1241
|
+
return this.resume ? { ...this.resume.plan.priorSpend } : undefined;
|
|
1242
|
+
}
|
|
1243
|
+
/** Close the run: `run:end`, then release the lock. Safe to call twice (one run:end). */
|
|
818
1244
|
end(status) {
|
|
819
|
-
if (this.began)
|
|
1245
|
+
if (this.began && !this.closed)
|
|
820
1246
|
this.append("run:end", { status });
|
|
1247
|
+
this.closed = true;
|
|
821
1248
|
this.closeFd();
|
|
822
1249
|
this.releaseLock();
|
|
823
1250
|
}
|