@sweny-ai/core 0.1.207 → 0.1.208
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/run-history.d.ts +36 -0
- package/dist/cli/run-history.js +100 -9
- package/dist/cli/try-capture.d.ts +59 -0
- package/dist/cli/try-capture.js +157 -0
- package/dist/cli/try.d.ts +26 -4
- package/dist/cli/try.js +91 -10
- package/package.json +1 -1
|
@@ -16,6 +16,15 @@ import { type SpendKind } from "../budget.js";
|
|
|
16
16
|
export declare const RUN_HISTORY_SCHEMA_VERSION = 1;
|
|
17
17
|
export declare const RUN_HISTORY_DIR: string;
|
|
18
18
|
export declare const RUN_HISTORY_KEEP = 200;
|
|
19
|
+
/**
|
|
20
|
+
* Version of the optional detail fields (per-node `tool_counts`, `tokens_in`, `tokens_out`,
|
|
21
|
+
* `num_turns`, and the record's `policy`), stamped as `detail_version` on records that carry them.
|
|
22
|
+
* `schema_version` stays 1: older records have no detail and read as such, and older readers ignore it.
|
|
23
|
+
*/
|
|
24
|
+
export declare const RUN_DETAIL_VERSION = 1;
|
|
25
|
+
/** At most this many distinct tool names are kept per node, each cut to {@link TOOL_NAME_MAX} characters. */
|
|
26
|
+
export declare const TOOL_COUNTS_MAX = 64;
|
|
27
|
+
export declare const TOOL_NAME_MAX = 80;
|
|
19
28
|
export type RunStatus = "success" | "failed" | "crashed";
|
|
20
29
|
export interface RunNodeRecord {
|
|
21
30
|
id: string;
|
|
@@ -27,6 +36,25 @@ export interface RunNodeRecord {
|
|
|
27
36
|
/** SDK-reported cost; null when none was reported (never estimated). */
|
|
28
37
|
cost_usd: number | null;
|
|
29
38
|
retries: number;
|
|
39
|
+
/**
|
|
40
|
+
* Calls per tool name (names and counts only, never inputs or outputs). Optional: absent on records
|
|
41
|
+
* written before `detail_version` 1, and when the node made no tool call.
|
|
42
|
+
*/
|
|
43
|
+
tool_counts?: Record<string, number>;
|
|
44
|
+
/** Input tokens, when the node reported usage. Optional (detail_version 1). */
|
|
45
|
+
tokens_in?: number;
|
|
46
|
+
/** Output tokens, when the node reported usage. Optional (detail_version 1). */
|
|
47
|
+
tokens_out?: number;
|
|
48
|
+
/** Model turns, when the node reported them. Optional (detail_version 1). */
|
|
49
|
+
num_turns?: number;
|
|
50
|
+
}
|
|
51
|
+
/** The agent floor the run ran under: booleans and a mode only. Optional (detail_version 1). */
|
|
52
|
+
export interface RunPolicyRecord {
|
|
53
|
+
/** The agent subprocess env was narrowed to the allowlist on every node. */
|
|
54
|
+
scoped: boolean;
|
|
55
|
+
sandbox: "off" | "auto" | "strict";
|
|
56
|
+
/** The sandbox was enabled for every node that reported it. */
|
|
57
|
+
sandboxed: boolean;
|
|
30
58
|
}
|
|
31
59
|
export interface RunTotals {
|
|
32
60
|
nodes_total: number;
|
|
@@ -65,6 +93,10 @@ export interface RunRecord {
|
|
|
65
93
|
* jump in `totals` that no node explains shows here.
|
|
66
94
|
*/
|
|
67
95
|
spend?: Partial<Record<SpendKind, RunKindSpend>>;
|
|
96
|
+
/** 1 when the optional detail fields below and on nodes are recorded. Absent on older records. */
|
|
97
|
+
detail_version?: number;
|
|
98
|
+
/** Absent when no node reported its floor. */
|
|
99
|
+
policy?: RunPolicyRecord;
|
|
68
100
|
}
|
|
69
101
|
export interface RunKindSpend {
|
|
70
102
|
tokens: number;
|
|
@@ -84,6 +116,10 @@ export interface NodeTimer {
|
|
|
84
116
|
lastResults: Map<string, NodeResult>;
|
|
85
117
|
}
|
|
86
118
|
export declare function createNodeTimer(now?: () => number): NodeTimer;
|
|
119
|
+
/** Calls per tool name, in first-seen order. Names are cut and capped; undefined when there are none. */
|
|
120
|
+
export declare function countTools(calls: ReadonlyArray<{
|
|
121
|
+
tool: string;
|
|
122
|
+
}> | undefined): Record<string, number> | undefined;
|
|
87
123
|
export interface BuildRunRecordInput {
|
|
88
124
|
runId: string;
|
|
89
125
|
workflow: Workflow;
|
package/dist/cli/run-history.js
CHANGED
|
@@ -20,6 +20,15 @@ import { openNoFollow } from "../safe-file.js";
|
|
|
20
20
|
export const RUN_HISTORY_SCHEMA_VERSION = 1;
|
|
21
21
|
export const RUN_HISTORY_DIR = path.join(".sweny", "runs");
|
|
22
22
|
export const RUN_HISTORY_KEEP = 200;
|
|
23
|
+
/**
|
|
24
|
+
* Version of the optional detail fields (per-node `tool_counts`, `tokens_in`, `tokens_out`,
|
|
25
|
+
* `num_turns`, and the record's `policy`), stamped as `detail_version` on records that carry them.
|
|
26
|
+
* `schema_version` stays 1: older records have no detail and read as such, and older readers ignore it.
|
|
27
|
+
*/
|
|
28
|
+
export const RUN_DETAIL_VERSION = 1;
|
|
29
|
+
/** At most this many distinct tool names are kept per node, each cut to {@link TOOL_NAME_MAX} characters. */
|
|
30
|
+
export const TOOL_COUNTS_MAX = 64;
|
|
31
|
+
export const TOOL_NAME_MAX = 80;
|
|
23
32
|
const RECORD_READ_MAX_BYTES = 1024 * 1024;
|
|
24
33
|
// ── Hash + ids ──────────────────────────────────────────────────
|
|
25
34
|
function canonicalize(value) {
|
|
@@ -78,6 +87,29 @@ export function createNodeTimer(now = Date.now) {
|
|
|
78
87
|
}
|
|
79
88
|
// ── Build the record ────────────────────────────────────────────
|
|
80
89
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
90
|
+
/** Calls per tool name, in first-seen order. Names are cut and capped; undefined when there are none. */
|
|
91
|
+
export function countTools(calls) {
|
|
92
|
+
const counts = new Map();
|
|
93
|
+
for (const c of calls ?? []) {
|
|
94
|
+
if (typeof c?.tool !== "string")
|
|
95
|
+
continue;
|
|
96
|
+
// eslint-disable-next-line no-control-regex
|
|
97
|
+
const name = c.tool.replace(/[\u0000-\u001f\u007f]/g, "").slice(0, TOOL_NAME_MAX);
|
|
98
|
+
if (name === "")
|
|
99
|
+
continue;
|
|
100
|
+
if (!counts.has(name) && counts.size >= TOOL_COUNTS_MAX)
|
|
101
|
+
continue;
|
|
102
|
+
counts.set(name, (counts.get(name) ?? 0) + 1);
|
|
103
|
+
}
|
|
104
|
+
if (counts.size === 0)
|
|
105
|
+
return undefined;
|
|
106
|
+
const out = {};
|
|
107
|
+
// defineProperty: a tool named `__proto__` stays a plain key.
|
|
108
|
+
for (const [k, v] of counts) {
|
|
109
|
+
Object.defineProperty(out, k, { value: v, enumerable: true, writable: true, configurable: true });
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
81
113
|
export function buildRunRecord(i) {
|
|
82
114
|
const summary = summarizeRun(i.results, i.durationMs, i.crashed, i.trace);
|
|
83
115
|
const retries = new Map();
|
|
@@ -91,6 +123,7 @@ export function buildRunRecord(i) {
|
|
|
91
123
|
const tokens = hasTokens
|
|
92
124
|
? (isNum(u.inputTokens) ? u.inputTokens : 0) + (isNum(u.outputTokens) ? u.outputTokens : 0)
|
|
93
125
|
: null;
|
|
126
|
+
const toolCounts = countTools(r.toolCalls);
|
|
94
127
|
return {
|
|
95
128
|
id,
|
|
96
129
|
status: r.status,
|
|
@@ -99,6 +132,10 @@ export function buildRunRecord(i) {
|
|
|
99
132
|
tokens,
|
|
100
133
|
cost_usd: u && isNum(u.costUsd) ? u.costUsd : null,
|
|
101
134
|
retries: retries.get(id) ?? 0,
|
|
135
|
+
...(toolCounts ? { tool_counts: toolCounts } : {}),
|
|
136
|
+
...(u && isNum(u.inputTokens) ? { tokens_in: u.inputTokens } : {}),
|
|
137
|
+
...(u && isNum(u.outputTokens) ? { tokens_out: u.outputTokens } : {}),
|
|
138
|
+
...(u && isNum(u.numTurns) ? { num_turns: u.numTurns } : {}),
|
|
102
139
|
};
|
|
103
140
|
});
|
|
104
141
|
const harness = [...i.results.values()].find((r) => r.harness)?.harness;
|
|
@@ -124,6 +161,16 @@ export function buildRunRecord(i) {
|
|
|
124
161
|
...(harness ? { harness: { id: harness.id, version: harness.version } } : {}),
|
|
125
162
|
...(degraded.length > 0 ? { degraded } : {}),
|
|
126
163
|
...(summary.spend ? { spend: recordSpend(summary.spend) } : {}),
|
|
164
|
+
detail_version: RUN_DETAIL_VERSION,
|
|
165
|
+
...(summary.policy?.envScope !== undefined && summary.policy.sandbox
|
|
166
|
+
? {
|
|
167
|
+
policy: {
|
|
168
|
+
scoped: summary.policy.envScope,
|
|
169
|
+
sandbox: summary.policy.sandbox.mode,
|
|
170
|
+
sandboxed: summary.policy.sandbox.started,
|
|
171
|
+
},
|
|
172
|
+
}
|
|
173
|
+
: {}),
|
|
127
174
|
};
|
|
128
175
|
}
|
|
129
176
|
function recordSpend(spend) {
|
|
@@ -205,6 +252,29 @@ const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
|
205
252
|
const num = (v) => (typeof v === "number" && Number.isFinite(v) ? v : 0);
|
|
206
253
|
const numOrNull = (v) => (typeof v === "number" && Number.isFinite(v) ? v : null);
|
|
207
254
|
const RUNGS = new Set(["expr", "decider", "agent"]);
|
|
255
|
+
const SANDBOX_MODES = new Set(["off", "auto", "strict"]);
|
|
256
|
+
const count = (v) => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
|
|
257
|
+
/**
|
|
258
|
+
* Per-tool counts read back: own keys only, names within the writer's own limits (no control
|
|
259
|
+
* characters, at most TOOL_NAME_MAX long, at most TOOL_COUNTS_MAX of them), values non-negative
|
|
260
|
+
* integers. Anything else is dropped; undefined when nothing is left.
|
|
261
|
+
*/
|
|
262
|
+
function parseToolCounts(v) {
|
|
263
|
+
if (!isObj(v))
|
|
264
|
+
return undefined;
|
|
265
|
+
const out = {};
|
|
266
|
+
let kept = 0;
|
|
267
|
+
for (const [k, c] of Object.entries(v)) {
|
|
268
|
+
if (kept >= TOOL_COUNTS_MAX)
|
|
269
|
+
break;
|
|
270
|
+
// eslint-disable-next-line no-control-regex
|
|
271
|
+
if (k === "" || k.length > TOOL_NAME_MAX || /[\u0000-\u001f\u007f]/.test(k) || !count(c))
|
|
272
|
+
continue;
|
|
273
|
+
Object.defineProperty(out, k, { value: c, enumerable: true, writable: true, configurable: true });
|
|
274
|
+
kept++;
|
|
275
|
+
}
|
|
276
|
+
return kept > 0 ? out : undefined;
|
|
277
|
+
}
|
|
208
278
|
/**
|
|
209
279
|
* A typed, allow-listed copy of a stored record, or null when it is not a run record. History is
|
|
210
280
|
* agent-writable, so a record is never passed through: only the documented fields are copied, numbers
|
|
@@ -235,15 +305,23 @@ function parseRecord(v) {
|
|
|
235
305
|
status: v.status,
|
|
236
306
|
nodes: v.nodes
|
|
237
307
|
.filter((n) => isObj(n) && typeof n.id === "string" && typeof n.status === "string")
|
|
238
|
-
.map((n) =>
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
308
|
+
.map((n) => {
|
|
309
|
+
const toolCounts = parseToolCounts(n.tool_counts);
|
|
310
|
+
return {
|
|
311
|
+
id: n.id,
|
|
312
|
+
status: n.status,
|
|
313
|
+
duration_ms: num(n.duration_ms),
|
|
314
|
+
tool_calls: num(n.tool_calls),
|
|
315
|
+
tokens: numOrNull(n.tokens),
|
|
316
|
+
cost_usd: numOrNull(n.cost_usd),
|
|
317
|
+
retries: num(n.retries),
|
|
318
|
+
// detail_version 1 fields: optional, kept only when well-formed.
|
|
319
|
+
...(toolCounts ? { tool_counts: toolCounts } : {}),
|
|
320
|
+
...(count(n.tokens_in) ? { tokens_in: n.tokens_in } : {}),
|
|
321
|
+
...(count(n.tokens_out) ? { tokens_out: n.tokens_out } : {}),
|
|
322
|
+
...(count(n.num_turns) ? { num_turns: n.num_turns } : {}),
|
|
323
|
+
};
|
|
324
|
+
}),
|
|
247
325
|
routes: v.routes
|
|
248
326
|
.filter((e) => isObj(e) && typeof e.from === "string" && typeof e.to === "string")
|
|
249
327
|
.map((e) => ({
|
|
@@ -263,6 +341,19 @@ function parseRecord(v) {
|
|
|
263
341
|
if (isObj(v.harness) && typeof v.harness.id === "string" && typeof v.harness.version === "string") {
|
|
264
342
|
out.harness = { id: v.harness.id, version: v.harness.version };
|
|
265
343
|
}
|
|
344
|
+
if (count(v.detail_version) && v.detail_version >= 1)
|
|
345
|
+
out.detail_version = v.detail_version;
|
|
346
|
+
if (isObj(v.policy) &&
|
|
347
|
+
typeof v.policy.scoped === "boolean" &&
|
|
348
|
+
typeof v.policy.sandboxed === "boolean" &&
|
|
349
|
+
typeof v.policy.sandbox === "string" &&
|
|
350
|
+
SANDBOX_MODES.has(v.policy.sandbox)) {
|
|
351
|
+
out.policy = {
|
|
352
|
+
scoped: v.policy.scoped,
|
|
353
|
+
sandbox: v.policy.sandbox,
|
|
354
|
+
sandboxed: v.policy.sandboxed,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
266
357
|
if (Array.isArray(v.degraded))
|
|
267
358
|
out.degraded = v.degraded.filter((d) => typeof d === "string");
|
|
268
359
|
if (isObj(v.spend)) {
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `sweny try --capture <run>`: turn a recorded run into the `sweny try` fixture (#537).
|
|
3
|
+
*
|
|
4
|
+
* Input is what a real run already left in `.sweny/runs/`: the history record
|
|
5
|
+
* (`<run-id>.json`, run-history.ts) and the answers (`<run-id>/answers.json`,
|
|
6
|
+
* run-answers.ts). Output is a `TryFixture` with `illustrative: false`, so
|
|
7
|
+
* `sweny try` labels it "a replay of a real run on <date>".
|
|
8
|
+
*
|
|
9
|
+
* Nothing is invented. Durations, tool-call counts, token and cost totals, the
|
|
10
|
+
* harness, the policy floor and the answer all come from the record. A run that
|
|
11
|
+
* lacks them (written before `detail_version` 1), failed, or answered nothing is
|
|
12
|
+
* refused with the reason, never padded. The answer is redacted a second time
|
|
13
|
+
* here (the journal's redactor plus the run's secret values), so a token that
|
|
14
|
+
* slipped past the first pass does not reach a committed file.
|
|
15
|
+
*
|
|
16
|
+
* Deterministic: fixed key order, no clock, no randomness. The same run always
|
|
17
|
+
* yields the same bytes.
|
|
18
|
+
*/
|
|
19
|
+
import { type RunAnswers } from "./run-answers.js";
|
|
20
|
+
import { type RunRecord } from "./run-history.js";
|
|
21
|
+
import { type TryFixture } from "./try.js";
|
|
22
|
+
export interface CaptureOptions {
|
|
23
|
+
/** `owner/name` of the repo the run was on (a record does not hold it). */
|
|
24
|
+
sampleRepo: string;
|
|
25
|
+
/** Extra secret values to scrub from the answer. Default: none beyond the redactor's patterns. */
|
|
26
|
+
secrets?: string[];
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* The fixture for one recorded run. Throws, naming the reason, when the run cannot honestly be one:
|
|
30
|
+
* not a success, no detail (older record), no harness, an unknown template or node, or no answers.
|
|
31
|
+
*/
|
|
32
|
+
export declare function buildTryFixture(record: RunRecord, answers: RunAnswers | null, opts: CaptureOptions): TryFixture;
|
|
33
|
+
/** The committed bytes: two-space JSON, a trailing newline. */
|
|
34
|
+
export declare function serializeTryFixture(fixture: TryFixture): string;
|
|
35
|
+
export interface CaptureSource {
|
|
36
|
+
record: RunRecord;
|
|
37
|
+
answers: RunAnswers | null;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Find the run a `--capture` argument names, through the same validating readers `sweny runs` uses
|
|
41
|
+
* (history is agent-writable, so a planted or linked file is skipped, not trusted). `ref` is
|
|
42
|
+
* `latest` (newest record under `<cwd>/.sweny/runs`), a run id or unique prefix, or a path to the run
|
|
43
|
+
* dir (`.sweny/runs/<id>`) or its record (`.sweny/runs/<id>.json`).
|
|
44
|
+
*/
|
|
45
|
+
export declare function resolveCaptureRun(ref: string, cwd?: string): CaptureSource;
|
|
46
|
+
export interface CaptureRequest {
|
|
47
|
+
ref: string;
|
|
48
|
+
repo?: string;
|
|
49
|
+
/** Write the fixture here. Default: return it for stdout. */
|
|
50
|
+
out?: string;
|
|
51
|
+
cwd?: string;
|
|
52
|
+
env?: Record<string, string | undefined>;
|
|
53
|
+
}
|
|
54
|
+
/** Resolve, convert and (when `out` is set) write. Returns the fixture text and where it went. */
|
|
55
|
+
export declare function captureTryFixture(req: CaptureRequest): {
|
|
56
|
+
text: string;
|
|
57
|
+
fixture: TryFixture;
|
|
58
|
+
wrote?: string;
|
|
59
|
+
};
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `sweny try --capture <run>`: turn a recorded run into the `sweny try` fixture (#537).
|
|
3
|
+
*
|
|
4
|
+
* Input is what a real run already left in `.sweny/runs/`: the history record
|
|
5
|
+
* (`<run-id>.json`, run-history.ts) and the answers (`<run-id>/answers.json`,
|
|
6
|
+
* run-answers.ts). Output is a `TryFixture` with `illustrative: false`, so
|
|
7
|
+
* `sweny try` labels it "a replay of a real run on <date>".
|
|
8
|
+
*
|
|
9
|
+
* Nothing is invented. Durations, tool-call counts, token and cost totals, the
|
|
10
|
+
* harness, the policy floor and the answer all come from the record. A run that
|
|
11
|
+
* lacks them (written before `detail_version` 1), failed, or answered nothing is
|
|
12
|
+
* refused with the reason, never padded. The answer is redacted a second time
|
|
13
|
+
* here (the journal's redactor plus the run's secret values), so a token that
|
|
14
|
+
* slipped past the first pass does not reach a committed file.
|
|
15
|
+
*
|
|
16
|
+
* Deterministic: fixed key order, no clock, no randomness. The same run always
|
|
17
|
+
* yields the same bytes.
|
|
18
|
+
*/
|
|
19
|
+
import fs from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import { redact, runSecretValues } from "../journal.js";
|
|
22
|
+
import { readRunAnswers } from "./run-answers.js";
|
|
23
|
+
import { RUN_HISTORY_DIR, RUN_ID_RE, findRun, listRuns } from "./run-history.js";
|
|
24
|
+
import { loadTryWorkflow, parseTryFixture } from "./try.js";
|
|
25
|
+
const REPO_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/;
|
|
26
|
+
function usageOf(n) {
|
|
27
|
+
const usage = {};
|
|
28
|
+
if (typeof n.tokens_in === "number")
|
|
29
|
+
usage.inputTokens = n.tokens_in;
|
|
30
|
+
if (typeof n.tokens_out === "number")
|
|
31
|
+
usage.outputTokens = n.tokens_out;
|
|
32
|
+
if (typeof n.cost_usd === "number")
|
|
33
|
+
usage.costUsd = n.cost_usd;
|
|
34
|
+
if (typeof n.num_turns === "number")
|
|
35
|
+
usage.numTurns = n.num_turns;
|
|
36
|
+
return Object.keys(usage).length > 0 ? usage : undefined;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* The fixture for one recorded run. Throws, naming the reason, when the run cannot honestly be one:
|
|
40
|
+
* not a success, no detail (older record), no harness, an unknown template or node, or no answers.
|
|
41
|
+
*/
|
|
42
|
+
export function buildTryFixture(record, answers, opts) {
|
|
43
|
+
const refuse = (why) => {
|
|
44
|
+
throw new Error(`cannot capture run ${record.run_id}: ${why}`);
|
|
45
|
+
};
|
|
46
|
+
if (!REPO_RE.test(opts.sampleRepo))
|
|
47
|
+
refuse(`--repo must look like owner/name, got "${opts.sampleRepo}"`);
|
|
48
|
+
if (record.status !== "success")
|
|
49
|
+
refuse(`it ended ${record.status}; a demo is captured from a successful run`);
|
|
50
|
+
if (typeof record.detail_version !== "number" || record.detail_version < 1) {
|
|
51
|
+
refuse("it predates per-tool call counts (no detail_version). Re-run with a sweny that records them");
|
|
52
|
+
}
|
|
53
|
+
if (!record.harness)
|
|
54
|
+
refuse("it recorded no harness");
|
|
55
|
+
if (!record.policy)
|
|
56
|
+
refuse("it recorded no policy floor (env scope and sandbox)");
|
|
57
|
+
if (!answers)
|
|
58
|
+
refuse("it has no answers.json");
|
|
59
|
+
if (answers.run_id !== record.run_id || answers.workflow_id !== record.workflow_id) {
|
|
60
|
+
refuse("answers.json belongs to a different run");
|
|
61
|
+
}
|
|
62
|
+
if (record.nodes.length === 0)
|
|
63
|
+
refuse("it recorded no nodes");
|
|
64
|
+
const workflow = loadTryWorkflow(record.workflow_id);
|
|
65
|
+
const secrets = opts.secrets ?? [];
|
|
66
|
+
const nodes = {};
|
|
67
|
+
for (const n of record.nodes) {
|
|
68
|
+
if (!workflow.nodes[n.id])
|
|
69
|
+
refuse(`node "${n.id}" is not in the ${record.workflow_id} template`);
|
|
70
|
+
if (n.status !== "success")
|
|
71
|
+
refuse(`node "${n.id}" ended ${n.status}`);
|
|
72
|
+
const usage = usageOf(n);
|
|
73
|
+
// Only nodes whose template declares an output schema: a free-form `summary` (what answers.json
|
|
74
|
+
// keeps for a node with no schema) is the agent's working notes, not the answer, and stays out.
|
|
75
|
+
const schema = workflow.nodes[n.id].output;
|
|
76
|
+
const declared = schema?.properties && Object.keys(schema.properties).length > 0 ? answers.nodes[n.id] : undefined;
|
|
77
|
+
const output = declared ? redact(declared, secrets).value : undefined;
|
|
78
|
+
nodes[n.id] = {
|
|
79
|
+
duration_ms: n.duration_ms,
|
|
80
|
+
tool_calls: { ...(n.tool_counts ?? {}) },
|
|
81
|
+
...(usage ? { usage } : {}),
|
|
82
|
+
...(output && Object.keys(output).length > 0 ? { output } : {}),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
if (!Object.values(nodes).some((n) => n.output))
|
|
86
|
+
refuse("no node recorded an answer");
|
|
87
|
+
const date = record.started_at.slice(0, 10);
|
|
88
|
+
return parseTryFixture({
|
|
89
|
+
schema_version: 1,
|
|
90
|
+
illustrative: false,
|
|
91
|
+
captured_at: record.started_at,
|
|
92
|
+
notice: `A replay of a real run of ${record.workflow_id} on ${opts.sampleRepo}, started ${record.started_at.slice(0, 19)}Z (${date}). ` +
|
|
93
|
+
"Durations, tool-call counts, token usage and cost are as recorded; the answer is redacted. " +
|
|
94
|
+
"Made by `sweny try --capture`; do not edit by hand.",
|
|
95
|
+
workflow: record.workflow_id,
|
|
96
|
+
sample_repo: opts.sampleRepo,
|
|
97
|
+
harness: { id: record.harness.id, version: record.harness.version },
|
|
98
|
+
policy: {
|
|
99
|
+
envScope: record.policy.scoped,
|
|
100
|
+
sandbox: record.policy.sandbox,
|
|
101
|
+
sandboxStarted: record.policy.sandboxed,
|
|
102
|
+
},
|
|
103
|
+
nodes,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
/** The committed bytes: two-space JSON, a trailing newline. */
|
|
107
|
+
export function serializeTryFixture(fixture) {
|
|
108
|
+
return `${JSON.stringify(fixture, null, 2)}\n`;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Find the run a `--capture` argument names, through the same validating readers `sweny runs` uses
|
|
112
|
+
* (history is agent-writable, so a planted or linked file is skipped, not trusted). `ref` is
|
|
113
|
+
* `latest` (newest record under `<cwd>/.sweny/runs`), a run id or unique prefix, or a path to the run
|
|
114
|
+
* dir (`.sweny/runs/<id>`) or its record (`.sweny/runs/<id>.json`).
|
|
115
|
+
*/
|
|
116
|
+
export function resolveCaptureRun(ref, cwd = process.cwd()) {
|
|
117
|
+
let root = cwd;
|
|
118
|
+
let wanted = ref;
|
|
119
|
+
if (ref !== "latest" && !RUN_ID_RE.test(ref)) {
|
|
120
|
+
const abs = path.resolve(cwd, ref);
|
|
121
|
+
const id = path.basename(abs).replace(/\.json$/, "");
|
|
122
|
+
const runsDir = path.dirname(abs);
|
|
123
|
+
const stateDir = path.dirname(runsDir);
|
|
124
|
+
if (path.basename(runsDir) !== path.basename(RUN_HISTORY_DIR) || path.basename(stateDir) !== ".sweny") {
|
|
125
|
+
// Not a path into .sweny/runs: maybe a unique id prefix.
|
|
126
|
+
wanted = ref;
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
root = path.dirname(stateDir);
|
|
130
|
+
wanted = id;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const runs = listRuns(root);
|
|
134
|
+
const record = wanted === "latest" ? runs[0] : findRun(runs, wanted);
|
|
135
|
+
if (!record) {
|
|
136
|
+
throw new Error(`no run "${ref}" under ${path.join(root, RUN_HISTORY_DIR)}` +
|
|
137
|
+
(runs.length === 0 ? " (no runs recorded there)" : ""));
|
|
138
|
+
}
|
|
139
|
+
return { record, answers: readRunAnswers(record.run_id, root) };
|
|
140
|
+
}
|
|
141
|
+
/** Resolve, convert and (when `out` is set) write. Returns the fixture text and where it went. */
|
|
142
|
+
export function captureTryFixture(req) {
|
|
143
|
+
const env = req.env ?? process.env;
|
|
144
|
+
const cwd = req.cwd ?? process.cwd();
|
|
145
|
+
const repo = req.repo ?? env.GITHUB_REPOSITORY;
|
|
146
|
+
if (!repo)
|
|
147
|
+
throw new Error("--repo owner/name is required (GITHUB_REPOSITORY is not set)");
|
|
148
|
+
const { record, answers } = resolveCaptureRun(req.ref, cwd);
|
|
149
|
+
const fixture = buildTryFixture(record, answers, { sampleRepo: repo, secrets: runSecretValues(undefined, env) });
|
|
150
|
+
const text = serializeTryFixture(fixture);
|
|
151
|
+
if (!req.out)
|
|
152
|
+
return { text, fixture };
|
|
153
|
+
const file = path.resolve(cwd, req.out);
|
|
154
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
155
|
+
fs.writeFileSync(file, text, { mode: 0o644 });
|
|
156
|
+
return { text, fixture, wrote: file };
|
|
157
|
+
}
|
package/dist/cli/try.d.ts
CHANGED
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
* executor is empty by construction.
|
|
10
10
|
*
|
|
11
11
|
* The recording lives in `try-fixture.json` (metadata plus the sample answer
|
|
12
|
-
* only). Until a real run is
|
|
12
|
+
* only). Until a real run is committed (`illustrative: false`, made by
|
|
13
|
+
* `sweny try --capture`, try-capture.ts) it is labelled illustrative; a real one
|
|
14
|
+
* says "replay of a real run on <date>" instead.
|
|
13
15
|
*/
|
|
14
16
|
import type { Command } from "commander";
|
|
15
17
|
import type { AgentHarness } from "../harness/types.js";
|
|
@@ -33,6 +35,11 @@ export interface TryFixture {
|
|
|
33
35
|
schema_version: 1;
|
|
34
36
|
/** True until the owner replaces the file with a recording of a real run. */
|
|
35
37
|
illustrative: boolean;
|
|
38
|
+
/**
|
|
39
|
+
* When the recorded run started (ISO 8601). Required when `illustrative` is false: the label says
|
|
40
|
+
* "replay of a real run on <date>".
|
|
41
|
+
*/
|
|
42
|
+
captured_at?: string;
|
|
36
43
|
notice: string;
|
|
37
44
|
/** Built-in template id the recording is of. */
|
|
38
45
|
workflow: string;
|
|
@@ -62,13 +69,19 @@ export declare function loadTryWorkflow(templateId: string): Workflow;
|
|
|
62
69
|
* reported through `onProgress` spread across that time so progress feels live.
|
|
63
70
|
*/
|
|
64
71
|
export declare function createReplayHarness(fixture: TryFixture, opts: {
|
|
65
|
-
paceMs: number;
|
|
72
|
+
paceMs: number | ((nodeId: string) => number);
|
|
66
73
|
}): AgentHarness;
|
|
67
74
|
export declare const TRY_BANNER_TITLE = "Recorded demo";
|
|
68
75
|
/** The one command to run next: scaffolds the same workflow on the user's repo and says how to run it. */
|
|
69
76
|
export declare const TRY_NEXT_COMMAND = "npx @sweny-ai/core new --template explain-repo";
|
|
70
77
|
/** Appended to `sweny try --help`. */
|
|
71
78
|
export declare const TRY_HELP_NOTE: string;
|
|
79
|
+
/** `YYYY-MM-DD` of the recorded run, or "" when the fixture carries none. */
|
|
80
|
+
export declare function capturedDate(fixture: TryFixture): string;
|
|
81
|
+
/** What the recording is, in one clause: the label `sweny try` shows. Illustrative until a real run is committed. */
|
|
82
|
+
export declare function tryLabel(fixture: TryFixture): string;
|
|
83
|
+
/** The note under `sweny try --help`, for the recording that ships. */
|
|
84
|
+
export declare function tryHelpNote(fixture: TryFixture): string;
|
|
72
85
|
/** The tagline, then the demo notice as one dim line. */
|
|
73
86
|
export declare function formatTryBanner(fixture: TryFixture, paint?: Paint, unicode?: boolean): string;
|
|
74
87
|
/** The closing line: one command. */
|
|
@@ -76,7 +89,11 @@ export declare function formatTryNext(paint?: Paint): string;
|
|
|
76
89
|
export interface TryOptions {
|
|
77
90
|
/** Replay instantly (and land the stamp without the one-frame pause). */
|
|
78
91
|
fast?: boolean;
|
|
79
|
-
/**
|
|
92
|
+
/**
|
|
93
|
+
* Milliseconds each node takes to replay, fixed. Default: the recorded timings scaled to about
|
|
94
|
+
* {@link REPLAY_TARGET_MS} in total, and instant when output is not a live terminal (non-TTY, CI,
|
|
95
|
+
* dumb terminal). Ignored with `fast`.
|
|
96
|
+
*/
|
|
80
97
|
paceMs?: number;
|
|
81
98
|
/** Also write the PR-comment markdown here. */
|
|
82
99
|
commentFile?: string;
|
|
@@ -94,7 +111,12 @@ export interface TryOptions {
|
|
|
94
111
|
columns?: number;
|
|
95
112
|
fixture?: TryFixture;
|
|
96
113
|
}
|
|
97
|
-
|
|
114
|
+
/** A default replay takes about this long in total, however long the real run took. */
|
|
115
|
+
export declare const REPLAY_TARGET_MS = 8000;
|
|
116
|
+
/** No node replays faster than this unless the replay is instant. */
|
|
117
|
+
export declare const REPLAY_NODE_MIN_MS = 250;
|
|
118
|
+
/** Per-node replay time: the recorded durations scaled to {@link REPLAY_TARGET_MS} in total (never slowed down). */
|
|
119
|
+
export declare function scaledPace(fixture: TryFixture, targetMs?: number): (nodeId: string) => number;
|
|
98
120
|
/** The line that opens the job summary preview (record.ts cuts the preview from here). */
|
|
99
121
|
export declare const SUMMARY_PREVIEW_LEAD = "In CI, the job summary gets this heading, this DAG and the receipt below:";
|
|
100
122
|
/** Replay the recording. Returns the process exit code. */
|
package/dist/cli/try.js
CHANGED
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
* executor is empty by construction.
|
|
10
10
|
*
|
|
11
11
|
* The recording lives in `try-fixture.json` (metadata plus the sample answer
|
|
12
|
-
* only). Until a real run is
|
|
12
|
+
* only). Until a real run is committed (`illustrative: false`, made by
|
|
13
|
+
* `sweny try --capture`, try-capture.ts) it is labelled illustrative; a real one
|
|
14
|
+
* says "replay of a real run on <date>" instead.
|
|
13
15
|
*/
|
|
14
16
|
import fs from "node:fs";
|
|
15
17
|
import { workspaceRoot, writeFileNoFollow } from "../safe-file.js";
|
|
@@ -24,9 +26,10 @@ import { formatReceiptDuration, summarizeRun } from "./run-output.js";
|
|
|
24
26
|
import { DagRenderer } from "./renderer.js";
|
|
25
27
|
import { createNodeProgress, toolCallsText } from "./progress.js";
|
|
26
28
|
import { createPaint } from "./style.js";
|
|
27
|
-
import { colorEnabled, glyphsFor, supportsUnicode, terminalColumns } from "./terminal.js";
|
|
29
|
+
import { colorEnabled, glyphsFor, richOutput, supportsUnicode, terminalColumns } from "./terminal.js";
|
|
28
30
|
import { writeReceipt } from "./ticket.js";
|
|
29
31
|
import { SWENY_TAGLINE } from "../theme.js";
|
|
32
|
+
const CAPTURED_AT_RE = /^\d{4}-\d{2}-\d{2}(T[\d:.]+Z)?$/;
|
|
30
33
|
const isObj = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
31
34
|
/** Shape check with a message naming the first bad field. Throws on a malformed fixture. */
|
|
32
35
|
export function parseTryFixture(raw) {
|
|
@@ -43,6 +46,12 @@ export function parseTryFixture(raw) {
|
|
|
43
46
|
}
|
|
44
47
|
if (typeof raw.illustrative !== "boolean")
|
|
45
48
|
bad("illustrative must be a boolean");
|
|
49
|
+
if (raw.illustrative === false && (typeof raw.captured_at !== "string" || !CAPTURED_AT_RE.test(raw.captured_at))) {
|
|
50
|
+
bad("a real recording (illustrative: false) needs captured_at, an ISO date such as 2026-11-12T14:00:00.000Z");
|
|
51
|
+
}
|
|
52
|
+
if (raw.captured_at !== undefined && (typeof raw.captured_at !== "string" || !CAPTURED_AT_RE.test(raw.captured_at))) {
|
|
53
|
+
bad("captured_at must be an ISO date");
|
|
54
|
+
}
|
|
46
55
|
if (!isObj(raw.harness) || typeof raw.harness.id !== "string" || typeof raw.harness.version !== "string") {
|
|
47
56
|
bad("harness must be { id, version }");
|
|
48
57
|
}
|
|
@@ -135,7 +144,8 @@ export function createReplayHarness(fixture, opts) {
|
|
|
135
144
|
for (let i = 0; i < count; i++)
|
|
136
145
|
toolCalls.push({ tool, input: {} });
|
|
137
146
|
}
|
|
138
|
-
const
|
|
147
|
+
const nodePace = typeof opts.paceMs === "function" ? opts.paceMs(id) : opts.paceMs;
|
|
148
|
+
const step = toolCalls.length > 0 ? nodePace / toolCalls.length : nodePace;
|
|
139
149
|
for (const tc of toolCalls) {
|
|
140
150
|
await sleep(step, req.signal);
|
|
141
151
|
req.onProgress?.(tc.tool);
|
|
@@ -161,20 +171,46 @@ export const TRY_NEXT_COMMAND = "npx @sweny-ai/core new --template explain-repo"
|
|
|
161
171
|
/** Appended to `sweny try --help`. */
|
|
162
172
|
export const TRY_HELP_NOTE = "The recording is ILLUSTRATIVE SAMPLE DATA (a fictional repo, hand-written timings and token counts), not output from a real run. " +
|
|
163
173
|
"Nothing runs: no agent, no network, no credentials read, no files written.";
|
|
174
|
+
/** `YYYY-MM-DD` of the recorded run, or "" when the fixture carries none. */
|
|
175
|
+
export function capturedDate(fixture) {
|
|
176
|
+
return fixture.captured_at?.slice(0, 10) ?? "";
|
|
177
|
+
}
|
|
178
|
+
/** What the recording is, in one clause: the label `sweny try` shows. Illustrative until a real run is committed. */
|
|
179
|
+
export function tryLabel(fixture) {
|
|
180
|
+
return fixture.illustrative
|
|
181
|
+
? `of ${fixture.workflow} on a sample repo (${fixture.sample_repo}), illustrative data`
|
|
182
|
+
: `of ${fixture.workflow} on ${fixture.sample_repo}, a replay of a real run on ${capturedDate(fixture)}`;
|
|
183
|
+
}
|
|
184
|
+
/** The note under `sweny try --help`, for the recording that ships. */
|
|
185
|
+
export function tryHelpNote(fixture) {
|
|
186
|
+
return fixture.illustrative
|
|
187
|
+
? TRY_HELP_NOTE
|
|
188
|
+
: `The recording is a replay of a real run of ${fixture.workflow} on ${fixture.sample_repo} (${capturedDate(fixture)}): ` +
|
|
189
|
+
"recorded tool-call counts, tokens, cost and answer (redacted), timings scaled for replay. " +
|
|
190
|
+
"Nothing runs now: no agent, no network, no credentials read, no files written.";
|
|
191
|
+
}
|
|
164
192
|
/** The tagline, then the demo notice as one dim line. */
|
|
165
193
|
export function formatTryBanner(fixture, paint = createPaint(false), unicode = true) {
|
|
166
194
|
const g = glyphsFor(unicode);
|
|
167
|
-
const data = fixture.illustrative ? ", illustrative data" : "";
|
|
168
195
|
return [
|
|
169
196
|
` ${paint.brand(g.brand)} ${paint.strong("SWEny")} ${SWENY_TAGLINE}`,
|
|
170
|
-
paint.muted(` ${TRY_BANNER_TITLE}
|
|
197
|
+
paint.muted(` ${TRY_BANNER_TITLE} ${tryLabel(fixture)}. No agent, network or credentials.`),
|
|
171
198
|
].join("\n");
|
|
172
199
|
}
|
|
173
200
|
/** The closing line: one command. */
|
|
174
201
|
export function formatTryNext(paint = createPaint(false)) {
|
|
175
202
|
return ` ${paint.muted("Run it on your repo:")} ${paint.strong(TRY_NEXT_COMMAND)}`;
|
|
176
203
|
}
|
|
177
|
-
|
|
204
|
+
/** A default replay takes about this long in total, however long the real run took. */
|
|
205
|
+
export const REPLAY_TARGET_MS = 8000;
|
|
206
|
+
/** No node replays faster than this unless the replay is instant. */
|
|
207
|
+
export const REPLAY_NODE_MIN_MS = 250;
|
|
208
|
+
/** Per-node replay time: the recorded durations scaled to {@link REPLAY_TARGET_MS} in total (never slowed down). */
|
|
209
|
+
export function scaledPace(fixture, targetMs = REPLAY_TARGET_MS) {
|
|
210
|
+
const total = Object.values(fixture.nodes).reduce((a, n) => a + n.duration_ms, 0);
|
|
211
|
+
const scale = total > targetMs ? targetMs / total : 1;
|
|
212
|
+
return (id) => Math.max(REPLAY_NODE_MIN_MS, Math.round((fixture.nodes[id]?.duration_ms ?? 0) * scale));
|
|
213
|
+
}
|
|
178
214
|
const silentLogger = { info() { }, warn() { }, error() { }, debug() { } };
|
|
179
215
|
/** The line that opens the job summary preview (record.ts cuts the preview from here). */
|
|
180
216
|
export const SUMMARY_PREVIEW_LEAD = "In CI, the job summary gets this heading, this DAG and the receipt below:";
|
|
@@ -182,8 +218,16 @@ export const SUMMARY_PREVIEW_LEAD = "In CI, the job summary gets this heading, t
|
|
|
182
218
|
export async function runTry(opts = {}) {
|
|
183
219
|
const write = opts.write ?? ((s) => void process.stdout.write(s));
|
|
184
220
|
const tty = opts.tty ?? Boolean(process.stdout.isTTY);
|
|
185
|
-
const paceMs = opts.fast ? 0 : Math.max(0, opts.paceMs ?? DEFAULT_TRY_PACE_MS);
|
|
186
221
|
const fixture = opts.fixture ?? loadTryFixture();
|
|
222
|
+
// Explicit pace wins. Otherwise real timings, scaled, on a live terminal only: a pipe, CI or a
|
|
223
|
+
// dumb terminal gets the replay at once (the receipt shows the recorded times either way).
|
|
224
|
+
const paceMs = opts.fast
|
|
225
|
+
? 0
|
|
226
|
+
: opts.paceMs !== undefined
|
|
227
|
+
? Math.max(0, opts.paceMs)
|
|
228
|
+
: richOutput({ isTTY: tty }, process.env)
|
|
229
|
+
? scaledPace(fixture)
|
|
230
|
+
: 0;
|
|
187
231
|
const workflow = loadTryWorkflow(fixture.workflow);
|
|
188
232
|
for (const id of Object.keys(fixture.nodes)) {
|
|
189
233
|
if (!workflow.nodes[id])
|
|
@@ -281,16 +325,53 @@ export async function runTry(opts = {}) {
|
|
|
281
325
|
write(`${formatTryNext(paint)}\n\n`);
|
|
282
326
|
return receipt.ok ? 0 : 1;
|
|
283
327
|
}
|
|
328
|
+
/** The shipped fixture's label, falling back to the illustrative wording when it cannot be read. */
|
|
329
|
+
function shippedHelp() {
|
|
330
|
+
try {
|
|
331
|
+
const fixture = loadTryFixture();
|
|
332
|
+
return {
|
|
333
|
+
description: fixture.illustrative
|
|
334
|
+
? "Replay a recorded demo run: no credentials, no network, no model calls (illustrative sample data)"
|
|
335
|
+
: "Replay a recorded real run: no credentials, no network, no model calls",
|
|
336
|
+
note: tryHelpNote(fixture),
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
catch {
|
|
340
|
+
return {
|
|
341
|
+
description: "Replay a recorded demo run: no credentials, no network, no model calls (illustrative sample data)",
|
|
342
|
+
note: TRY_HELP_NOTE,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
}
|
|
284
346
|
export function registerTryCommand(program) {
|
|
347
|
+
const help = shippedHelp();
|
|
285
348
|
return program
|
|
286
349
|
.command("try")
|
|
287
|
-
.description(
|
|
350
|
+
.description(help.description)
|
|
288
351
|
.option("--fast", "Replay instantly instead of pacing the progress")
|
|
289
|
-
.option("--pace <ms>", `
|
|
352
|
+
.option("--pace <ms>", `Fixed milliseconds per node (default: the recorded timings scaled to about ${REPLAY_TARGET_MS / 1000}s, instant when not a live terminal)`)
|
|
290
353
|
.option("--comment-file <path>", "Also write the PR-comment markdown to <path>")
|
|
291
354
|
.option("--record <file.svg>", "Write the demo as an animated, self-contained SVG terminal recording to <file.svg>")
|
|
292
|
-
.
|
|
355
|
+
.option("--capture <run>", "Convert a recorded run (latest, a run id, or a path under .sweny/runs) into a try fixture instead of replaying")
|
|
356
|
+
.option("--repo <owner/name>", "With --capture: the repo the run was on (default: GITHUB_REPOSITORY)")
|
|
357
|
+
.option("--out <file>", "With --capture: write the fixture to <file> (default: stdout)")
|
|
358
|
+
.addHelpText("after", `\n${help.note}\n`)
|
|
293
359
|
.action(async (options) => {
|
|
360
|
+
if (options.capture) {
|
|
361
|
+
try {
|
|
362
|
+
const { captureTryFixture } = await import("./try-capture.js");
|
|
363
|
+
const res = captureTryFixture({ ref: options.capture, repo: options.repo, out: options.out });
|
|
364
|
+
if (res.wrote)
|
|
365
|
+
process.stderr.write(` wrote ${res.wrote} (replay of run ${res.fixture.captured_at})\n`);
|
|
366
|
+
else
|
|
367
|
+
process.stdout.write(res.text);
|
|
368
|
+
}
|
|
369
|
+
catch (err) {
|
|
370
|
+
console.error(`\n could not capture the run: ${err instanceof Error ? err.message : err}\n`);
|
|
371
|
+
process.exitCode = 1;
|
|
372
|
+
}
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
294
375
|
if (options.record) {
|
|
295
376
|
try {
|
|
296
377
|
const { recordTrySvgToFile } = await import("./record.js");
|