@cad0p/pi-memory-budget 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +21 -0
- package/README.md +142 -2
- package/docs/consumer-verification-section.md +28 -0
- package/docs/verification.md +55 -0
- package/package.json +63 -4
- package/src/args.mjs +199 -0
- package/src/budget.mjs +546 -0
- package/src/cli.mjs +96 -0
- package/src/config.mjs +180 -0
- package/src/fixture.mjs +491 -0
- package/src/gc-harness.js +67 -0
- package/src/index.mjs +13 -0
- package/src/orchestrate.mjs +164 -0
- package/src/pi.mjs +107 -0
- package/src/report.mjs +188 -0
- package/src/runner.mjs +347 -0
- package/src/sampler.mjs +162 -0
- package/src/stats.mjs +163 -0
package/src/config.mjs
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// budget.json loading and merging with CLI options.
|
|
2
|
+
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { CASES, UsageError } from "./args.mjs";
|
|
6
|
+
import { resolveBudgets } from "./budget.mjs";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* @typedef {object} ConfigFile
|
|
10
|
+
* @property {Record<string, number>} [budgets]
|
|
11
|
+
* @property {Record<string, string | { reason: string }>} [waivers] budget key -> reason
|
|
12
|
+
* @property {string[]} [extensions]
|
|
13
|
+
* @property {string[]} [baselineExtensions]
|
|
14
|
+
* @property {string} [cwd]
|
|
15
|
+
* @property {Record<string, string>} [env]
|
|
16
|
+
* @property {string | Record<string, string>} [setup] shell command, or case name ("*" for all) -> command
|
|
17
|
+
* @property {number} [runs]
|
|
18
|
+
* @property {number} [window]
|
|
19
|
+
* @property {number} [steadySeconds]
|
|
20
|
+
* @property {number} [settleMax]
|
|
21
|
+
* @property {number} [settleMin]
|
|
22
|
+
* @property {number} [sessionMb]
|
|
23
|
+
* @property {string[]} [cases]
|
|
24
|
+
* @property {number} [seed]
|
|
25
|
+
* @property {string[]} [piArgs]
|
|
26
|
+
* @property {boolean} [gc] GC normalization (default true)
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const KNOWN = new Set([
|
|
30
|
+
"budgets",
|
|
31
|
+
"waivers",
|
|
32
|
+
"extensions",
|
|
33
|
+
"baselineExtensions",
|
|
34
|
+
"cwd",
|
|
35
|
+
"env",
|
|
36
|
+
"setup",
|
|
37
|
+
"runs",
|
|
38
|
+
"window",
|
|
39
|
+
"steadySeconds",
|
|
40
|
+
"settleMax",
|
|
41
|
+
"settleMin",
|
|
42
|
+
"sessionMb",
|
|
43
|
+
"cases",
|
|
44
|
+
"seed",
|
|
45
|
+
"piArgs",
|
|
46
|
+
"gc",
|
|
47
|
+
"$schema",
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* @param {string} file
|
|
52
|
+
* @returns {ConfigFile}
|
|
53
|
+
*/
|
|
54
|
+
export function loadConfigFile(file) {
|
|
55
|
+
let raw;
|
|
56
|
+
try {
|
|
57
|
+
raw = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
58
|
+
} catch (e) {
|
|
59
|
+
throw new UsageError(
|
|
60
|
+
`cannot read config ${file}: ${e instanceof Error ? e.message : e}`,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
64
|
+
throw new UsageError(`${file}: expected a JSON object`);
|
|
65
|
+
for (const k of Object.keys(raw)) {
|
|
66
|
+
if (!KNOWN.has(k))
|
|
67
|
+
throw new UsageError(
|
|
68
|
+
`${file}: unknown key "${k}" (known: ${[...KNOWN].join(", ")})`,
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
if (raw.gc !== undefined && typeof raw.gc !== "boolean")
|
|
72
|
+
throw new UsageError(`${file}: "gc" must be a boolean`);
|
|
73
|
+
return raw;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* @typedef {object} ResolvedOptions
|
|
78
|
+
* @property {string[]} extensions absolute
|
|
79
|
+
* @property {string[]} baselineExtensions absolute
|
|
80
|
+
* @property {string} cwd absolute
|
|
81
|
+
* @property {Record<string, string>} env
|
|
82
|
+
* @property {Record<string, string>} setup case -> command ("*" = every case)
|
|
83
|
+
* @property {number} runs
|
|
84
|
+
* @property {number} window
|
|
85
|
+
* @property {number} steadySeconds
|
|
86
|
+
* @property {number} settleMax
|
|
87
|
+
* @property {number} settleMin
|
|
88
|
+
* @property {number} sessionMb
|
|
89
|
+
* @property {string[]} cases
|
|
90
|
+
* @property {number} seed
|
|
91
|
+
* @property {string | undefined} pi
|
|
92
|
+
* @property {string[]} piArgs
|
|
93
|
+
* @property {boolean} gc
|
|
94
|
+
* @property {string} fixturesDir
|
|
95
|
+
* @property {import("./budget.mjs").Budgets} budgets
|
|
96
|
+
* @property {Record<string, string>} waivers
|
|
97
|
+
*/
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* CLI values win over the config file, which wins over defaults. Relative
|
|
101
|
+
* paths from the CLI resolve against the process cwd, from the config file
|
|
102
|
+
* against the file's directory.
|
|
103
|
+
* @param {import("./args.mjs").RunOptions} cli
|
|
104
|
+
* @param {string} tmpdir
|
|
105
|
+
* @returns {ResolvedOptions}
|
|
106
|
+
*/
|
|
107
|
+
export function resolveOptions(cli, tmpdir) {
|
|
108
|
+
const cfgPath = cli.config ? path.resolve(cli.config) : undefined;
|
|
109
|
+
const cfg = cfgPath ? loadConfigFile(cfgPath) : {};
|
|
110
|
+
const cfgDir = cfgPath ? path.dirname(cfgPath) : process.cwd();
|
|
111
|
+
/** @param {string[]} list @param {string} base */
|
|
112
|
+
const abs = (list, base) => list.map((p) => path.resolve(base, p));
|
|
113
|
+
|
|
114
|
+
const extensions = [
|
|
115
|
+
...abs(cfg.extensions ?? [], cfgDir),
|
|
116
|
+
...abs(cli.extensions, process.cwd()),
|
|
117
|
+
];
|
|
118
|
+
if (extensions.length === 0)
|
|
119
|
+
throw new UsageError(
|
|
120
|
+
"run needs at least one --extension <path> (or 'extensions' in the config)",
|
|
121
|
+
);
|
|
122
|
+
const cwd = cli.cwd
|
|
123
|
+
? path.resolve(cli.cwd)
|
|
124
|
+
: cfg.cwd
|
|
125
|
+
? path.resolve(cfgDir, cfg.cwd)
|
|
126
|
+
: process.cwd();
|
|
127
|
+
|
|
128
|
+
/** @type {Record<string, string>} */
|
|
129
|
+
const env = { ...(cfg.env ?? {}) };
|
|
130
|
+
for (const e of cli.env) {
|
|
131
|
+
const i = e.indexOf("=");
|
|
132
|
+
env[e.slice(0, i)] = e.slice(i + 1);
|
|
133
|
+
}
|
|
134
|
+
/** @type {Record<string, string>} */
|
|
135
|
+
const setup =
|
|
136
|
+
typeof cfg.setup === "string"
|
|
137
|
+
? { "*": cfg.setup }
|
|
138
|
+
: { ...(cfg.setup ?? {}) };
|
|
139
|
+
for (const k of Object.keys(setup)) {
|
|
140
|
+
if (k !== "*" && !CASES.includes(k))
|
|
141
|
+
throw new UsageError(
|
|
142
|
+
`setup key "${k}" must be "*" or one of ${CASES.join(", ")}`,
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
const cases = cli.cases ?? cfg.cases ?? CASES;
|
|
146
|
+
const bad = cases.filter((c) => !CASES.includes(c));
|
|
147
|
+
if (bad.length)
|
|
148
|
+
throw new UsageError(`unknown case(s) in config: ${bad.join(", ")}`);
|
|
149
|
+
const { budgets, waivers } = resolveBudgets(
|
|
150
|
+
cfg.budgets,
|
|
151
|
+
/** @type {Record<string, unknown> | undefined} */ (cfg.waivers),
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
return {
|
|
155
|
+
extensions,
|
|
156
|
+
baselineExtensions: [
|
|
157
|
+
...abs(cfg.baselineExtensions ?? [], cfgDir),
|
|
158
|
+
...abs(cli.baselineExtensions, process.cwd()),
|
|
159
|
+
],
|
|
160
|
+
cwd,
|
|
161
|
+
env,
|
|
162
|
+
setup,
|
|
163
|
+
runs: cli.runs ?? cfg.runs ?? 3,
|
|
164
|
+
window: cli.window ?? cfg.window ?? 30,
|
|
165
|
+
steadySeconds: cli.steadySeconds ?? cfg.steadySeconds ?? 10,
|
|
166
|
+
settleMax: cli.settleMax ?? cfg.settleMax ?? 120,
|
|
167
|
+
settleMin: cli.settleMin ?? cfg.settleMin ?? 30,
|
|
168
|
+
sessionMb: cli.sessionMb ?? cfg.sessionMb ?? 200,
|
|
169
|
+
cases,
|
|
170
|
+
seed: cli.seed ?? cfg.seed ?? 1,
|
|
171
|
+
pi: cli.pi,
|
|
172
|
+
piArgs: [...(cfg.piArgs ?? []), ...cli.piArgs],
|
|
173
|
+
gc: cli.noGc ? false : cfg.gc !== false,
|
|
174
|
+
fixturesDir: cli.fixturesDir
|
|
175
|
+
? path.resolve(cli.fixturesDir)
|
|
176
|
+
: path.join(tmpdir, "pi-memory-budget"),
|
|
177
|
+
budgets,
|
|
178
|
+
waivers,
|
|
179
|
+
};
|
|
180
|
+
}
|
package/src/fixture.mjs
ADDED
|
@@ -0,0 +1,491 @@
|
|
|
1
|
+
// Deterministic generator for synthetic pi session files (JSONL, version 3).
|
|
2
|
+
// Format reference: pi docs/session-format.md and docs/message-types.md.
|
|
3
|
+
// Everything is generated from the seed: no real session content is used.
|
|
4
|
+
|
|
5
|
+
import fs from "node:fs";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
|
|
8
|
+
const MB = 1024 * 1024;
|
|
9
|
+
export const FIXTURE_FORMAT = 1;
|
|
10
|
+
|
|
11
|
+
/** @param {number} seed */
|
|
12
|
+
export function mulberry32(seed) {
|
|
13
|
+
let a = seed >>> 0;
|
|
14
|
+
return () => {
|
|
15
|
+
a = (a + 0x6d2b79f5) >>> 0;
|
|
16
|
+
let t = a;
|
|
17
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
18
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
19
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const WORDS = (
|
|
24
|
+
"the of and to in is it that for with as on be this by from or an are was not " +
|
|
25
|
+
"file path error build test run line code value function module config result " +
|
|
26
|
+
"session entry message agent tool call output input memory buffer string array " +
|
|
27
|
+
"object import export const let return async await promise stream parse render " +
|
|
28
|
+
"commit branch merge review issue check status warn info debug trace request " +
|
|
29
|
+
"response handler server client socket timeout retry cache index search query " +
|
|
30
|
+
"update delete create read write append remove list map filter reduce sort"
|
|
31
|
+
).split(" ");
|
|
32
|
+
|
|
33
|
+
/** @type {{ name: string, args: (r: () => number) => Record<string, unknown> }[]} */
|
|
34
|
+
const TOOLS = [
|
|
35
|
+
{
|
|
36
|
+
name: "bash",
|
|
37
|
+
args: (r) => ({ command: `echo ${word(r)} && ls -la ${word(r)}` }),
|
|
38
|
+
},
|
|
39
|
+
{ name: "read", args: (r) => ({ path: `src/${word(r)}/${word(r)}.ts` }) },
|
|
40
|
+
{
|
|
41
|
+
name: "edit",
|
|
42
|
+
args: (r) => ({
|
|
43
|
+
path: `src/${word(r)}.ts`,
|
|
44
|
+
edits: [{ oldText: sentence(r, 6), newText: sentence(r, 8) }],
|
|
45
|
+
}),
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
name: "write",
|
|
49
|
+
args: (r) => ({ path: `docs/${word(r)}.md`, content: sentence(r, 40) }),
|
|
50
|
+
},
|
|
51
|
+
{ name: "grep", args: (r) => ({ pattern: word(r), path: "src" }) },
|
|
52
|
+
];
|
|
53
|
+
const CUSTOM_TYPES = [
|
|
54
|
+
"synthetic-ext-state",
|
|
55
|
+
"synthetic-tracker",
|
|
56
|
+
"synthetic-usage-note",
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
/** @param {() => number} r */
|
|
60
|
+
function word(r) {
|
|
61
|
+
return WORDS[Math.floor(r() * WORDS.length)];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* @param {() => number} r
|
|
66
|
+
* @param {number} n
|
|
67
|
+
*/
|
|
68
|
+
function sentence(r, n) {
|
|
69
|
+
const out = [];
|
|
70
|
+
for (let i = 0; i < n; i++) out.push(word(r));
|
|
71
|
+
return out.join(" ");
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Pool of pseudo-random text that payloads are sliced from (cheap to generate
|
|
76
|
+
* 200 MB of content from).
|
|
77
|
+
* @param {() => number} r
|
|
78
|
+
*/
|
|
79
|
+
function makePool(r) {
|
|
80
|
+
const parts = [];
|
|
81
|
+
let len = 0;
|
|
82
|
+
while (len < 1.5 * MB) {
|
|
83
|
+
const line = sentence(r, 4 + Math.floor(r() * 14));
|
|
84
|
+
parts.push(line);
|
|
85
|
+
len += line.length + 1;
|
|
86
|
+
}
|
|
87
|
+
return parts.join("\n");
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* @typedef {object} FixtureOptions
|
|
92
|
+
* @property {number} mb target size in MiB
|
|
93
|
+
* @property {number} [seed]
|
|
94
|
+
* @property {string} [cwd] cwd recorded in the session header (default: filesystem root, which always exists)
|
|
95
|
+
*/
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* @typedef {object} FixtureStats
|
|
99
|
+
* @property {number} bytes
|
|
100
|
+
* @property {number} entries
|
|
101
|
+
* @property {number} branches entries whose parent already has a child
|
|
102
|
+
* @property {Record<string, number>} types counts per entry type (messages by role)
|
|
103
|
+
*/
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Writes a synthetic session file of about `mb` MiB.
|
|
107
|
+
* @param {string} file output path
|
|
108
|
+
* @param {FixtureOptions} opts
|
|
109
|
+
* @returns {FixtureStats}
|
|
110
|
+
*/
|
|
111
|
+
export function writeFixture(file, opts) {
|
|
112
|
+
const seed = opts.seed ?? 1;
|
|
113
|
+
const targetBytes = Math.round(opts.mb * MB);
|
|
114
|
+
const r = mulberry32(seed);
|
|
115
|
+
const pool = makePool(r);
|
|
116
|
+
const salt = Math.floor(r() * 0xffffffff) >>> 0;
|
|
117
|
+
const baseMs = Date.UTC(2026, 0, 1);
|
|
118
|
+
// pi refuses a session whose stored cwd does not exist, so default to one that does.
|
|
119
|
+
const sessionCwd = opts.cwd ?? path.parse(process.cwd()).root;
|
|
120
|
+
|
|
121
|
+
let counter = 0;
|
|
122
|
+
let clock = 0;
|
|
123
|
+
const nextId = () => {
|
|
124
|
+
// Bijective 32-bit mixing of a counter: unique ids without a lookup set.
|
|
125
|
+
let x = (counter++ ^ salt) >>> 0;
|
|
126
|
+
x = Math.imul(x ^ (x >>> 16), 0x45d9f3b) >>> 0;
|
|
127
|
+
x = Math.imul(x ^ (x >>> 16), 0x45d9f3b) >>> 0;
|
|
128
|
+
x = (x ^ (x >>> 16)) >>> 0;
|
|
129
|
+
return x.toString(16).padStart(8, "0");
|
|
130
|
+
};
|
|
131
|
+
/** @param {number} n */
|
|
132
|
+
const text = (n) => {
|
|
133
|
+
const start = Math.floor(r() * (pool.length - n));
|
|
134
|
+
return pool.slice(start, start + n);
|
|
135
|
+
};
|
|
136
|
+
/** log-uniform integer in [lo, hi]
|
|
137
|
+
* @param {number} lo @param {number} hi */
|
|
138
|
+
const logInt = (lo, hi) =>
|
|
139
|
+
Math.floor(Math.exp(Math.log(lo) + r() * (Math.log(hi) - Math.log(lo))));
|
|
140
|
+
|
|
141
|
+
const fd = fs.openSync(file, "w");
|
|
142
|
+
let bytes = 0;
|
|
143
|
+
let entries = 0;
|
|
144
|
+
let branches = 0;
|
|
145
|
+
/** @type {Record<string, number>} */
|
|
146
|
+
const types = {};
|
|
147
|
+
/** @type {string[]} */
|
|
148
|
+
let chunk = [];
|
|
149
|
+
let chunkBytes = 0;
|
|
150
|
+
const flush = () => {
|
|
151
|
+
if (chunk.length) fs.writeSync(fd, chunk.join(""));
|
|
152
|
+
chunk = [];
|
|
153
|
+
chunkBytes = 0;
|
|
154
|
+
};
|
|
155
|
+
/** @param {object} obj @param {string} [kind] */
|
|
156
|
+
const emit = (obj, kind) => {
|
|
157
|
+
const line = `${JSON.stringify(obj)}\n`;
|
|
158
|
+
const n = Buffer.byteLength(line);
|
|
159
|
+
chunk.push(line);
|
|
160
|
+
chunkBytes += n;
|
|
161
|
+
bytes += n;
|
|
162
|
+
if (kind) types[kind] = (types[kind] ?? 0) + 1;
|
|
163
|
+
if (chunkBytes > 4 * MB) flush();
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const uuid = () => {
|
|
167
|
+
const h = () =>
|
|
168
|
+
Math.floor(r() * 0xffffffff)
|
|
169
|
+
.toString(16)
|
|
170
|
+
.padStart(8, "0");
|
|
171
|
+
const a = h();
|
|
172
|
+
const b = h();
|
|
173
|
+
const c = h();
|
|
174
|
+
const d = h();
|
|
175
|
+
return `${a}-${b.slice(0, 4)}-7${b.slice(5, 8)}-a${c.slice(1, 4)}-${c.slice(4)}${d}`;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
try {
|
|
179
|
+
emit(
|
|
180
|
+
{
|
|
181
|
+
type: "session",
|
|
182
|
+
version: 3,
|
|
183
|
+
id: uuid(),
|
|
184
|
+
timestamp: new Date(baseMs).toISOString(),
|
|
185
|
+
cwd: sessionCwd,
|
|
186
|
+
},
|
|
187
|
+
"session",
|
|
188
|
+
);
|
|
189
|
+
|
|
190
|
+
/** @type {string | null} */
|
|
191
|
+
/** @type {string | null} */
|
|
192
|
+
let tip = null;
|
|
193
|
+
/** @type {string[]} recent user entries usable as branch points */
|
|
194
|
+
const branchPoints = [];
|
|
195
|
+
/** @type {Set<string>} parents that already have a child */
|
|
196
|
+
const hasChild = new Set();
|
|
197
|
+
/** @type {string[]} */
|
|
198
|
+
const labelable = [];
|
|
199
|
+
const stamp = () => {
|
|
200
|
+
clock += 1500 + Math.floor(r() * 6000);
|
|
201
|
+
return {
|
|
202
|
+
iso: new Date(baseMs + clock).toISOString(),
|
|
203
|
+
ms: baseMs + clock,
|
|
204
|
+
};
|
|
205
|
+
};
|
|
206
|
+
/**
|
|
207
|
+
* @param {object} body
|
|
208
|
+
* @param {string} kind
|
|
209
|
+
* @param {string | null} [parent]
|
|
210
|
+
*/
|
|
211
|
+
const add = (body, kind, parent) => {
|
|
212
|
+
const id = nextId();
|
|
213
|
+
const parentId = parent === undefined ? tip : parent;
|
|
214
|
+
if (parentId !== null && hasChild.has(parentId)) branches++;
|
|
215
|
+
if (parentId !== null) hasChild.add(parentId);
|
|
216
|
+
emit({ ...body, id, parentId, timestamp: stamp().iso }, kind);
|
|
217
|
+
entries++;
|
|
218
|
+
tip = id;
|
|
219
|
+
return id;
|
|
220
|
+
};
|
|
221
|
+
/** @param {Record<string, unknown>} body */
|
|
222
|
+
const message = (body) => ({
|
|
223
|
+
type: "message",
|
|
224
|
+
message: { ...body, timestamp: baseMs + clock },
|
|
225
|
+
});
|
|
226
|
+
/** @param {number} out */
|
|
227
|
+
const usage = (out) => {
|
|
228
|
+
const input = 800 + Math.floor(r() * 40000);
|
|
229
|
+
return {
|
|
230
|
+
input,
|
|
231
|
+
output: out,
|
|
232
|
+
cacheRead: Math.floor(r() * 90000),
|
|
233
|
+
cacheWrite: Math.floor(r() * 3000),
|
|
234
|
+
totalTokens: input + out,
|
|
235
|
+
cost: {
|
|
236
|
+
input: 0.003,
|
|
237
|
+
output: 0.004,
|
|
238
|
+
cacheRead: 0.001,
|
|
239
|
+
cacheWrite: 0.002,
|
|
240
|
+
total: 0.01,
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
add(
|
|
246
|
+
message({
|
|
247
|
+
role: "system",
|
|
248
|
+
content: "",
|
|
249
|
+
sections: { preamble: text(600), cwd: sessionCwd },
|
|
250
|
+
toolsAdded: TOOLS.map((t) => ({
|
|
251
|
+
name: t.name,
|
|
252
|
+
description: text(120),
|
|
253
|
+
parameters: { type: "object" },
|
|
254
|
+
})),
|
|
255
|
+
}),
|
|
256
|
+
"message:system",
|
|
257
|
+
);
|
|
258
|
+
add(
|
|
259
|
+
{
|
|
260
|
+
type: "model_change",
|
|
261
|
+
provider: "anthropic",
|
|
262
|
+
modelId: "claude-sonnet-4-5",
|
|
263
|
+
},
|
|
264
|
+
"model_change",
|
|
265
|
+
);
|
|
266
|
+
add(
|
|
267
|
+
{ type: "thinking_level_change", thinkingLevel: "medium" },
|
|
268
|
+
"thinking_level_change",
|
|
269
|
+
);
|
|
270
|
+
add({ type: "session_info", name: "synthetic session" }, "session_info");
|
|
271
|
+
|
|
272
|
+
let turn = 0;
|
|
273
|
+
while (bytes < targetBytes) {
|
|
274
|
+
turn++;
|
|
275
|
+
// Occasionally continue from an earlier user entry: a branch.
|
|
276
|
+
/** @type {string | null} */
|
|
277
|
+
let parent = tip;
|
|
278
|
+
if (turn > 20 && branchPoints.length > 5 && r() < 0.004) {
|
|
279
|
+
parent = branchPoints[Math.floor(r() * branchPoints.length)];
|
|
280
|
+
if (r() < 0.5) {
|
|
281
|
+
parent = add(
|
|
282
|
+
{
|
|
283
|
+
type: "branch_summary",
|
|
284
|
+
fromId: tip,
|
|
285
|
+
summary: text(200 + Math.floor(r() * 800)),
|
|
286
|
+
details: { readFiles: [`src/${word(r)}.ts`], modifiedFiles: [] },
|
|
287
|
+
},
|
|
288
|
+
"branch_summary",
|
|
289
|
+
parent,
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
const userId = add(
|
|
294
|
+
message({ role: "user", content: text(logInt(20, 2500)) }),
|
|
295
|
+
"message:user",
|
|
296
|
+
parent,
|
|
297
|
+
);
|
|
298
|
+
branchPoints.push(userId);
|
|
299
|
+
if (branchPoints.length > 400) branchPoints.splice(0, 100);
|
|
300
|
+
labelable.push(userId);
|
|
301
|
+
|
|
302
|
+
const steps = 1 + Math.floor(r() * 4);
|
|
303
|
+
for (let s = 0; s < steps; s++) {
|
|
304
|
+
const nCalls = r() < 0.15 ? 0 : 1 + Math.floor(r() * 3);
|
|
305
|
+
/** @type {object[]} */
|
|
306
|
+
const content = [];
|
|
307
|
+
if (r() < 0.8) {
|
|
308
|
+
content.push({
|
|
309
|
+
type: "thinking",
|
|
310
|
+
thinking: text(logInt(100, 4000)),
|
|
311
|
+
thinkingSignature: text(120).replace(/\s+/g, ""),
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
if (r() < 0.7 || nCalls === 0)
|
|
315
|
+
content.push({ type: "text", text: text(logInt(30, 1800)) });
|
|
316
|
+
/** @type {{id: string, name: string}[]} */
|
|
317
|
+
const calls = [];
|
|
318
|
+
for (let c = 0; c < nCalls; c++) {
|
|
319
|
+
const tool = TOOLS[Math.floor(r() * TOOLS.length)];
|
|
320
|
+
const callId = `toolu_${nextIdLike(r)}`;
|
|
321
|
+
calls.push({ id: callId, name: tool.name });
|
|
322
|
+
content.push({
|
|
323
|
+
type: "toolCall",
|
|
324
|
+
id: callId,
|
|
325
|
+
name: tool.name,
|
|
326
|
+
arguments: tool.args(r),
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
const out = 50 + Math.floor(r() * 1500);
|
|
330
|
+
add(
|
|
331
|
+
message({
|
|
332
|
+
role: "assistant",
|
|
333
|
+
content,
|
|
334
|
+
api: "anthropic-messages",
|
|
335
|
+
provider: "anthropic",
|
|
336
|
+
model: "claude-sonnet-4-5",
|
|
337
|
+
thinkingLevel: "medium",
|
|
338
|
+
usage: usage(out),
|
|
339
|
+
stopReason: nCalls ? "toolUse" : "stop",
|
|
340
|
+
}),
|
|
341
|
+
"message:assistant",
|
|
342
|
+
);
|
|
343
|
+
for (const call of calls) {
|
|
344
|
+
// tiny (tens of bytes) up to ~50 KB, heavy on the small side
|
|
345
|
+
const size = r() < 0.04 ? logInt(20000, 50000) : logInt(20, 6000);
|
|
346
|
+
const id = add(
|
|
347
|
+
message({
|
|
348
|
+
role: "toolResult",
|
|
349
|
+
toolCallId: call.id,
|
|
350
|
+
toolName: call.name,
|
|
351
|
+
content: [{ type: "text", text: text(size) }],
|
|
352
|
+
isError: r() < 0.05,
|
|
353
|
+
}),
|
|
354
|
+
"message:toolResult",
|
|
355
|
+
);
|
|
356
|
+
if (r() < 0.02) labelable.push(id);
|
|
357
|
+
}
|
|
358
|
+
if (nCalls === 0) break;
|
|
359
|
+
}
|
|
360
|
+
if (r() < 0.06) {
|
|
361
|
+
add(
|
|
362
|
+
{
|
|
363
|
+
type: "custom",
|
|
364
|
+
customType: CUSTOM_TYPES[Math.floor(r() * CUSTOM_TYPES.length)],
|
|
365
|
+
data: { count: turn, note: text(logInt(10, 300)) },
|
|
366
|
+
},
|
|
367
|
+
"custom",
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
if (r() < 0.012) {
|
|
371
|
+
add(
|
|
372
|
+
{
|
|
373
|
+
type: "custom_message",
|
|
374
|
+
customType: "synthetic-injected-context",
|
|
375
|
+
content: text(logInt(100, 2000)),
|
|
376
|
+
display: r() < 0.5,
|
|
377
|
+
},
|
|
378
|
+
"custom_message",
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
if (r() < 0.02) {
|
|
382
|
+
// usage: model-attributed accounting that is not a message (e.g. cache
|
|
383
|
+
// warming); hidden from the conversation tree
|
|
384
|
+
add(
|
|
385
|
+
{
|
|
386
|
+
type: "usage",
|
|
387
|
+
kind: "cache_warm",
|
|
388
|
+
provider: "anthropic",
|
|
389
|
+
model: "claude-sonnet-4-5",
|
|
390
|
+
usage: {
|
|
391
|
+
input: 0,
|
|
392
|
+
output: 0,
|
|
393
|
+
cacheRead: 20000 + Math.floor(r() * 80000),
|
|
394
|
+
cacheWrite: 0,
|
|
395
|
+
totalTokens: 0,
|
|
396
|
+
cost: {
|
|
397
|
+
input: 0,
|
|
398
|
+
output: 0,
|
|
399
|
+
cacheRead: 0.01,
|
|
400
|
+
cacheWrite: 0,
|
|
401
|
+
total: 0.01,
|
|
402
|
+
},
|
|
403
|
+
},
|
|
404
|
+
},
|
|
405
|
+
"usage",
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
if (r() < 0.004) {
|
|
409
|
+
// compaction: summary + prompt checkpoint; firstKeptEntryId keeps the
|
|
410
|
+
// tail of the history, like a real long session
|
|
411
|
+
const kept = branchPoints[Math.max(0, branchPoints.length - 1)];
|
|
412
|
+
add(
|
|
413
|
+
{
|
|
414
|
+
type: "compaction",
|
|
415
|
+
summary: text(logInt(500, 4000)),
|
|
416
|
+
firstKeptEntryId: kept,
|
|
417
|
+
tokensBefore: 60000 + Math.floor(r() * 100000),
|
|
418
|
+
systemMessage: {
|
|
419
|
+
role: "system",
|
|
420
|
+
content: "",
|
|
421
|
+
sections: { preamble: text(600) },
|
|
422
|
+
toolsAdded: [],
|
|
423
|
+
timestamp: baseMs + clock,
|
|
424
|
+
},
|
|
425
|
+
details: {
|
|
426
|
+
readFiles: [`src/${word(r)}.ts`, `docs/${word(r)}.md`],
|
|
427
|
+
modifiedFiles: [`src/${word(r)}.ts`],
|
|
428
|
+
},
|
|
429
|
+
},
|
|
430
|
+
"compaction",
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
if (r() < 0.01) {
|
|
434
|
+
add(
|
|
435
|
+
{
|
|
436
|
+
type: "label",
|
|
437
|
+
targetId: labelable[Math.floor(r() * labelable.length)],
|
|
438
|
+
label: `checkpoint-${turn}`,
|
|
439
|
+
},
|
|
440
|
+
"label",
|
|
441
|
+
);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
flush();
|
|
445
|
+
} finally {
|
|
446
|
+
fs.closeSync(fd);
|
|
447
|
+
}
|
|
448
|
+
return { bytes, entries, branches, types };
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** @param {() => number} r */
|
|
452
|
+
function nextIdLike(r) {
|
|
453
|
+
let s = "";
|
|
454
|
+
for (let i = 0; i < 16; i++) s += Math.floor(r() * 16).toString(16);
|
|
455
|
+
return s;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Cache path of a fixture for (seed, mb) under dir.
|
|
460
|
+
* @param {string} dir
|
|
461
|
+
* @param {number} mb
|
|
462
|
+
* @param {number} seed
|
|
463
|
+
*/
|
|
464
|
+
export function fixturePath(dir, mb, seed) {
|
|
465
|
+
return path.join(dir, `session-v${FIXTURE_FORMAT}-seed${seed}-${mb}mb.jsonl`);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* Returns the cached fixture for (seed, mb), generating it when absent. The
|
|
470
|
+
* file is written under a temporary name and renamed, so a half-written file
|
|
471
|
+
* is never reused.
|
|
472
|
+
* @param {string} dir
|
|
473
|
+
* @param {number} mb
|
|
474
|
+
* @param {number} seed
|
|
475
|
+
* @returns {{ file: string, generated: boolean }}
|
|
476
|
+
*/
|
|
477
|
+
export function ensureFixture(dir, mb, seed) {
|
|
478
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
479
|
+
const file = fixturePath(dir, mb, seed);
|
|
480
|
+
if (fs.existsSync(file) && fs.statSync(file).size > 0)
|
|
481
|
+
return { file, generated: false };
|
|
482
|
+
const tmp = `${file}.${process.pid}.tmp`;
|
|
483
|
+
try {
|
|
484
|
+
writeFixture(tmp, { mb, seed });
|
|
485
|
+
fs.renameSync(tmp, file);
|
|
486
|
+
} catch (err) {
|
|
487
|
+
fs.rmSync(tmp, { force: true });
|
|
488
|
+
throw err;
|
|
489
|
+
}
|
|
490
|
+
return { file, generated: true };
|
|
491
|
+
}
|