@remnic/bench 9.3.729 → 9.3.730
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +403 -2
- package/dist/index.js +1702 -305
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,3 +1,598 @@
|
|
|
1
|
+
// src/benchmarks/remnic/memcorrect/third-party/shared.ts
|
|
2
|
+
var MissingCredentialError = class extends Error {
|
|
3
|
+
reason;
|
|
4
|
+
constructor(system, missing) {
|
|
5
|
+
const reason = `${system} MemCorrect adapter is not runnable: missing ${missing.join(", ")}. Provide the required credentials (env vars or config) to exercise this adapter; skipped \u2014 no keys in CI.`;
|
|
6
|
+
super(reason);
|
|
7
|
+
this.name = "MissingCredentialError";
|
|
8
|
+
this.reason = reason;
|
|
9
|
+
}
|
|
10
|
+
};
|
|
11
|
+
function requireCredentials(system, missing) {
|
|
12
|
+
if (missing.length > 0) {
|
|
13
|
+
throw new MissingCredentialError(system, missing);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
function resolveFetch(override) {
|
|
17
|
+
if (override) return override;
|
|
18
|
+
if (typeof fetch === "function") return fetch;
|
|
19
|
+
throw new Error(
|
|
20
|
+
"No fetch implementation available. Provide `fetch` in the adapter config or run in a runtime with a global fetch (Node 18+)."
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
async function httpJson(fetchImpl, method, url, options = {}) {
|
|
24
|
+
const { headers = {}, body, timeoutMs } = options;
|
|
25
|
+
const controller = new AbortController();
|
|
26
|
+
const { promise: timeoutPromise, resolve: resolveTimeout } = Promise.withResolvers();
|
|
27
|
+
const timer = timeoutMs !== void 0 ? setTimeout(() => resolveTimeout(), timeoutMs) : void 0;
|
|
28
|
+
if (timer) timer.unref?.();
|
|
29
|
+
const fetchPromise = fetchImpl(url, {
|
|
30
|
+
method,
|
|
31
|
+
headers: { "Content-Type": "application/json", ...headers },
|
|
32
|
+
body: body !== void 0 ? JSON.stringify(body) : void 0,
|
|
33
|
+
signal: controller.signal
|
|
34
|
+
});
|
|
35
|
+
fetchPromise.catch(() => {
|
|
36
|
+
});
|
|
37
|
+
let response;
|
|
38
|
+
try {
|
|
39
|
+
if (timer) {
|
|
40
|
+
const winner = await Promise.race([
|
|
41
|
+
fetchPromise.then((r) => r),
|
|
42
|
+
timeoutPromise.then(() => "timeout")
|
|
43
|
+
]);
|
|
44
|
+
if (winner === "timeout") {
|
|
45
|
+
controller.abort();
|
|
46
|
+
throw new Error(`Request timed out after ${timeoutMs}ms: ${method} ${url}`);
|
|
47
|
+
}
|
|
48
|
+
response = winner;
|
|
49
|
+
} else {
|
|
50
|
+
response = await fetchPromise;
|
|
51
|
+
}
|
|
52
|
+
} finally {
|
|
53
|
+
if (timer) clearTimeout(timer);
|
|
54
|
+
}
|
|
55
|
+
if (response.status === 204) return null;
|
|
56
|
+
const text = await response.text();
|
|
57
|
+
if (!response.ok) {
|
|
58
|
+
const excerpt = text.length > 500 ? `${text.slice(0, 500)}\u2026` : text;
|
|
59
|
+
throw new HttpError(method, url, response.status, excerpt);
|
|
60
|
+
}
|
|
61
|
+
if (text.length === 0) return null;
|
|
62
|
+
try {
|
|
63
|
+
return JSON.parse(text);
|
|
64
|
+
} catch {
|
|
65
|
+
return text;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
var HttpError = class extends Error {
|
|
69
|
+
status;
|
|
70
|
+
method;
|
|
71
|
+
url;
|
|
72
|
+
bodyExcerpt;
|
|
73
|
+
constructor(method, url, status, bodyExcerpt) {
|
|
74
|
+
super(`HTTP ${status} from ${method} ${url}: ${bodyExcerpt}`);
|
|
75
|
+
this.name = "HttpError";
|
|
76
|
+
this.method = method;
|
|
77
|
+
this.url = url;
|
|
78
|
+
this.status = status;
|
|
79
|
+
this.bodyExcerpt = bodyExcerpt;
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
function delay(ms) {
|
|
83
|
+
if (ms <= 0) return Promise.resolve();
|
|
84
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
85
|
+
setTimeout(() => resolve(), ms);
|
|
86
|
+
return promise;
|
|
87
|
+
}
|
|
88
|
+
var runSuffixCounter = 0;
|
|
89
|
+
function uniqueRunSuffix() {
|
|
90
|
+
runSuffixCounter += 1;
|
|
91
|
+
const pid = typeof process !== "undefined" && typeof process.pid === "number" ? process.pid : 0;
|
|
92
|
+
return `${pid.toString(36)}-${Date.now().toString(36)}-${runSuffixCounter.toString(36)}`;
|
|
93
|
+
}
|
|
94
|
+
function isNotFoundDelete(err) {
|
|
95
|
+
return err instanceof HttpError && (err.status === 404 || err.status === 422);
|
|
96
|
+
}
|
|
97
|
+
function isConflict(err) {
|
|
98
|
+
return err instanceof HttpError && err.status === 409;
|
|
99
|
+
}
|
|
100
|
+
async function resetTrackedIds(system, ids, deleteFn) {
|
|
101
|
+
const snapshot = [...ids];
|
|
102
|
+
const failed = [];
|
|
103
|
+
for (const id of snapshot) {
|
|
104
|
+
try {
|
|
105
|
+
await deleteFn(id);
|
|
106
|
+
ids.delete(id);
|
|
107
|
+
} catch (err) {
|
|
108
|
+
if (isNotFoundDelete(err)) {
|
|
109
|
+
ids.delete(id);
|
|
110
|
+
} else {
|
|
111
|
+
failed.push(id);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (failed.length > 0) {
|
|
116
|
+
throw new Error(
|
|
117
|
+
`${system} reset could not clean ${failed.length} session(s): ${failed.join(", ")}. Remote data may remain; the failed ids are retained for the next reset() retry.`
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// src/benchmarks/remnic/memcorrect/third-party/mem0-adapter.ts
|
|
123
|
+
var Mem0MemCorrectAdapter = class {
|
|
124
|
+
label;
|
|
125
|
+
mode;
|
|
126
|
+
baseUrl;
|
|
127
|
+
apiKey;
|
|
128
|
+
userIdPrefix;
|
|
129
|
+
ossAuthMode;
|
|
130
|
+
pollIntervalMs;
|
|
131
|
+
maxPolls;
|
|
132
|
+
fetchImpl;
|
|
133
|
+
timeoutMs;
|
|
134
|
+
/** Session-scoped user_ids we have ingested under, for precise reset. */
|
|
135
|
+
knownSessions = /* @__PURE__ */ new Set();
|
|
136
|
+
constructor(config = {}) {
|
|
137
|
+
this.mode = config.mode ?? (config.baseUrl && !config.baseUrl.includes("api.mem0.ai") ? "oss" : "hosted");
|
|
138
|
+
this.baseUrl = config.baseUrl?.replace(/\/+$/, "") ?? (this.mode === "hosted" ? "https://api.mem0.ai" : "");
|
|
139
|
+
this.apiKey = config.apiKey;
|
|
140
|
+
this.userIdPrefix = config.userIdPrefix ?? `memcorrect-${uniqueRunSuffix()}`;
|
|
141
|
+
this.ossAuthMode = config.ossAuthMode ?? "x-api-key";
|
|
142
|
+
this.pollIntervalMs = config.pollIntervalMs ?? 500;
|
|
143
|
+
this.maxPolls = config.maxPolls ?? 120;
|
|
144
|
+
this.fetchImpl = resolveFetch(config.fetch);
|
|
145
|
+
this.timeoutMs = config.timeoutMs;
|
|
146
|
+
this.label = `mem0-${this.mode}`;
|
|
147
|
+
}
|
|
148
|
+
/** Whether this adapter has the credentials needed to run. */
|
|
149
|
+
isConfigured() {
|
|
150
|
+
if (!this.apiKey) return false;
|
|
151
|
+
if (this.mode === "oss" && !this.baseUrl) return false;
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
userIdFor(sessionKey) {
|
|
155
|
+
return `${this.userIdPrefix}:${sessionKey}`;
|
|
156
|
+
}
|
|
157
|
+
ensureReady() {
|
|
158
|
+
const missing = [];
|
|
159
|
+
if (!this.apiKey) missing.push("apiKey (MEM0_API_KEY)");
|
|
160
|
+
if (this.mode === "oss" && !this.baseUrl)
|
|
161
|
+
missing.push("baseUrl (MEM0_BASE_URL)");
|
|
162
|
+
requireCredentials("Mem0", missing);
|
|
163
|
+
}
|
|
164
|
+
authHeaders() {
|
|
165
|
+
if (this.mode === "hosted") {
|
|
166
|
+
return { Authorization: `Token ${this.apiKey}` };
|
|
167
|
+
}
|
|
168
|
+
if (this.ossAuthMode === "bearer") {
|
|
169
|
+
return { Authorization: `Bearer ${this.apiKey}` };
|
|
170
|
+
}
|
|
171
|
+
return { "X-API-Key": this.apiKey };
|
|
172
|
+
}
|
|
173
|
+
async reset() {
|
|
174
|
+
this.ensureReady();
|
|
175
|
+
await resetTrackedIds("Mem0", this.knownSessions, async (sessionKey) => {
|
|
176
|
+
const userId = this.userIdFor(sessionKey);
|
|
177
|
+
if (this.mode === "oss") {
|
|
178
|
+
await httpJson(
|
|
179
|
+
this.fetchImpl,
|
|
180
|
+
"DELETE",
|
|
181
|
+
`${this.baseUrl}/memories?user_id=${encodeURIComponent(userId)}`,
|
|
182
|
+
{ headers: this.authHeaders(), timeoutMs: this.timeoutMs }
|
|
183
|
+
);
|
|
184
|
+
} else {
|
|
185
|
+
await httpJson(
|
|
186
|
+
this.fetchImpl,
|
|
187
|
+
"DELETE",
|
|
188
|
+
`${this.baseUrl}/v1/memories/?user_id=${encodeURIComponent(userId)}`,
|
|
189
|
+
{ headers: this.authHeaders(), timeoutMs: this.timeoutMs }
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
async ingestTurn(sessionKey, role, text, _at) {
|
|
195
|
+
this.ensureReady();
|
|
196
|
+
const userId = this.userIdFor(sessionKey);
|
|
197
|
+
this.knownSessions.add(sessionKey);
|
|
198
|
+
if (this.mode === "oss") {
|
|
199
|
+
await httpJson(this.fetchImpl, "POST", `${this.baseUrl}/memories`, {
|
|
200
|
+
headers: this.authHeaders(),
|
|
201
|
+
body: {
|
|
202
|
+
messages: [{ role, content: text }],
|
|
203
|
+
user_id: userId
|
|
204
|
+
},
|
|
205
|
+
timeoutMs: this.timeoutMs
|
|
206
|
+
});
|
|
207
|
+
} else {
|
|
208
|
+
const addResponse = await httpJson(
|
|
209
|
+
this.fetchImpl,
|
|
210
|
+
"POST",
|
|
211
|
+
`${this.baseUrl}/v3/memories/add/`,
|
|
212
|
+
{
|
|
213
|
+
headers: this.authHeaders(),
|
|
214
|
+
body: {
|
|
215
|
+
messages: [{ role, content: text }],
|
|
216
|
+
user_id: userId
|
|
217
|
+
},
|
|
218
|
+
timeoutMs: this.timeoutMs
|
|
219
|
+
}
|
|
220
|
+
);
|
|
221
|
+
if (addResponse?.status === "FAILED") {
|
|
222
|
+
throw new Error(
|
|
223
|
+
`Mem0 hosted add failed immediately: ${addResponse.error ?? addResponse.status}`
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
const eventId = addResponse?.event_id;
|
|
227
|
+
if (eventId) {
|
|
228
|
+
await this.pollEvent(eventId);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
async recall(query, sessionKey) {
|
|
233
|
+
this.ensureReady();
|
|
234
|
+
const userId = this.userIdFor(sessionKey);
|
|
235
|
+
let results;
|
|
236
|
+
if (this.mode === "oss") {
|
|
237
|
+
const body = await httpJson(
|
|
238
|
+
this.fetchImpl,
|
|
239
|
+
"POST",
|
|
240
|
+
`${this.baseUrl}/search`,
|
|
241
|
+
{
|
|
242
|
+
headers: this.authHeaders(),
|
|
243
|
+
// Mem0 OSS v3 aligns search with the platform API: entity IDs go
|
|
244
|
+
// inside `filters` and the count parameter is `top_k`, not
|
|
245
|
+
// top-level `user_id`/`limit` (oss-v2-to-v3 migration).
|
|
246
|
+
body: { query, filters: { user_id: userId }, top_k: 10 },
|
|
247
|
+
timeoutMs: this.timeoutMs
|
|
248
|
+
}
|
|
249
|
+
);
|
|
250
|
+
results = normalizeSearchResults(body);
|
|
251
|
+
} else {
|
|
252
|
+
const body = await httpJson(
|
|
253
|
+
this.fetchImpl,
|
|
254
|
+
"POST",
|
|
255
|
+
`${this.baseUrl}/v3/memories/search/`,
|
|
256
|
+
{
|
|
257
|
+
headers: this.authHeaders(),
|
|
258
|
+
body: { query, filters: { user_id: userId }, top_k: 10 },
|
|
259
|
+
timeoutMs: this.timeoutMs
|
|
260
|
+
}
|
|
261
|
+
);
|
|
262
|
+
results = normalizeSearchResults(body);
|
|
263
|
+
}
|
|
264
|
+
return results.map((r) => r.memory).filter((m) => typeof m === "string" && m.length > 0);
|
|
265
|
+
}
|
|
266
|
+
async correct(text, sessionKey, _at) {
|
|
267
|
+
await this.ingestTurn(sessionKey, "user", text, _at ?? (/* @__PURE__ */ new Date()).toISOString());
|
|
268
|
+
}
|
|
269
|
+
async runMaintenance() {
|
|
270
|
+
}
|
|
271
|
+
/** Poll the hosted event endpoint until the add is processed. */
|
|
272
|
+
async pollEvent(eventId) {
|
|
273
|
+
for (let i = 0; i < this.maxPolls; i++) {
|
|
274
|
+
await delay(this.pollIntervalMs);
|
|
275
|
+
const status = await httpJson(
|
|
276
|
+
this.fetchImpl,
|
|
277
|
+
"GET",
|
|
278
|
+
`${this.baseUrl}/v1/event/${eventId}/`,
|
|
279
|
+
{
|
|
280
|
+
headers: this.authHeaders(),
|
|
281
|
+
timeoutMs: this.timeoutMs
|
|
282
|
+
}
|
|
283
|
+
);
|
|
284
|
+
if (status?.status === "SUCCEEDED") return;
|
|
285
|
+
if (status?.status === "FAILED") {
|
|
286
|
+
throw new Error(
|
|
287
|
+
`Mem0 add event ${eventId} failed: ${status.error ?? "unknown"}`
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
throw new Error(
|
|
292
|
+
`Mem0 add event ${eventId} did not complete after ${this.maxPolls} polls`
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
function normalizeSearchResults(body) {
|
|
297
|
+
if (!body) return [];
|
|
298
|
+
if (Array.isArray(body)) return body;
|
|
299
|
+
return body.results ?? [];
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// src/benchmarks/remnic/memcorrect/third-party/zep-adapter.ts
|
|
303
|
+
var ZepMemCorrectAdapter = class {
|
|
304
|
+
label = "zep";
|
|
305
|
+
baseUrl;
|
|
306
|
+
apiKey;
|
|
307
|
+
sessionPrefix;
|
|
308
|
+
settleMs;
|
|
309
|
+
fetchImpl;
|
|
310
|
+
timeoutMs;
|
|
311
|
+
/** Sessions we have ensured exist, to avoid redundant POST /sessions calls. */
|
|
312
|
+
knownSessions = /* @__PURE__ */ new Set();
|
|
313
|
+
/** True when turns have been ingested since the last settle, so recall() can
|
|
314
|
+
* wait for Zep's async graph pipeline before a scored read. */
|
|
315
|
+
pendingIngest = false;
|
|
316
|
+
constructor(config = {}) {
|
|
317
|
+
this.baseUrl = config.baseUrl?.replace(/\/+$/, "") ?? "https://api.getzep.com/api/v2";
|
|
318
|
+
this.apiKey = config.apiKey;
|
|
319
|
+
this.sessionPrefix = config.sessionPrefix ?? `memcorrect-${uniqueRunSuffix()}`;
|
|
320
|
+
this.settleMs = config.settleMs ?? 0;
|
|
321
|
+
this.fetchImpl = resolveFetch(config.fetch);
|
|
322
|
+
this.timeoutMs = config.timeoutMs;
|
|
323
|
+
}
|
|
324
|
+
isConfigured() {
|
|
325
|
+
return !!this.apiKey;
|
|
326
|
+
}
|
|
327
|
+
sessionIdFor(sessionKey) {
|
|
328
|
+
return `${this.sessionPrefix}:${sessionKey}`;
|
|
329
|
+
}
|
|
330
|
+
ensureReady() {
|
|
331
|
+
requireCredentials("Zep", this.apiKey ? [] : ["apiKey (ZEP_API_KEY)"]);
|
|
332
|
+
}
|
|
333
|
+
authHeaders() {
|
|
334
|
+
return { Authorization: `Api-Key ${this.apiKey}` };
|
|
335
|
+
}
|
|
336
|
+
async reset() {
|
|
337
|
+
this.ensureReady();
|
|
338
|
+
await resetTrackedIds("Zep", this.knownSessions, async (sessionId) => {
|
|
339
|
+
try {
|
|
340
|
+
await httpJson(
|
|
341
|
+
this.fetchImpl,
|
|
342
|
+
"DELETE",
|
|
343
|
+
`${this.baseUrl}/sessions/${encodeURIComponent(sessionId)}`,
|
|
344
|
+
{ headers: this.authHeaders(), timeoutMs: this.timeoutMs }
|
|
345
|
+
);
|
|
346
|
+
} catch (err) {
|
|
347
|
+
if (!isNotFoundDelete(err)) throw err;
|
|
348
|
+
}
|
|
349
|
+
await httpJson(
|
|
350
|
+
this.fetchImpl,
|
|
351
|
+
"DELETE",
|
|
352
|
+
`${this.baseUrl}/users/${encodeURIComponent(sessionId)}`,
|
|
353
|
+
{ headers: this.authHeaders(), timeoutMs: this.timeoutMs }
|
|
354
|
+
);
|
|
355
|
+
});
|
|
356
|
+
this.pendingIngest = false;
|
|
357
|
+
}
|
|
358
|
+
/** Ensure the Zep session (and its user) exist before adding memory. */
|
|
359
|
+
async ensureSession(sessionId) {
|
|
360
|
+
if (this.knownSessions.has(sessionId)) return;
|
|
361
|
+
try {
|
|
362
|
+
await httpJson(this.fetchImpl, "POST", `${this.baseUrl}/users`, {
|
|
363
|
+
headers: this.authHeaders(),
|
|
364
|
+
body: { user_id: sessionId },
|
|
365
|
+
timeoutMs: this.timeoutMs
|
|
366
|
+
});
|
|
367
|
+
} catch (err) {
|
|
368
|
+
if (!isConflict(err)) throw err;
|
|
369
|
+
}
|
|
370
|
+
try {
|
|
371
|
+
await httpJson(
|
|
372
|
+
this.fetchImpl,
|
|
373
|
+
"POST",
|
|
374
|
+
`${this.baseUrl}/sessions`,
|
|
375
|
+
{
|
|
376
|
+
headers: this.authHeaders(),
|
|
377
|
+
body: { session_id: sessionId, user_id: sessionId },
|
|
378
|
+
timeoutMs: this.timeoutMs
|
|
379
|
+
}
|
|
380
|
+
);
|
|
381
|
+
} catch (err) {
|
|
382
|
+
if (!isConflict(err)) throw err;
|
|
383
|
+
}
|
|
384
|
+
this.knownSessions.add(sessionId);
|
|
385
|
+
}
|
|
386
|
+
async ingestTurn(sessionKey, role, text, _at) {
|
|
387
|
+
this.ensureReady();
|
|
388
|
+
const sessionId = this.sessionIdFor(sessionKey);
|
|
389
|
+
await this.ensureSession(sessionId);
|
|
390
|
+
const roleType = role === "user" ? "user" : "assistant";
|
|
391
|
+
await httpJson(
|
|
392
|
+
this.fetchImpl,
|
|
393
|
+
"POST",
|
|
394
|
+
`${this.baseUrl}/sessions/${encodeURIComponent(sessionId)}/memory`,
|
|
395
|
+
{
|
|
396
|
+
headers: this.authHeaders(),
|
|
397
|
+
body: {
|
|
398
|
+
messages: [{ role, role_type: roleType, content: text }]
|
|
399
|
+
},
|
|
400
|
+
timeoutMs: this.timeoutMs
|
|
401
|
+
}
|
|
402
|
+
);
|
|
403
|
+
this.pendingIngest = true;
|
|
404
|
+
}
|
|
405
|
+
async recall(query, sessionKey) {
|
|
406
|
+
this.ensureReady();
|
|
407
|
+
if (this.pendingIngest && this.settleMs > 0) {
|
|
408
|
+
await delay(this.settleMs);
|
|
409
|
+
this.pendingIngest = false;
|
|
410
|
+
}
|
|
411
|
+
const sessionId = this.sessionIdFor(sessionKey);
|
|
412
|
+
await this.ensureSession(sessionId);
|
|
413
|
+
const results = await httpJson(
|
|
414
|
+
this.fetchImpl,
|
|
415
|
+
"POST",
|
|
416
|
+
`${this.baseUrl}/graph/search`,
|
|
417
|
+
{
|
|
418
|
+
headers: this.authHeaders(),
|
|
419
|
+
body: { user_id: sessionId, query, scope: "edges", limit: 10 },
|
|
420
|
+
timeoutMs: this.timeoutMs
|
|
421
|
+
}
|
|
422
|
+
);
|
|
423
|
+
if (!results || !results.edges) return [];
|
|
424
|
+
const strings = [];
|
|
425
|
+
for (const edge of results.edges) {
|
|
426
|
+
const fact3 = edge.fact;
|
|
427
|
+
if (fact3 && fact3.trim().length > 0) {
|
|
428
|
+
strings.push(fact3.trim());
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return strings;
|
|
432
|
+
}
|
|
433
|
+
async correct(text, sessionKey, _at) {
|
|
434
|
+
await this.ingestTurn(
|
|
435
|
+
sessionKey,
|
|
436
|
+
"user",
|
|
437
|
+
text,
|
|
438
|
+
_at ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
async runMaintenance() {
|
|
442
|
+
if (this.settleMs > 0 && this.pendingIngest) {
|
|
443
|
+
await delay(this.settleMs);
|
|
444
|
+
this.pendingIngest = false;
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
// src/benchmarks/remnic/memcorrect/third-party/letta-adapter.ts
|
|
450
|
+
var LettaMemCorrectAdapter = class {
|
|
451
|
+
label = "letta";
|
|
452
|
+
baseUrl;
|
|
453
|
+
apiKey;
|
|
454
|
+
model;
|
|
455
|
+
agentNamePrefix;
|
|
456
|
+
personaBlock;
|
|
457
|
+
fetchImpl;
|
|
458
|
+
timeoutMs;
|
|
459
|
+
/** Maps MemCorrect sessionKey → Letta agent_id. */
|
|
460
|
+
agentsBySession = /* @__PURE__ */ new Map();
|
|
461
|
+
constructor(config = {}) {
|
|
462
|
+
this.baseUrl = config.baseUrl?.replace(/\/+$/, "") ?? "";
|
|
463
|
+
this.apiKey = config.apiKey;
|
|
464
|
+
this.model = config.model;
|
|
465
|
+
this.agentNamePrefix = config.agentNamePrefix ?? "memcorrect";
|
|
466
|
+
this.personaBlock = config.personaBlock ?? "You are a memory benchmark agent. Store every fact the user states in your human memory block. When the user corrects a fact, use memory_replace to update it.";
|
|
467
|
+
this.fetchImpl = resolveFetch(config.fetch);
|
|
468
|
+
this.timeoutMs = config.timeoutMs;
|
|
469
|
+
}
|
|
470
|
+
isConfigured() {
|
|
471
|
+
return !!(this.apiKey && this.baseUrl && this.model);
|
|
472
|
+
}
|
|
473
|
+
ensureReady() {
|
|
474
|
+
const missing = [];
|
|
475
|
+
if (!this.apiKey) missing.push("apiKey (LETTA_API_KEY)");
|
|
476
|
+
if (!this.baseUrl) missing.push("baseUrl (LETTA_BASE_URL)");
|
|
477
|
+
if (!this.model) missing.push("model (LETTA_MODEL)");
|
|
478
|
+
requireCredentials("Letta", missing);
|
|
479
|
+
}
|
|
480
|
+
authHeaders() {
|
|
481
|
+
return this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {};
|
|
482
|
+
}
|
|
483
|
+
async reset() {
|
|
484
|
+
this.ensureReady();
|
|
485
|
+
const failed = [];
|
|
486
|
+
for (const [sessionKey, agentId] of [...this.agentsBySession]) {
|
|
487
|
+
try {
|
|
488
|
+
await httpJson(
|
|
489
|
+
this.fetchImpl,
|
|
490
|
+
"DELETE",
|
|
491
|
+
`${this.baseUrl}/v1/agents/${encodeURIComponent(agentId)}`,
|
|
492
|
+
{ headers: this.authHeaders(), timeoutMs: this.timeoutMs }
|
|
493
|
+
);
|
|
494
|
+
this.agentsBySession.delete(sessionKey);
|
|
495
|
+
} catch (err) {
|
|
496
|
+
if (isNotFoundDelete(err)) {
|
|
497
|
+
this.agentsBySession.delete(sessionKey);
|
|
498
|
+
} else {
|
|
499
|
+
failed.push(agentId);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
if (failed.length > 0) {
|
|
504
|
+
throw new Error(
|
|
505
|
+
`Letta reset could not clean ${failed.length} agent(s): ${failed.join(", ")}. The failed agents are retained in agentsBySession for the next reset() retry.`
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
/** Create a Letta agent for the session if one does not yet exist. */
|
|
510
|
+
async ensureAgent(sessionKey) {
|
|
511
|
+
const existing = this.agentsBySession.get(sessionKey);
|
|
512
|
+
if (existing) return existing;
|
|
513
|
+
const agent = await httpJson(
|
|
514
|
+
this.fetchImpl,
|
|
515
|
+
"POST",
|
|
516
|
+
`${this.baseUrl}/v1/agents/`,
|
|
517
|
+
{
|
|
518
|
+
headers: this.authHeaders(),
|
|
519
|
+
body: {
|
|
520
|
+
name: `${this.agentNamePrefix}-${sessionKey}`,
|
|
521
|
+
agent_type: "memgpt_agent",
|
|
522
|
+
model: this.model,
|
|
523
|
+
memory_blocks: [
|
|
524
|
+
{ label: "human", value: "", limit: 5e3 },
|
|
525
|
+
{ label: "persona", value: this.personaBlock, limit: 5e3 }
|
|
526
|
+
]
|
|
527
|
+
},
|
|
528
|
+
timeoutMs: this.timeoutMs
|
|
529
|
+
}
|
|
530
|
+
);
|
|
531
|
+
const agentId = agent?.id;
|
|
532
|
+
if (!agentId) {
|
|
533
|
+
throw new Error(
|
|
534
|
+
`Letta agent creation did not return an agent id for session ${sessionKey}`
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
this.agentsBySession.set(sessionKey, agentId);
|
|
538
|
+
return agentId;
|
|
539
|
+
}
|
|
540
|
+
async ingestTurn(sessionKey, role, text, _at) {
|
|
541
|
+
this.ensureReady();
|
|
542
|
+
const agentId = await this.ensureAgent(sessionKey);
|
|
543
|
+
await httpJson(
|
|
544
|
+
this.fetchImpl,
|
|
545
|
+
"POST",
|
|
546
|
+
`${this.baseUrl}/v1/agents/${encodeURIComponent(agentId)}/messages`,
|
|
547
|
+
{
|
|
548
|
+
headers: this.authHeaders(),
|
|
549
|
+
body: {
|
|
550
|
+
messages: [{ role, content: text }],
|
|
551
|
+
// Letta's /messages endpoint defaults to streaming (SSE). Request a
|
|
552
|
+
// single JSON response so httpJson parses it and ingestTurn resolves
|
|
553
|
+
// only after the agent (and its memory tools) finish processing.
|
|
554
|
+
stream: false
|
|
555
|
+
},
|
|
556
|
+
timeoutMs: this.timeoutMs
|
|
557
|
+
}
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
async recall(_query, sessionKey) {
|
|
561
|
+
this.ensureReady();
|
|
562
|
+
const agentId = await this.ensureAgent(sessionKey);
|
|
563
|
+
const response = await httpJson(
|
|
564
|
+
this.fetchImpl,
|
|
565
|
+
"GET",
|
|
566
|
+
`${this.baseUrl}/v1/agents/${encodeURIComponent(agentId)}/core-memory/blocks`,
|
|
567
|
+
{ headers: this.authHeaders(), timeoutMs: this.timeoutMs }
|
|
568
|
+
);
|
|
569
|
+
if (!response) return [];
|
|
570
|
+
const blocks = Array.isArray(response) ? response : response.memory ?? response.blocks ?? [];
|
|
571
|
+
const strings = [];
|
|
572
|
+
for (const block of blocks) {
|
|
573
|
+
if (block.label === "persona") continue;
|
|
574
|
+
const value = block.value ?? block.text ?? block.content;
|
|
575
|
+
if (value && value.trim().length > 0) {
|
|
576
|
+
for (const line of value.split(/\n+/)) {
|
|
577
|
+
const trimmed = line.trim();
|
|
578
|
+
if (trimmed) strings.push(trimmed);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
return strings;
|
|
583
|
+
}
|
|
584
|
+
async correct(text, sessionKey, _at) {
|
|
585
|
+
await this.ingestTurn(
|
|
586
|
+
sessionKey,
|
|
587
|
+
"user",
|
|
588
|
+
text,
|
|
589
|
+
_at ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
590
|
+
);
|
|
591
|
+
}
|
|
592
|
+
async runMaintenance() {
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
|
|
1
596
|
// src/ingestion-types.ts
|
|
2
597
|
var REQUIRED_FRONTMATTER_FIELDS = ["title", "type", "state", "created", "see-also"];
|
|
3
598
|
|
|
@@ -2945,7 +3540,7 @@ async function withTimeout(label, timeoutMs, fn, onTimeout) {
|
|
|
2945
3540
|
() => void 0,
|
|
2946
3541
|
() => void 0
|
|
2947
3542
|
),
|
|
2948
|
-
|
|
3543
|
+
delay2(BENCHMARK_TIMEOUT_ABORT_GRACE_MS)
|
|
2949
3544
|
]);
|
|
2950
3545
|
}
|
|
2951
3546
|
throw error;
|
|
@@ -2955,7 +3550,7 @@ async function withTimeout(label, timeoutMs, fn, onTimeout) {
|
|
|
2955
3550
|
}
|
|
2956
3551
|
}
|
|
2957
3552
|
}
|
|
2958
|
-
function
|
|
3553
|
+
function delay2(ms) {
|
|
2959
3554
|
return new Promise((resolve) => {
|
|
2960
3555
|
setTimeout(resolve, ms);
|
|
2961
3556
|
});
|
|
@@ -6285,34 +6880,20 @@ function createAnthropicProvider(config) {
|
|
|
6285
6880
|
return new AnthropicProvider(config);
|
|
6286
6881
|
}
|
|
6287
6882
|
|
|
6288
|
-
// src/providers/
|
|
6883
|
+
// src/providers/claude-cli.ts
|
|
6289
6884
|
import { spawn } from "child_process";
|
|
6290
|
-
import {
|
|
6291
|
-
import { mkdir as mkdir5, mkdtemp as mkdtemp2, readFile as readFile6, rm as rm2, writeFile as writeFile5 } from "fs/promises";
|
|
6885
|
+
import { mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
|
|
6292
6886
|
import os3 from "os";
|
|
6293
6887
|
import path6 from "path";
|
|
6294
|
-
var
|
|
6295
|
-
var
|
|
6296
|
-
var
|
|
6297
|
-
var
|
|
6298
|
-
|
|
6299
|
-
|
|
6300
|
-
"SIGTERM"
|
|
6301
|
-
];
|
|
6302
|
-
var CODEX_CLI_FORCED_PARENT_EXIT_MS = 1e3;
|
|
6303
|
-
var CODEX_CLI_DIAGNOSTICS_DIR_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_DIR";
|
|
6304
|
-
var CODEX_CLI_DIAGNOSTICS_MODE_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_MODE";
|
|
6305
|
-
var CODEX_CLI_EXECUTABLE_ENV = "REMNIC_BENCH_CODEX_CLI_EXECUTABLE";
|
|
6306
|
-
var CODEX_CLI_TRANSPORT_ENV = "REMNIC_BENCH_CODEX_CLI_TRANSPORT";
|
|
6307
|
-
var CODEX_CLI_VERSION_TIMEOUT_MS = 5e3;
|
|
6308
|
-
var CODEX_CLI_HEALTH_CACHE_TTL_MS = 3e4;
|
|
6309
|
-
var OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
|
|
6310
|
-
var OPENAI_BASE_URL_ENV = "OPENAI_BASE_URL";
|
|
6311
|
-
var OPENAI_RESPONSES_BASE_URL = "https://api.openai.com/v1";
|
|
6312
|
-
var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
6888
|
+
var CLAUDE_CLI_STDIO_LIMIT = 64e3;
|
|
6889
|
+
var CLAUDE_CLI_PARENT_SIGNALS = ["SIGHUP", "SIGINT", "SIGTERM"];
|
|
6890
|
+
var CLAUDE_CLI_FORCED_PARENT_EXIT_MS = 1e3;
|
|
6891
|
+
var CLAUDE_CLI_EXECUTABLE_ENV = "REMNIC_BENCH_CLAUDE_CLI_EXECUTABLE";
|
|
6892
|
+
var CLAUDE_CLI_VERSION_TIMEOUT_MS = 5e3;
|
|
6893
|
+
var CLAUDE_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
6313
6894
|
"ALL_PROXY",
|
|
6314
6895
|
"APPDATA",
|
|
6315
|
-
"
|
|
6896
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
6316
6897
|
"COLORTERM",
|
|
6317
6898
|
"COMSPEC",
|
|
6318
6899
|
"FORCE_COLOR",
|
|
@@ -6326,9 +6907,6 @@ var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
|
6326
6907
|
"NODE_EXTRA_CA_CERTS",
|
|
6327
6908
|
"NO_PROXY",
|
|
6328
6909
|
"NUMBER_OF_PROCESSORS",
|
|
6329
|
-
OPENAI_BASE_URL_ENV,
|
|
6330
|
-
"OPENAI_ORGANIZATION",
|
|
6331
|
-
"OPENAI_PROJECT",
|
|
6332
6910
|
"OS",
|
|
6333
6911
|
"PATH",
|
|
6334
6912
|
"PATHEXT",
|
|
@@ -6354,17 +6932,84 @@ var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
|
6354
6932
|
"XDG_DATA_HOME",
|
|
6355
6933
|
"XDG_RUNTIME_DIR"
|
|
6356
6934
|
]);
|
|
6357
|
-
var
|
|
6358
|
-
var
|
|
6359
|
-
var
|
|
6360
|
-
var
|
|
6361
|
-
|
|
6935
|
+
var DEFAULT_USAGE_LIMIT_MAX_WAIT_MS = 30 * 60 * 1e3;
|
|
6936
|
+
var USAGE_LIMIT_BASE_BACKOFF_MS = 6e4;
|
|
6937
|
+
var USAGE_LIMIT_MAX_STEP_MS = 10 * 60 * 1e3;
|
|
6938
|
+
var activeClaudeCliChildPids = /* @__PURE__ */ new Set();
|
|
6939
|
+
var claudeCliParentCleanupInstalled = false;
|
|
6940
|
+
var ClaudeCliConcurrencyGate = class {
|
|
6941
|
+
active = 0;
|
|
6942
|
+
limit;
|
|
6943
|
+
waiters = [];
|
|
6944
|
+
constructor(limit) {
|
|
6945
|
+
this.limit = normalizeGateLimit(limit);
|
|
6946
|
+
}
|
|
6947
|
+
async run(fn) {
|
|
6948
|
+
await this.acquire();
|
|
6949
|
+
try {
|
|
6950
|
+
return await fn();
|
|
6951
|
+
} finally {
|
|
6952
|
+
this.release();
|
|
6953
|
+
}
|
|
6954
|
+
}
|
|
6955
|
+
/** Raises the gate's limit if `limit` is higher than the current one;
|
|
6956
|
+
* never lowers it. Immediately wakes any queued waiters that fit under
|
|
6957
|
+
* the new, higher limit — see `getSharedClaudeCliGate` for the policy
|
|
6958
|
+
* this implements (shared max across all constructed instances). */
|
|
6959
|
+
raiseLimit(limit) {
|
|
6960
|
+
const normalized = normalizeGateLimit(limit);
|
|
6961
|
+
if (normalized <= this.limit) {
|
|
6962
|
+
return;
|
|
6963
|
+
}
|
|
6964
|
+
this.limit = normalized;
|
|
6965
|
+
while (this.active < this.limit) {
|
|
6966
|
+
const next = this.waiters.shift();
|
|
6967
|
+
if (!next) {
|
|
6968
|
+
break;
|
|
6969
|
+
}
|
|
6970
|
+
next();
|
|
6971
|
+
}
|
|
6972
|
+
}
|
|
6973
|
+
acquire() {
|
|
6974
|
+
if (this.active < this.limit) {
|
|
6975
|
+
this.active += 1;
|
|
6976
|
+
return Promise.resolve();
|
|
6977
|
+
}
|
|
6978
|
+
return new Promise((resolve) => {
|
|
6979
|
+
this.waiters.push(() => {
|
|
6980
|
+
this.active += 1;
|
|
6981
|
+
resolve();
|
|
6982
|
+
});
|
|
6983
|
+
});
|
|
6984
|
+
}
|
|
6985
|
+
release() {
|
|
6986
|
+
this.active = Math.max(0, this.active - 1);
|
|
6987
|
+
const next = this.waiters.shift();
|
|
6988
|
+
if (next) {
|
|
6989
|
+
next();
|
|
6990
|
+
}
|
|
6991
|
+
}
|
|
6992
|
+
};
|
|
6993
|
+
function normalizeGateLimit(limit) {
|
|
6994
|
+
return Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1;
|
|
6995
|
+
}
|
|
6996
|
+
var sharedClaudeCliGate;
|
|
6997
|
+
function getSharedClaudeCliGate(requestedLimit) {
|
|
6998
|
+
if (!sharedClaudeCliGate) {
|
|
6999
|
+
sharedClaudeCliGate = new ClaudeCliConcurrencyGate(requestedLimit);
|
|
7000
|
+
return sharedClaudeCliGate;
|
|
7001
|
+
}
|
|
7002
|
+
sharedClaudeCliGate.raiseLimit(requestedLimit);
|
|
7003
|
+
return sharedClaudeCliGate;
|
|
7004
|
+
}
|
|
7005
|
+
var ClaudeCliProvider = class {
|
|
7006
|
+
provider = "claude-cli";
|
|
6362
7007
|
id;
|
|
6363
7008
|
name;
|
|
6364
7009
|
config;
|
|
6365
|
-
|
|
6366
|
-
|
|
6367
|
-
|
|
7010
|
+
runClaudeCli;
|
|
7011
|
+
runClaudeVersion;
|
|
7012
|
+
gate;
|
|
6368
7013
|
usage = {
|
|
6369
7014
|
inputTokens: 0,
|
|
6370
7015
|
outputTokens: 0,
|
|
@@ -6372,112 +7017,801 @@ var CodexCliProvider = class {
|
|
|
6372
7017
|
};
|
|
6373
7018
|
constructor(config, deps = {}) {
|
|
6374
7019
|
this.config = config;
|
|
6375
|
-
this.
|
|
6376
|
-
this.
|
|
6377
|
-
this.
|
|
6378
|
-
this.id = `
|
|
7020
|
+
this.runClaudeCli = deps.runClaudeCli ?? runClaudeCliCommand;
|
|
7021
|
+
this.runClaudeVersion = deps.runClaudeVersion ?? runClaudeVersionCommand;
|
|
7022
|
+
this.gate = getSharedClaudeCliGate(config.concurrency ?? 1);
|
|
7023
|
+
this.id = `claude-cli:${config.model}`;
|
|
6379
7024
|
this.name = config.model;
|
|
6380
7025
|
}
|
|
6381
7026
|
async complete(prompt, opts = {}) {
|
|
6382
|
-
|
|
6383
|
-
|
|
6384
|
-
|
|
7027
|
+
return this.gate.run(() => this.completeSerialized(prompt, opts));
|
|
7028
|
+
}
|
|
7029
|
+
async completeSerialized(prompt, opts) {
|
|
7030
|
+
if (opts.signal?.aborted) {
|
|
7031
|
+
throw claudeCliAbortError(opts.signal);
|
|
6385
7032
|
}
|
|
6386
|
-
const
|
|
6387
|
-
|
|
6388
|
-
);
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
const
|
|
6394
|
-
let diagnostics;
|
|
6395
|
-
let diagnosticsFinished = false;
|
|
6396
|
-
const finishDiagnostics = async (outcome) => {
|
|
6397
|
-
if (diagnosticsFinished) {
|
|
6398
|
-
return;
|
|
6399
|
-
}
|
|
6400
|
-
diagnosticsFinished = true;
|
|
6401
|
-
await finishCodexCliDiagnostics(diagnostics, startedAt, outcome);
|
|
6402
|
-
};
|
|
7033
|
+
const startedAt = performance.now();
|
|
7034
|
+
const maxAttempts = normalizeClaudeCliMaxAttempts(this.config.retryOptions?.maxAttempts);
|
|
7035
|
+
const usageLimitBudgetMs = normalizeUsageLimitMaxWaitMs(this.config.retryOptions?.max429WaitMs);
|
|
7036
|
+
const loopStartedAt = performance.now();
|
|
7037
|
+
let transientAttempt = 1;
|
|
7038
|
+
let usageLimitAttempt = 1;
|
|
7039
|
+
while (true) {
|
|
7040
|
+
const tempDir = await mkdtemp2(path6.join(os3.tmpdir(), "remnic-claude-cli-"));
|
|
6403
7041
|
try {
|
|
6404
|
-
|
|
6405
|
-
const
|
|
6406
|
-
diagnostics = await startCodexCliDiagnostics({
|
|
6407
|
-
config: this.config,
|
|
6408
|
-
request,
|
|
6409
|
-
reasoningEffort: this.config.reasoningEffort ?? DEFAULT_REASONING_EFFORT,
|
|
6410
|
-
serviceTier: DEFAULT_SERVICE_TIER,
|
|
6411
|
-
retry: { attempt, maxAttempts }
|
|
6412
|
-
});
|
|
6413
|
-
const result = await this.runCodexCli(request);
|
|
7042
|
+
const request = this.buildRunRequest(prompt, opts, tempDir);
|
|
7043
|
+
const result = await this.runClaudeCli(request);
|
|
6414
7044
|
if (result.status !== 0) {
|
|
7045
|
+
if (isClaudeUsageLimitSignal(`${result.stderr}
|
|
7046
|
+
${result.stdout}`)) {
|
|
7047
|
+
await sleepBeforeUsageLimitRetry({
|
|
7048
|
+
attempt: usageLimitAttempt,
|
|
7049
|
+
loopStartedAt,
|
|
7050
|
+
budgetMs: usageLimitBudgetMs,
|
|
7051
|
+
failureSummary: summarizeProcessOutput(result.stderr, result.stdout),
|
|
7052
|
+
signal: opts.signal
|
|
7053
|
+
});
|
|
7054
|
+
usageLimitAttempt += 1;
|
|
7055
|
+
continue;
|
|
7056
|
+
}
|
|
6415
7057
|
const exitLabel = result.signal ? `signal ${result.signal}` : `exit ${result.status ?? "unknown"}`;
|
|
6416
7058
|
const error = new Error(
|
|
6417
|
-
`
|
|
7059
|
+
`Claude CLI completion failed (${exitLabel}): ${summarizeProcessOutput(result.stderr, result.stdout)}`
|
|
6418
7060
|
);
|
|
6419
|
-
if (
|
|
6420
|
-
|
|
6421
|
-
|
|
6422
|
-
|
|
6423
|
-
|
|
6424
|
-
|
|
7061
|
+
if (transientAttempt < maxAttempts && isRetryableClaudeCliResult(result)) {
|
|
7062
|
+
await sleepBeforeClaudeCliRetry({
|
|
7063
|
+
attempt: transientAttempt,
|
|
7064
|
+
baseBackoffMs: this.config.retryOptions?.baseBackoffMs,
|
|
7065
|
+
maxStepMs: 3e4,
|
|
7066
|
+
capMs: Number.POSITIVE_INFINITY,
|
|
7067
|
+
signal: opts.signal
|
|
6425
7068
|
});
|
|
6426
|
-
|
|
6427
|
-
attempt,
|
|
6428
|
-
this.config.retryOptions?.baseBackoffMs,
|
|
6429
|
-
opts.signal
|
|
6430
|
-
);
|
|
7069
|
+
transientAttempt += 1;
|
|
6431
7070
|
continue;
|
|
6432
7071
|
}
|
|
6433
|
-
await finishDiagnostics({ result, error });
|
|
6434
7072
|
throw error;
|
|
6435
7073
|
}
|
|
6436
|
-
const
|
|
7074
|
+
const payload = parseClaudeCliJsonResult(result.stdout);
|
|
7075
|
+
if (isClaudeCliErrorFlagSet(payload.is_error)) {
|
|
7076
|
+
if (isClaudeUsageLimitSignal(
|
|
7077
|
+
`${result.stderr}
|
|
7078
|
+
${payload.error ?? ""}
|
|
7079
|
+
${payload.result ?? ""}`
|
|
7080
|
+
)) {
|
|
7081
|
+
await sleepBeforeUsageLimitRetry({
|
|
7082
|
+
attempt: usageLimitAttempt,
|
|
7083
|
+
loopStartedAt,
|
|
7084
|
+
budgetMs: usageLimitBudgetMs,
|
|
7085
|
+
failureSummary: summarizeProcessOutput(result.stderr, result.stdout),
|
|
7086
|
+
signal: opts.signal
|
|
7087
|
+
});
|
|
7088
|
+
usageLimitAttempt += 1;
|
|
7089
|
+
continue;
|
|
7090
|
+
}
|
|
7091
|
+
throw new Error(
|
|
7092
|
+
`Claude CLI reported is_error: ${payload.error?.trim() || payload.result?.trim() || summarizeProcessOutput(result.stderr, result.stdout)}`
|
|
7093
|
+
);
|
|
7094
|
+
}
|
|
7095
|
+
const text = typeof payload.result === "string" ? payload.result.trim() : "";
|
|
6437
7096
|
if (text.length === 0) {
|
|
6438
|
-
|
|
6439
|
-
|
|
7097
|
+
if (isClaudeUsageLimitSignal(`${result.stderr}
|
|
7098
|
+
${payload.error ?? ""}`)) {
|
|
7099
|
+
await sleepBeforeUsageLimitRetry({
|
|
7100
|
+
attempt: usageLimitAttempt,
|
|
7101
|
+
loopStartedAt,
|
|
7102
|
+
budgetMs: usageLimitBudgetMs,
|
|
7103
|
+
failureSummary: summarizeProcessOutput(result.stderr, result.stdout),
|
|
7104
|
+
signal: opts.signal
|
|
7105
|
+
});
|
|
7106
|
+
usageLimitAttempt += 1;
|
|
7107
|
+
continue;
|
|
7108
|
+
}
|
|
7109
|
+
throw new Error(
|
|
7110
|
+
`Claude CLI completion returned no result text: ${summarizeProcessOutput(result.stderr, result.stdout)}`
|
|
6440
7111
|
);
|
|
6441
|
-
await finishDiagnostics({ result, error });
|
|
6442
|
-
throw error;
|
|
6443
7112
|
}
|
|
6444
|
-
|
|
6445
|
-
const
|
|
6446
|
-
|
|
6447
|
-
${result.stdout}`,
|
|
6448
|
-
text
|
|
6449
|
-
);
|
|
6450
|
-
this.recordUsage(tokens.input, tokens.output);
|
|
7113
|
+
const inputTokens = nonNegativeInt(payload.usage?.input_tokens);
|
|
7114
|
+
const outputTokens = nonNegativeInt(payload.usage?.output_tokens);
|
|
7115
|
+
this.recordUsage(inputTokens, outputTokens);
|
|
6451
7116
|
return {
|
|
6452
7117
|
text,
|
|
6453
|
-
tokens,
|
|
7118
|
+
tokens: { input: inputTokens, output: outputTokens },
|
|
6454
7119
|
latencyMs: Math.round(performance.now() - startedAt),
|
|
6455
7120
|
model: this.config.model
|
|
6456
7121
|
};
|
|
6457
|
-
} catch (error) {
|
|
6458
|
-
lastError = error;
|
|
6459
|
-
await finishDiagnostics({ error });
|
|
6460
|
-
throw error;
|
|
6461
7122
|
} finally {
|
|
6462
7123
|
await rm2(tempDir, { force: true, recursive: true });
|
|
6463
7124
|
}
|
|
6464
7125
|
}
|
|
6465
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6466
7126
|
}
|
|
6467
7127
|
async discover() {
|
|
6468
|
-
const version = await this.
|
|
6469
|
-
|
|
6470
|
-
|
|
7128
|
+
const version = await this.runClaudeVersion(
|
|
7129
|
+
resolveClaudeCliExecutable(this.config),
|
|
7130
|
+
buildIsolatedClaudeEnv(this.config)
|
|
6471
7131
|
);
|
|
6472
7132
|
if (version.status !== 0) {
|
|
6473
7133
|
throw new Error(
|
|
6474
|
-
`
|
|
7134
|
+
`Claude CLI discovery failed: ${version.stderr.trim() || `exit ${version.status ?? "unknown"}`}`
|
|
6475
7135
|
);
|
|
6476
7136
|
}
|
|
6477
7137
|
return [
|
|
6478
7138
|
{
|
|
6479
7139
|
id: this.config.model,
|
|
6480
|
-
name: `${this.config.model} (
|
|
7140
|
+
name: `${this.config.model} (Claude CLI)`,
|
|
7141
|
+
contextLength: 0,
|
|
7142
|
+
capabilities: ["completion"]
|
|
7143
|
+
}
|
|
7144
|
+
];
|
|
7145
|
+
}
|
|
7146
|
+
getUsage() {
|
|
7147
|
+
return { ...this.usage };
|
|
7148
|
+
}
|
|
7149
|
+
resetUsage() {
|
|
7150
|
+
this.usage = {
|
|
7151
|
+
inputTokens: 0,
|
|
7152
|
+
outputTokens: 0,
|
|
7153
|
+
totalTokens: 0
|
|
7154
|
+
};
|
|
7155
|
+
}
|
|
7156
|
+
recordUsage(inputTokens, outputTokens) {
|
|
7157
|
+
this.usage = {
|
|
7158
|
+
inputTokens: this.usage.inputTokens + inputTokens,
|
|
7159
|
+
outputTokens: this.usage.outputTokens + outputTokens,
|
|
7160
|
+
totalTokens: this.usage.totalTokens + inputTokens + outputTokens
|
|
7161
|
+
};
|
|
7162
|
+
}
|
|
7163
|
+
buildRunRequest(prompt, opts, cwd) {
|
|
7164
|
+
return {
|
|
7165
|
+
executable: resolveClaudeCliExecutable(this.config),
|
|
7166
|
+
args: buildClaudeCliArgs(this.config, opts.systemPrompt),
|
|
7167
|
+
input: buildClaudeCompletionPrompt(prompt),
|
|
7168
|
+
cwd,
|
|
7169
|
+
timeoutMs: this.config.retryOptions?.timeoutMs,
|
|
7170
|
+
signal: opts.signal,
|
|
7171
|
+
env: buildIsolatedClaudeEnv(this.config, opts.maxTokens)
|
|
7172
|
+
};
|
|
7173
|
+
}
|
|
7174
|
+
};
|
|
7175
|
+
function buildClaudeCliArgs(config, systemPrompt) {
|
|
7176
|
+
const args = [
|
|
7177
|
+
"--print",
|
|
7178
|
+
"--model",
|
|
7179
|
+
config.model,
|
|
7180
|
+
"--output-format",
|
|
7181
|
+
"json",
|
|
7182
|
+
"--input-format",
|
|
7183
|
+
"text",
|
|
7184
|
+
"--safe-mode",
|
|
7185
|
+
"--strict-mcp-config",
|
|
7186
|
+
"--tools",
|
|
7187
|
+
"",
|
|
7188
|
+
"--no-session-persistence"
|
|
7189
|
+
];
|
|
7190
|
+
const trimmedSystemPrompt = systemPrompt?.trim();
|
|
7191
|
+
if (trimmedSystemPrompt) {
|
|
7192
|
+
args.push("--system-prompt", trimmedSystemPrompt);
|
|
7193
|
+
}
|
|
7194
|
+
return args;
|
|
7195
|
+
}
|
|
7196
|
+
function buildClaudeCompletionPrompt(userPrompt) {
|
|
7197
|
+
const payload = { userPrompt };
|
|
7198
|
+
return [
|
|
7199
|
+
"You are a benchmark evaluation endpoint, not a coding agent.",
|
|
7200
|
+
"Use only the explicit JSON payload below.",
|
|
7201
|
+
"Any additional scoring/format protocol instructions are provided via the",
|
|
7202
|
+
"system prompt, which takes priority over this user payload.",
|
|
7203
|
+
"Do not use tools, do not read or write files, do not browse, and do not use persisted memory.",
|
|
7204
|
+
"Return only the final answer text. If the request asks for JSON, return raw JSON only.",
|
|
7205
|
+
"",
|
|
7206
|
+
"BENCHMARK_REQUEST_JSON:",
|
|
7207
|
+
JSON.stringify(payload, null, 2)
|
|
7208
|
+
].join("\n");
|
|
7209
|
+
}
|
|
7210
|
+
var CLAUDE_CLI_MAX_OUTPUT_TOKENS_ENV = "CLAUDE_CODE_MAX_OUTPUT_TOKENS";
|
|
7211
|
+
function buildIsolatedClaudeEnv(config, maxTokens) {
|
|
7212
|
+
const env = {};
|
|
7213
|
+
for (const [key, value] of Object.entries(process.env)) {
|
|
7214
|
+
if (value !== void 0 && isAllowedClaudeRuntimeEnvKey(key)) {
|
|
7215
|
+
env[key] = value;
|
|
7216
|
+
}
|
|
7217
|
+
}
|
|
7218
|
+
if (config.apiKey && config.apiKey.trim().length > 0) {
|
|
7219
|
+
env.ANTHROPIC_API_KEY = config.apiKey.trim();
|
|
7220
|
+
}
|
|
7221
|
+
if (config.baseUrl && config.baseUrl.trim().length > 0) {
|
|
7222
|
+
env.ANTHROPIC_BASE_URL = config.baseUrl.trim();
|
|
7223
|
+
}
|
|
7224
|
+
if (typeof maxTokens === "number" && Number.isFinite(maxTokens) && maxTokens > 0) {
|
|
7225
|
+
env[CLAUDE_CLI_MAX_OUTPUT_TOKENS_ENV] = String(Math.floor(maxTokens));
|
|
7226
|
+
}
|
|
7227
|
+
return env;
|
|
7228
|
+
}
|
|
7229
|
+
function isAllowedClaudeRuntimeEnvKey(key) {
|
|
7230
|
+
const normalized = key.toUpperCase();
|
|
7231
|
+
return CLAUDE_CLI_RUNTIME_ENV_ALLOWLIST.has(normalized) || normalized.startsWith("LC_");
|
|
7232
|
+
}
|
|
7233
|
+
function resolveClaudeCliExecutable(config) {
|
|
7234
|
+
const configured = config.executable ?? process.env[CLAUDE_CLI_EXECUTABLE_ENV];
|
|
7235
|
+
if (configured === void 0) {
|
|
7236
|
+
return "claude";
|
|
7237
|
+
}
|
|
7238
|
+
const trimmed = configured.trim();
|
|
7239
|
+
if (trimmed.length === 0) {
|
|
7240
|
+
throw new Error(`${CLAUDE_CLI_EXECUTABLE_ENV} / claude-cli executable must not be empty`);
|
|
7241
|
+
}
|
|
7242
|
+
return expandHomeRelativePath(trimmed);
|
|
7243
|
+
}
|
|
7244
|
+
function expandHomeRelativePath(value) {
|
|
7245
|
+
if (value === "~") {
|
|
7246
|
+
return os3.homedir();
|
|
7247
|
+
}
|
|
7248
|
+
if (value.startsWith("~/") || value.startsWith("~\\")) {
|
|
7249
|
+
return path6.join(os3.homedir(), value.slice(2));
|
|
7250
|
+
}
|
|
7251
|
+
return value;
|
|
7252
|
+
}
|
|
7253
|
+
function parseClaudeCliJsonResult(stdout) {
|
|
7254
|
+
const trimmed = stdout.trim();
|
|
7255
|
+
if (trimmed.length === 0) {
|
|
7256
|
+
return { is_error: true, error: "Claude CLI produced no stdout." };
|
|
7257
|
+
}
|
|
7258
|
+
try {
|
|
7259
|
+
const parsed = JSON.parse(trimmed);
|
|
7260
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
7261
|
+
return { is_error: true, error: trimmed.slice(-1e3) };
|
|
7262
|
+
}
|
|
7263
|
+
return parsed;
|
|
7264
|
+
} catch {
|
|
7265
|
+
return { is_error: true, error: trimmed.slice(-1e3) };
|
|
7266
|
+
}
|
|
7267
|
+
}
|
|
7268
|
+
function nonNegativeInt(value) {
|
|
7269
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0;
|
|
7270
|
+
}
|
|
7271
|
+
function isClaudeCliErrorFlagSet(value) {
|
|
7272
|
+
return value === true;
|
|
7273
|
+
}
|
|
7274
|
+
var USAGE_LIMIT_SIGNAL_REGEX = /\b(?:usage limit|session limit|weekly limit|opus limit|sonnet limit|rate[- ]limit(?:ed)?|too many requests|quota exceeded|429)\b/i;
|
|
7275
|
+
function isClaudeUsageLimitSignal(combinedOutput) {
|
|
7276
|
+
return USAGE_LIMIT_SIGNAL_REGEX.test(combinedOutput);
|
|
7277
|
+
}
|
|
7278
|
+
function isRetryableClaudeCliResult(result) {
|
|
7279
|
+
if (!result.signal) {
|
|
7280
|
+
return false;
|
|
7281
|
+
}
|
|
7282
|
+
const stderr = result.stderr.toLowerCase();
|
|
7283
|
+
if (stderr.includes("timed out after") || stderr.includes("aborted by benchmark timeout")) {
|
|
7284
|
+
return false;
|
|
7285
|
+
}
|
|
7286
|
+
return true;
|
|
7287
|
+
}
|
|
7288
|
+
function normalizeClaudeCliMaxAttempts(value) {
|
|
7289
|
+
if (value === void 0) {
|
|
7290
|
+
return 3;
|
|
7291
|
+
}
|
|
7292
|
+
if (!Number.isFinite(value) || value < 1) {
|
|
7293
|
+
return 1;
|
|
7294
|
+
}
|
|
7295
|
+
return Math.min(10, Math.floor(value));
|
|
7296
|
+
}
|
|
7297
|
+
function normalizeUsageLimitMaxWaitMs(value) {
|
|
7298
|
+
if (value === void 0) {
|
|
7299
|
+
return DEFAULT_USAGE_LIMIT_MAX_WAIT_MS;
|
|
7300
|
+
}
|
|
7301
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
7302
|
+
return DEFAULT_USAGE_LIMIT_MAX_WAIT_MS;
|
|
7303
|
+
}
|
|
7304
|
+
return value;
|
|
7305
|
+
}
|
|
7306
|
+
async function sleepBeforeUsageLimitRetry(options) {
|
|
7307
|
+
const delayMs = Math.min(
|
|
7308
|
+
USAGE_LIMIT_BASE_BACKOFF_MS * Math.pow(2, options.attempt - 1),
|
|
7309
|
+
USAGE_LIMIT_MAX_STEP_MS
|
|
7310
|
+
);
|
|
7311
|
+
const remainingBudgetMs = options.budgetMs - (performance.now() - options.loopStartedAt);
|
|
7312
|
+
if (delayMs > remainingBudgetMs) {
|
|
7313
|
+
throw new Error(
|
|
7314
|
+
`Claude CLI usage-limit backoff budget (${options.budgetMs}ms) exhausted: ${options.failureSummary}`
|
|
7315
|
+
);
|
|
7316
|
+
}
|
|
7317
|
+
await sleepBeforeClaudeCliRetry({
|
|
7318
|
+
attempt: options.attempt,
|
|
7319
|
+
baseBackoffMs: USAGE_LIMIT_BASE_BACKOFF_MS,
|
|
7320
|
+
maxStepMs: USAGE_LIMIT_MAX_STEP_MS,
|
|
7321
|
+
capMs: remainingBudgetMs,
|
|
7322
|
+
signal: options.signal
|
|
7323
|
+
});
|
|
7324
|
+
}
|
|
7325
|
+
async function sleepBeforeClaudeCliRetry(options) {
|
|
7326
|
+
const baseBackoffMs = options.baseBackoffMs !== void 0 && Number.isFinite(options.baseBackoffMs) && options.baseBackoffMs > 0 ? options.baseBackoffMs : 1e3;
|
|
7327
|
+
const uncappedDelayMs = baseBackoffMs * Math.pow(2, options.attempt - 1);
|
|
7328
|
+
const delayMs = Math.max(0, Math.min(uncappedDelayMs, options.maxStepMs, options.capMs));
|
|
7329
|
+
const signal = options.signal;
|
|
7330
|
+
if (!signal) {
|
|
7331
|
+
await new Promise((resolve) => {
|
|
7332
|
+
setTimeout(resolve, delayMs);
|
|
7333
|
+
});
|
|
7334
|
+
return;
|
|
7335
|
+
}
|
|
7336
|
+
if (signal.aborted) {
|
|
7337
|
+
throw claudeCliAbortError(signal);
|
|
7338
|
+
}
|
|
7339
|
+
await new Promise((resolve, reject) => {
|
|
7340
|
+
const cleanup = () => {
|
|
7341
|
+
signal.removeEventListener("abort", onAbort);
|
|
7342
|
+
};
|
|
7343
|
+
const onAbort = () => {
|
|
7344
|
+
clearTimeout(timeout);
|
|
7345
|
+
cleanup();
|
|
7346
|
+
reject(claudeCliAbortError(signal));
|
|
7347
|
+
};
|
|
7348
|
+
const timeout = setTimeout(() => {
|
|
7349
|
+
cleanup();
|
|
7350
|
+
resolve();
|
|
7351
|
+
}, delayMs);
|
|
7352
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
7353
|
+
});
|
|
7354
|
+
}
|
|
7355
|
+
function claudeCliAbortError(signal) {
|
|
7356
|
+
if (signal.reason instanceof Error) {
|
|
7357
|
+
return signal.reason;
|
|
7358
|
+
}
|
|
7359
|
+
if (signal.reason !== void 0) {
|
|
7360
|
+
return new Error(String(signal.reason));
|
|
7361
|
+
}
|
|
7362
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
7363
|
+
}
|
|
7364
|
+
function summarizeProcessOutput(stderr, stdout) {
|
|
7365
|
+
const summary = [stderr.trim(), stdout.trim()].filter((value) => value.length > 0).join("\n").trim();
|
|
7366
|
+
return summary.length > 0 ? summary.slice(-1e3) : "no process output";
|
|
7367
|
+
}
|
|
7368
|
+
function runClaudeVersionCommand(executable, env) {
|
|
7369
|
+
return new Promise((resolve, reject) => {
|
|
7370
|
+
const child = spawn(executable, ["--version"], {
|
|
7371
|
+
env,
|
|
7372
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
7373
|
+
detached: process.platform !== "win32",
|
|
7374
|
+
windowsHide: true
|
|
7375
|
+
});
|
|
7376
|
+
let stderr = "";
|
|
7377
|
+
let timedOut = false;
|
|
7378
|
+
let killTimeout;
|
|
7379
|
+
const terminateChild = (signal) => {
|
|
7380
|
+
if (child.pid && process.platform !== "win32") {
|
|
7381
|
+
try {
|
|
7382
|
+
process.kill(-child.pid, signal);
|
|
7383
|
+
return;
|
|
7384
|
+
} catch {
|
|
7385
|
+
}
|
|
7386
|
+
}
|
|
7387
|
+
child.kill(signal);
|
|
7388
|
+
};
|
|
7389
|
+
const timeout = setTimeout(() => {
|
|
7390
|
+
timedOut = true;
|
|
7391
|
+
terminateChild("SIGTERM");
|
|
7392
|
+
killTimeout = setTimeout(() => {
|
|
7393
|
+
terminateChild("SIGKILL");
|
|
7394
|
+
}, 1e3);
|
|
7395
|
+
killTimeout.unref();
|
|
7396
|
+
}, CLAUDE_CLI_VERSION_TIMEOUT_MS);
|
|
7397
|
+
timeout.unref();
|
|
7398
|
+
child.stderr?.setEncoding("utf8");
|
|
7399
|
+
child.stderr?.on("data", (chunk) => {
|
|
7400
|
+
stderr = appendBounded(stderr, chunk);
|
|
7401
|
+
});
|
|
7402
|
+
child.on("error", (error) => {
|
|
7403
|
+
clearTimeout(timeout);
|
|
7404
|
+
if (killTimeout) {
|
|
7405
|
+
clearTimeout(killTimeout);
|
|
7406
|
+
}
|
|
7407
|
+
reject(error);
|
|
7408
|
+
});
|
|
7409
|
+
child.on("close", (status) => {
|
|
7410
|
+
clearTimeout(timeout);
|
|
7411
|
+
if (killTimeout) {
|
|
7412
|
+
clearTimeout(killTimeout);
|
|
7413
|
+
}
|
|
7414
|
+
resolve({
|
|
7415
|
+
status: timedOut ? status ?? 124 : status,
|
|
7416
|
+
stderr: timedOut ? appendBounded(stderr, `
|
|
7417
|
+
Claude CLI --version timed out after ${CLAUDE_CLI_VERSION_TIMEOUT_MS}ms.`) : stderr
|
|
7418
|
+
});
|
|
7419
|
+
});
|
|
7420
|
+
});
|
|
7421
|
+
}
|
|
7422
|
+
function runClaudeCliCommand(request) {
|
|
7423
|
+
return new Promise((resolve, reject) => {
|
|
7424
|
+
if (request.signal?.aborted) {
|
|
7425
|
+
resolve({
|
|
7426
|
+
status: 124,
|
|
7427
|
+
signal: null,
|
|
7428
|
+
stdout: "",
|
|
7429
|
+
stderr: "Claude CLI aborted before start."
|
|
7430
|
+
});
|
|
7431
|
+
return;
|
|
7432
|
+
}
|
|
7433
|
+
const child = spawn(request.executable, request.args, {
|
|
7434
|
+
cwd: request.cwd,
|
|
7435
|
+
env: request.env,
|
|
7436
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
7437
|
+
detached: process.platform !== "win32",
|
|
7438
|
+
windowsHide: true
|
|
7439
|
+
});
|
|
7440
|
+
if (child.pid) {
|
|
7441
|
+
registerActiveClaudeCliChild(child.pid);
|
|
7442
|
+
}
|
|
7443
|
+
let stdout = "";
|
|
7444
|
+
let stderr = "";
|
|
7445
|
+
let timedOut = false;
|
|
7446
|
+
let aborted = false;
|
|
7447
|
+
let killTimeout;
|
|
7448
|
+
const clearKillTimeout = () => {
|
|
7449
|
+
if (killTimeout) {
|
|
7450
|
+
clearTimeout(killTimeout);
|
|
7451
|
+
killTimeout = void 0;
|
|
7452
|
+
}
|
|
7453
|
+
};
|
|
7454
|
+
const terminateChild = (signal) => {
|
|
7455
|
+
if (child.pid && process.platform !== "win32") {
|
|
7456
|
+
try {
|
|
7457
|
+
process.kill(-child.pid, signal);
|
|
7458
|
+
return;
|
|
7459
|
+
} catch {
|
|
7460
|
+
}
|
|
7461
|
+
}
|
|
7462
|
+
child.kill(signal);
|
|
7463
|
+
};
|
|
7464
|
+
const scheduleForcedKill = () => {
|
|
7465
|
+
clearKillTimeout();
|
|
7466
|
+
killTimeout = setTimeout(() => {
|
|
7467
|
+
terminateChild("SIGKILL");
|
|
7468
|
+
}, 1e3);
|
|
7469
|
+
killTimeout.unref();
|
|
7470
|
+
};
|
|
7471
|
+
const onAbort = () => {
|
|
7472
|
+
if (aborted) {
|
|
7473
|
+
return;
|
|
7474
|
+
}
|
|
7475
|
+
aborted = true;
|
|
7476
|
+
stderr = appendBounded(stderr, "\nClaude CLI aborted by benchmark timeout.");
|
|
7477
|
+
terminateChild("SIGTERM");
|
|
7478
|
+
scheduleForcedKill();
|
|
7479
|
+
};
|
|
7480
|
+
request.signal?.addEventListener("abort", onAbort, { once: true });
|
|
7481
|
+
if (request.signal?.aborted) {
|
|
7482
|
+
onAbort();
|
|
7483
|
+
}
|
|
7484
|
+
const timeout = request.timeoutMs ? setTimeout(() => {
|
|
7485
|
+
timedOut = true;
|
|
7486
|
+
terminateChild("SIGTERM");
|
|
7487
|
+
scheduleForcedKill();
|
|
7488
|
+
}, request.timeoutMs) : void 0;
|
|
7489
|
+
timeout?.unref();
|
|
7490
|
+
child.stdout?.setEncoding("utf8");
|
|
7491
|
+
child.stderr?.setEncoding("utf8");
|
|
7492
|
+
child.stdout?.on("data", (chunk) => {
|
|
7493
|
+
stdout = appendBounded(stdout, chunk);
|
|
7494
|
+
});
|
|
7495
|
+
child.stderr?.on("data", (chunk) => {
|
|
7496
|
+
stderr = appendBounded(stderr, chunk);
|
|
7497
|
+
});
|
|
7498
|
+
child.stdin?.on("error", (error) => {
|
|
7499
|
+
stderr = appendBounded(stderr, `
|
|
7500
|
+
Claude CLI stdin error: ${error.code ?? error.message}`);
|
|
7501
|
+
});
|
|
7502
|
+
child.on("error", (error) => {
|
|
7503
|
+
if (timeout) {
|
|
7504
|
+
clearTimeout(timeout);
|
|
7505
|
+
}
|
|
7506
|
+
clearKillTimeout();
|
|
7507
|
+
if (child.pid) {
|
|
7508
|
+
unregisterActiveClaudeCliChild(child.pid);
|
|
7509
|
+
}
|
|
7510
|
+
request.signal?.removeEventListener("abort", onAbort);
|
|
7511
|
+
reject(error);
|
|
7512
|
+
});
|
|
7513
|
+
child.on("close", (status, signal) => {
|
|
7514
|
+
if (timeout) {
|
|
7515
|
+
clearTimeout(timeout);
|
|
7516
|
+
}
|
|
7517
|
+
clearKillTimeout();
|
|
7518
|
+
if (child.pid) {
|
|
7519
|
+
unregisterActiveClaudeCliChild(child.pid);
|
|
7520
|
+
}
|
|
7521
|
+
request.signal?.removeEventListener("abort", onAbort);
|
|
7522
|
+
if (timedOut) {
|
|
7523
|
+
resolve({
|
|
7524
|
+
status: status ?? 124,
|
|
7525
|
+
signal,
|
|
7526
|
+
stdout,
|
|
7527
|
+
stderr: appendBounded(stderr, `
|
|
7528
|
+
Claude CLI timed out after ${request.timeoutMs}ms.`)
|
|
7529
|
+
});
|
|
7530
|
+
return;
|
|
7531
|
+
}
|
|
7532
|
+
if (aborted) {
|
|
7533
|
+
resolve({
|
|
7534
|
+
status: status ?? 124,
|
|
7535
|
+
signal,
|
|
7536
|
+
stdout,
|
|
7537
|
+
stderr
|
|
7538
|
+
});
|
|
7539
|
+
return;
|
|
7540
|
+
}
|
|
7541
|
+
resolve({ status, signal, stdout, stderr });
|
|
7542
|
+
});
|
|
7543
|
+
try {
|
|
7544
|
+
child.stdin?.end(request.input);
|
|
7545
|
+
} catch (error) {
|
|
7546
|
+
stderr = appendBounded(stderr, `
|
|
7547
|
+
Claude CLI stdin error: ${error instanceof Error ? error.message : String(error)}`);
|
|
7548
|
+
}
|
|
7549
|
+
});
|
|
7550
|
+
}
|
|
7551
|
+
function registerActiveClaudeCliChild(pid) {
|
|
7552
|
+
installClaudeCliParentCleanup();
|
|
7553
|
+
activeClaudeCliChildPids.add(pid);
|
|
7554
|
+
}
|
|
7555
|
+
function unregisterActiveClaudeCliChild(pid) {
|
|
7556
|
+
activeClaudeCliChildPids.delete(pid);
|
|
7557
|
+
}
|
|
7558
|
+
function installClaudeCliParentCleanup() {
|
|
7559
|
+
if (claudeCliParentCleanupInstalled) {
|
|
7560
|
+
return;
|
|
7561
|
+
}
|
|
7562
|
+
claudeCliParentCleanupInstalled = true;
|
|
7563
|
+
process.once("exit", () => {
|
|
7564
|
+
terminateActiveClaudeCliChildren("SIGTERM");
|
|
7565
|
+
});
|
|
7566
|
+
for (const signal of CLAUDE_CLI_PARENT_SIGNALS) {
|
|
7567
|
+
process.once(signal, () => {
|
|
7568
|
+
const activeChildren = activeClaudeCliChildPids.size;
|
|
7569
|
+
terminateActiveClaudeCliChildren(signal === "SIGINT" ? "SIGINT" : "SIGTERM");
|
|
7570
|
+
process.exitCode = signalExitCode(signal);
|
|
7571
|
+
setTimeout(
|
|
7572
|
+
() => {
|
|
7573
|
+
terminateActiveClaudeCliChildren("SIGKILL");
|
|
7574
|
+
process.exit(signalExitCode(signal));
|
|
7575
|
+
},
|
|
7576
|
+
activeChildren > 0 ? CLAUDE_CLI_FORCED_PARENT_EXIT_MS : 0
|
|
7577
|
+
);
|
|
7578
|
+
});
|
|
7579
|
+
}
|
|
7580
|
+
}
|
|
7581
|
+
function terminateActiveClaudeCliChildren(signal) {
|
|
7582
|
+
for (const pid of activeClaudeCliChildPids) {
|
|
7583
|
+
terminateClaudeCliChildPid(pid, signal);
|
|
7584
|
+
}
|
|
7585
|
+
}
|
|
7586
|
+
function terminateClaudeCliChildPid(pid, signal) {
|
|
7587
|
+
if (process.platform !== "win32") {
|
|
7588
|
+
try {
|
|
7589
|
+
process.kill(-pid, signal);
|
|
7590
|
+
return;
|
|
7591
|
+
} catch {
|
|
7592
|
+
}
|
|
7593
|
+
}
|
|
7594
|
+
try {
|
|
7595
|
+
process.kill(pid, signal);
|
|
7596
|
+
} catch {
|
|
7597
|
+
}
|
|
7598
|
+
}
|
|
7599
|
+
function signalExitCode(signal) {
|
|
7600
|
+
switch (signal) {
|
|
7601
|
+
case "SIGHUP":
|
|
7602
|
+
return 129;
|
|
7603
|
+
case "SIGINT":
|
|
7604
|
+
return 130;
|
|
7605
|
+
case "SIGTERM":
|
|
7606
|
+
return 143;
|
|
7607
|
+
default:
|
|
7608
|
+
return 1;
|
|
7609
|
+
}
|
|
7610
|
+
}
|
|
7611
|
+
function appendBounded(existing, next) {
|
|
7612
|
+
const combined = existing + next;
|
|
7613
|
+
if (combined.length <= CLAUDE_CLI_STDIO_LIMIT) {
|
|
7614
|
+
return combined;
|
|
7615
|
+
}
|
|
7616
|
+
return combined.slice(combined.length - CLAUDE_CLI_STDIO_LIMIT);
|
|
7617
|
+
}
|
|
7618
|
+
function createClaudeCliProvider(config, deps) {
|
|
7619
|
+
return new ClaudeCliProvider(config, deps);
|
|
7620
|
+
}
|
|
7621
|
+
|
|
7622
|
+
// src/providers/codex-cli.ts
|
|
7623
|
+
import { spawn as spawn2 } from "child_process";
|
|
7624
|
+
import { createHash as createHash5, randomUUID } from "crypto";
|
|
7625
|
+
import { mkdir as mkdir5, mkdtemp as mkdtemp3, readFile as readFile6, rm as rm3, writeFile as writeFile5 } from "fs/promises";
|
|
7626
|
+
import os4 from "os";
|
|
7627
|
+
import path7 from "path";
|
|
7628
|
+
var DEFAULT_REASONING_EFFORT = "xhigh";
|
|
7629
|
+
var DEFAULT_SERVICE_TIER = "fast";
|
|
7630
|
+
var CODEX_CLI_STDIO_LIMIT = 64e3;
|
|
7631
|
+
var CODEX_CLI_PARENT_SIGNALS = [
|
|
7632
|
+
"SIGHUP",
|
|
7633
|
+
"SIGINT",
|
|
7634
|
+
"SIGTERM"
|
|
7635
|
+
];
|
|
7636
|
+
var CODEX_CLI_FORCED_PARENT_EXIT_MS = 1e3;
|
|
7637
|
+
var CODEX_CLI_DIAGNOSTICS_DIR_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_DIR";
|
|
7638
|
+
var CODEX_CLI_DIAGNOSTICS_MODE_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_MODE";
|
|
7639
|
+
var CODEX_CLI_EXECUTABLE_ENV = "REMNIC_BENCH_CODEX_CLI_EXECUTABLE";
|
|
7640
|
+
var CODEX_CLI_TRANSPORT_ENV = "REMNIC_BENCH_CODEX_CLI_TRANSPORT";
|
|
7641
|
+
var CODEX_CLI_VERSION_TIMEOUT_MS = 5e3;
|
|
7642
|
+
var CODEX_CLI_HEALTH_CACHE_TTL_MS = 3e4;
|
|
7643
|
+
var OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
|
|
7644
|
+
var OPENAI_BASE_URL_ENV = "OPENAI_BASE_URL";
|
|
7645
|
+
var OPENAI_RESPONSES_BASE_URL = "https://api.openai.com/v1";
|
|
7646
|
+
var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
|
|
7647
|
+
"ALL_PROXY",
|
|
7648
|
+
"APPDATA",
|
|
7649
|
+
"CODEX_HOME",
|
|
7650
|
+
"COLORTERM",
|
|
7651
|
+
"COMSPEC",
|
|
7652
|
+
"FORCE_COLOR",
|
|
7653
|
+
"HOME",
|
|
7654
|
+
"HOMEDRIVE",
|
|
7655
|
+
"HOMEPATH",
|
|
7656
|
+
"LANG",
|
|
7657
|
+
"LOCALAPPDATA",
|
|
7658
|
+
"LOGNAME",
|
|
7659
|
+
"NO_COLOR",
|
|
7660
|
+
"NODE_EXTRA_CA_CERTS",
|
|
7661
|
+
"NO_PROXY",
|
|
7662
|
+
"NUMBER_OF_PROCESSORS",
|
|
7663
|
+
OPENAI_BASE_URL_ENV,
|
|
7664
|
+
"OPENAI_ORGANIZATION",
|
|
7665
|
+
"OPENAI_PROJECT",
|
|
7666
|
+
"OS",
|
|
7667
|
+
"PATH",
|
|
7668
|
+
"PATHEXT",
|
|
7669
|
+
"PROGRAMDATA",
|
|
7670
|
+
"PROCESSOR_ARCHITECTURE",
|
|
7671
|
+
"HTTP_PROXY",
|
|
7672
|
+
"HTTPS_PROXY",
|
|
7673
|
+
"SHELL",
|
|
7674
|
+
"SSL_CERT_DIR",
|
|
7675
|
+
"SSL_CERT_FILE",
|
|
7676
|
+
"SYSTEMDRIVE",
|
|
7677
|
+
"SYSTEMROOT",
|
|
7678
|
+
"TEMP",
|
|
7679
|
+
"TERM",
|
|
7680
|
+
"TMP",
|
|
7681
|
+
"TMPDIR",
|
|
7682
|
+
"USER",
|
|
7683
|
+
"USERNAME",
|
|
7684
|
+
"USERPROFILE",
|
|
7685
|
+
"WINDIR",
|
|
7686
|
+
"XDG_CACHE_HOME",
|
|
7687
|
+
"XDG_CONFIG_HOME",
|
|
7688
|
+
"XDG_DATA_HOME",
|
|
7689
|
+
"XDG_RUNTIME_DIR"
|
|
7690
|
+
]);
|
|
7691
|
+
var activeCodexCliChildPids = /* @__PURE__ */ new Set();
|
|
7692
|
+
var codexCliParentCleanupInstalled = false;
|
|
7693
|
+
var codexCliHealthCache = /* @__PURE__ */ new Map();
|
|
7694
|
+
var CodexCliProvider = class {
|
|
7695
|
+
provider = "codex-cli";
|
|
7696
|
+
id;
|
|
7697
|
+
name;
|
|
7698
|
+
config;
|
|
7699
|
+
runCodexCli;
|
|
7700
|
+
runCodexVersion;
|
|
7701
|
+
shouldProbeCliHealth;
|
|
7702
|
+
usage = {
|
|
7703
|
+
inputTokens: 0,
|
|
7704
|
+
outputTokens: 0,
|
|
7705
|
+
totalTokens: 0
|
|
7706
|
+
};
|
|
7707
|
+
constructor(config, deps = {}) {
|
|
7708
|
+
this.config = config;
|
|
7709
|
+
this.runCodexCli = deps.runCodexCli ?? runCodexCliCommand;
|
|
7710
|
+
this.runCodexVersion = deps.runCodexVersion ?? runCodexVersionCommand;
|
|
7711
|
+
this.shouldProbeCliHealth = deps.runCodexCli === void 0;
|
|
7712
|
+
this.id = `codex-cli:${config.model}`;
|
|
7713
|
+
this.name = config.model;
|
|
7714
|
+
}
|
|
7715
|
+
async complete(prompt, opts = {}) {
|
|
7716
|
+
const startedAt = performance.now();
|
|
7717
|
+
if (await this.shouldUseResponsesFallback()) {
|
|
7718
|
+
return this.completeViaResponsesApi(prompt, opts, startedAt);
|
|
7719
|
+
}
|
|
7720
|
+
const maxAttempts = normalizeCodexCliMaxAttempts(
|
|
7721
|
+
this.config.retryOptions?.maxAttempts
|
|
7722
|
+
);
|
|
7723
|
+
let lastError;
|
|
7724
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
7725
|
+
const tempDir = await mkdtemp3(path7.join(os4.tmpdir(), "remnic-codex-cli-"));
|
|
7726
|
+
const workspacePath = path7.join(tempDir, "workspace");
|
|
7727
|
+
const outputPath = path7.join(tempDir, "last-message.txt");
|
|
7728
|
+
let diagnostics;
|
|
7729
|
+
let diagnosticsFinished = false;
|
|
7730
|
+
const finishDiagnostics = async (outcome) => {
|
|
7731
|
+
if (diagnosticsFinished) {
|
|
7732
|
+
return;
|
|
7733
|
+
}
|
|
7734
|
+
diagnosticsFinished = true;
|
|
7735
|
+
await finishCodexCliDiagnostics(diagnostics, startedAt, outcome);
|
|
7736
|
+
};
|
|
7737
|
+
try {
|
|
7738
|
+
await mkdir5(workspacePath, { recursive: true });
|
|
7739
|
+
const request = this.buildRunRequest(prompt, opts, workspacePath, outputPath);
|
|
7740
|
+
diagnostics = await startCodexCliDiagnostics({
|
|
7741
|
+
config: this.config,
|
|
7742
|
+
request,
|
|
7743
|
+
reasoningEffort: this.config.reasoningEffort ?? DEFAULT_REASONING_EFFORT,
|
|
7744
|
+
serviceTier: DEFAULT_SERVICE_TIER,
|
|
7745
|
+
retry: { attempt, maxAttempts }
|
|
7746
|
+
});
|
|
7747
|
+
const result = await this.runCodexCli(request);
|
|
7748
|
+
if (result.status !== 0) {
|
|
7749
|
+
const exitLabel = result.signal ? `signal ${result.signal}` : `exit ${result.status ?? "unknown"}`;
|
|
7750
|
+
const error = new Error(
|
|
7751
|
+
`Codex CLI completion failed (${exitLabel}): ${summarizeProcessOutput2(result.stderr, result.stdout)}`
|
|
7752
|
+
);
|
|
7753
|
+
if (attempt < maxAttempts && isRetryableCodexCliResult(result)) {
|
|
7754
|
+
lastError = error;
|
|
7755
|
+
await finishDiagnostics({
|
|
7756
|
+
result,
|
|
7757
|
+
error,
|
|
7758
|
+
transientFailure: true
|
|
7759
|
+
});
|
|
7760
|
+
await sleepBeforeCodexCliRetry(
|
|
7761
|
+
attempt,
|
|
7762
|
+
this.config.retryOptions?.baseBackoffMs,
|
|
7763
|
+
opts.signal
|
|
7764
|
+
);
|
|
7765
|
+
continue;
|
|
7766
|
+
}
|
|
7767
|
+
await finishDiagnostics({ result, error });
|
|
7768
|
+
throw error;
|
|
7769
|
+
}
|
|
7770
|
+
const text = result.outputText.trim();
|
|
7771
|
+
if (text.length === 0) {
|
|
7772
|
+
const error = new Error(
|
|
7773
|
+
`Codex CLI completion returned no final message: ${summarizeProcessOutput2(result.stderr, result.stdout)}`
|
|
7774
|
+
);
|
|
7775
|
+
await finishDiagnostics({ result, error });
|
|
7776
|
+
throw error;
|
|
7777
|
+
}
|
|
7778
|
+
await finishDiagnostics({ result });
|
|
7779
|
+
const tokens = parseCodexTokenUsage(
|
|
7780
|
+
`${result.stderr}
|
|
7781
|
+
${result.stdout}`,
|
|
7782
|
+
text
|
|
7783
|
+
);
|
|
7784
|
+
this.recordUsage(tokens.input, tokens.output);
|
|
7785
|
+
return {
|
|
7786
|
+
text,
|
|
7787
|
+
tokens,
|
|
7788
|
+
latencyMs: Math.round(performance.now() - startedAt),
|
|
7789
|
+
model: this.config.model
|
|
7790
|
+
};
|
|
7791
|
+
} catch (error) {
|
|
7792
|
+
lastError = error;
|
|
7793
|
+
await finishDiagnostics({ error });
|
|
7794
|
+
throw error;
|
|
7795
|
+
} finally {
|
|
7796
|
+
await rm3(tempDir, { force: true, recursive: true });
|
|
7797
|
+
}
|
|
7798
|
+
}
|
|
7799
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
7800
|
+
}
|
|
7801
|
+
async discover() {
|
|
7802
|
+
const version = await this.runCodexVersion(
|
|
7803
|
+
resolveCodexCliExecutable(this.config),
|
|
7804
|
+
buildIsolatedCodexEnv()
|
|
7805
|
+
);
|
|
7806
|
+
if (version.status !== 0) {
|
|
7807
|
+
throw new Error(
|
|
7808
|
+
`Codex CLI discovery failed: ${version.stderr.trim() || `exit ${version.status ?? "unknown"}`}`
|
|
7809
|
+
);
|
|
7810
|
+
}
|
|
7811
|
+
return [
|
|
7812
|
+
{
|
|
7813
|
+
id: this.config.model,
|
|
7814
|
+
name: `${this.config.model} (Codex CLI)`,
|
|
6481
7815
|
contextLength: 0,
|
|
6482
7816
|
capabilities: ["completion"]
|
|
6483
7817
|
}
|
|
@@ -6686,7 +8020,7 @@ function extractResponsesOutputText(payload) {
|
|
|
6686
8020
|
}
|
|
6687
8021
|
function runCodexVersionCommand(executable, env) {
|
|
6688
8022
|
return new Promise((resolve, reject) => {
|
|
6689
|
-
const child =
|
|
8023
|
+
const child = spawn2(executable, ["--version"], {
|
|
6690
8024
|
env,
|
|
6691
8025
|
stdio: ["ignore", "ignore", "pipe"],
|
|
6692
8026
|
detached: process.platform !== "win32",
|
|
@@ -6716,7 +8050,7 @@ function runCodexVersionCommand(executable, env) {
|
|
|
6716
8050
|
timeout.unref();
|
|
6717
8051
|
child.stderr?.setEncoding("utf8");
|
|
6718
8052
|
child.stderr?.on("data", (chunk) => {
|
|
6719
|
-
stderr =
|
|
8053
|
+
stderr = appendBounded2(stderr, chunk);
|
|
6720
8054
|
});
|
|
6721
8055
|
child.on("error", (error) => {
|
|
6722
8056
|
clearTimeout(timeout);
|
|
@@ -6732,7 +8066,7 @@ function runCodexVersionCommand(executable, env) {
|
|
|
6732
8066
|
}
|
|
6733
8067
|
resolve({
|
|
6734
8068
|
status: timedOut ? status ?? 124 : status,
|
|
6735
|
-
stderr: timedOut ?
|
|
8069
|
+
stderr: timedOut ? appendBounded2(
|
|
6736
8070
|
stderr,
|
|
6737
8071
|
`
|
|
6738
8072
|
Codex CLI --version timed out after ${CODEX_CLI_VERSION_TIMEOUT_MS}ms.`
|
|
@@ -6752,7 +8086,7 @@ function resolveCodexCliExecutable(config) {
|
|
|
6752
8086
|
`${CODEX_CLI_EXECUTABLE_ENV} / codex-cli executable must not be empty`
|
|
6753
8087
|
);
|
|
6754
8088
|
}
|
|
6755
|
-
return
|
|
8089
|
+
return expandHomeRelativePath2(trimmed);
|
|
6756
8090
|
}
|
|
6757
8091
|
function buildCodexCompletionPrompt(userPrompt, systemPrompt) {
|
|
6758
8092
|
const payload = {
|
|
@@ -6810,10 +8144,10 @@ async function startCodexCliDiagnostics(args) {
|
|
|
6810
8144
|
model: args.config.model,
|
|
6811
8145
|
reasoningEffort: args.reasoningEffort,
|
|
6812
8146
|
serviceTier: args.serviceTier,
|
|
6813
|
-
executable:
|
|
8147
|
+
executable: path7.basename(args.request.executable),
|
|
6814
8148
|
...args.request.timeoutMs ? { timeoutMs: args.request.timeoutMs } : {},
|
|
6815
|
-
workspaceBasename:
|
|
6816
|
-
outputBasename:
|
|
8149
|
+
workspaceBasename: path7.basename(args.request.workspacePath),
|
|
8150
|
+
outputBasename: path7.basename(args.request.outputPath),
|
|
6817
8151
|
prompt: promptStats,
|
|
6818
8152
|
command: {
|
|
6819
8153
|
args: redactCodexCliArgs(args.request.args)
|
|
@@ -6821,7 +8155,7 @@ async function startCodexCliDiagnostics(args) {
|
|
|
6821
8155
|
retry: args.retry,
|
|
6822
8156
|
...mode === "full" ? { fullPrompt: args.request.input } : {}
|
|
6823
8157
|
};
|
|
6824
|
-
const filePath =
|
|
8158
|
+
const filePath = path7.join(diagnosticsDir, `${id}.json`);
|
|
6825
8159
|
await writeCodexCliDiagnosticRecord(filePath, record);
|
|
6826
8160
|
return { path: filePath, record };
|
|
6827
8161
|
} catch {
|
|
@@ -6873,14 +8207,14 @@ async function writeCodexCliDiagnosticRecord(filePath, record) {
|
|
|
6873
8207
|
function resolveCodexCliDiagnosticsDir(config) {
|
|
6874
8208
|
const dir = config.diagnosticsDir ?? process.env[CODEX_CLI_DIAGNOSTICS_DIR_ENV];
|
|
6875
8209
|
const trimmed = typeof dir === "string" ? dir.trim() : "";
|
|
6876
|
-
return trimmed.length > 0 ?
|
|
8210
|
+
return trimmed.length > 0 ? path7.resolve(expandHomeRelativePath2(trimmed)) : void 0;
|
|
6877
8211
|
}
|
|
6878
|
-
function
|
|
8212
|
+
function expandHomeRelativePath2(value) {
|
|
6879
8213
|
if (value === "~") {
|
|
6880
|
-
return
|
|
8214
|
+
return os4.homedir();
|
|
6881
8215
|
}
|
|
6882
8216
|
if (value.startsWith("~/") || value.startsWith("~\\")) {
|
|
6883
|
-
return
|
|
8217
|
+
return path7.join(os4.homedir(), value.slice(2));
|
|
6884
8218
|
}
|
|
6885
8219
|
return value;
|
|
6886
8220
|
}
|
|
@@ -6939,7 +8273,7 @@ function runCodexCliCommand(request) {
|
|
|
6939
8273
|
});
|
|
6940
8274
|
return;
|
|
6941
8275
|
}
|
|
6942
|
-
const child =
|
|
8276
|
+
const child = spawn2(request.executable, request.args, {
|
|
6943
8277
|
cwd: request.workspacePath,
|
|
6944
8278
|
env: request.env,
|
|
6945
8279
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -6982,7 +8316,7 @@ function runCodexCliCommand(request) {
|
|
|
6982
8316
|
return;
|
|
6983
8317
|
}
|
|
6984
8318
|
aborted = true;
|
|
6985
|
-
stderr =
|
|
8319
|
+
stderr = appendBounded2(stderr, "\nCodex CLI aborted by benchmark timeout.");
|
|
6986
8320
|
terminateChild("SIGTERM");
|
|
6987
8321
|
scheduleForcedKill();
|
|
6988
8322
|
};
|
|
@@ -6999,13 +8333,13 @@ function runCodexCliCommand(request) {
|
|
|
6999
8333
|
child.stdout?.setEncoding("utf8");
|
|
7000
8334
|
child.stderr?.setEncoding("utf8");
|
|
7001
8335
|
child.stdout?.on("data", (chunk) => {
|
|
7002
|
-
stdout =
|
|
8336
|
+
stdout = appendBounded2(stdout, chunk);
|
|
7003
8337
|
});
|
|
7004
8338
|
child.stderr?.on("data", (chunk) => {
|
|
7005
|
-
stderr =
|
|
8339
|
+
stderr = appendBounded2(stderr, chunk);
|
|
7006
8340
|
});
|
|
7007
8341
|
child.stdin?.on("error", (error) => {
|
|
7008
|
-
stderr =
|
|
8342
|
+
stderr = appendBounded2(
|
|
7009
8343
|
stderr,
|
|
7010
8344
|
`
|
|
7011
8345
|
Codex CLI stdin error: ${error.code ?? error.message}`
|
|
@@ -7036,7 +8370,7 @@ Codex CLI stdin error: ${error.code ?? error.message}`
|
|
|
7036
8370
|
status: status ?? 124,
|
|
7037
8371
|
signal,
|
|
7038
8372
|
stdout,
|
|
7039
|
-
stderr:
|
|
8373
|
+
stderr: appendBounded2(
|
|
7040
8374
|
stderr,
|
|
7041
8375
|
`
|
|
7042
8376
|
Codex CLI timed out after ${request.timeoutMs}ms.`
|
|
@@ -7065,7 +8399,7 @@ Codex CLI timed out after ${request.timeoutMs}ms.`
|
|
|
7065
8399
|
try {
|
|
7066
8400
|
child.stdin?.end(request.input);
|
|
7067
8401
|
} catch (error) {
|
|
7068
|
-
stderr =
|
|
8402
|
+
stderr = appendBounded2(
|
|
7069
8403
|
stderr,
|
|
7070
8404
|
`
|
|
7071
8405
|
Codex CLI stdin error: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -7092,11 +8426,11 @@ function installCodexCliParentCleanup() {
|
|
|
7092
8426
|
process.once(signal, () => {
|
|
7093
8427
|
const activeChildren = activeCodexCliChildPids.size;
|
|
7094
8428
|
terminateActiveCodexCliChildren(signal === "SIGINT" ? "SIGINT" : "SIGTERM");
|
|
7095
|
-
process.exitCode =
|
|
8429
|
+
process.exitCode = signalExitCode2(signal);
|
|
7096
8430
|
setTimeout(
|
|
7097
8431
|
() => {
|
|
7098
8432
|
terminateActiveCodexCliChildren("SIGKILL");
|
|
7099
|
-
process.exit(
|
|
8433
|
+
process.exit(signalExitCode2(signal));
|
|
7100
8434
|
},
|
|
7101
8435
|
activeChildren > 0 ? CODEX_CLI_FORCED_PARENT_EXIT_MS : 0
|
|
7102
8436
|
);
|
|
@@ -7121,7 +8455,7 @@ function terminateCodexCliChildPid(pid, signal) {
|
|
|
7121
8455
|
} catch {
|
|
7122
8456
|
}
|
|
7123
8457
|
}
|
|
7124
|
-
function
|
|
8458
|
+
function signalExitCode2(signal) {
|
|
7125
8459
|
switch (signal) {
|
|
7126
8460
|
case "SIGHUP":
|
|
7127
8461
|
return 129;
|
|
@@ -7140,7 +8474,7 @@ async function readCodexOutput(outputPath, stdout) {
|
|
|
7140
8474
|
return stdout;
|
|
7141
8475
|
}
|
|
7142
8476
|
}
|
|
7143
|
-
function
|
|
8477
|
+
function appendBounded2(existing, next) {
|
|
7144
8478
|
const combined = existing + next;
|
|
7145
8479
|
if (combined.length <= CODEX_CLI_STDIO_LIMIT) {
|
|
7146
8480
|
return combined;
|
|
@@ -7210,7 +8544,7 @@ function codexCliAbortError(signal) {
|
|
|
7210
8544
|
}
|
|
7211
8545
|
return new DOMException("The operation was aborted.", "AbortError");
|
|
7212
8546
|
}
|
|
7213
|
-
function
|
|
8547
|
+
function summarizeProcessOutput2(stderr, stdout) {
|
|
7214
8548
|
const summary = [stderr.trim(), stdout.trim()].filter((value) => value.length > 0).join("\n").trim();
|
|
7215
8549
|
return summary.length > 0 ? summary.slice(-1e3) : "no process output";
|
|
7216
8550
|
}
|
|
@@ -7247,19 +8581,19 @@ function createCodexCliProvider(config, deps) {
|
|
|
7247
8581
|
// src/reporter.ts
|
|
7248
8582
|
import { execSync } from "child_process";
|
|
7249
8583
|
import { mkdir as mkdir7, readFile as readFile7, writeFile as writeFile7 } from "fs/promises";
|
|
7250
|
-
import
|
|
8584
|
+
import path10 from "path";
|
|
7251
8585
|
|
|
7252
8586
|
// src/filename-safety.ts
|
|
7253
|
-
import
|
|
8587
|
+
import path8 from "path";
|
|
7254
8588
|
function sanitizeFilenameSegment(value) {
|
|
7255
8589
|
const sanitized = value.trim().replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
7256
8590
|
return sanitized.length > 0 ? sanitized : "unknown";
|
|
7257
8591
|
}
|
|
7258
8592
|
function resolveContainedPath(root, ...segments) {
|
|
7259
|
-
const outputRoot =
|
|
7260
|
-
const filePath =
|
|
7261
|
-
const relativePath =
|
|
7262
|
-
if (relativePath === ".." || relativePath.startsWith(`..${
|
|
8593
|
+
const outputRoot = path8.resolve(root);
|
|
8594
|
+
const filePath = path8.resolve(outputRoot, ...segments);
|
|
8595
|
+
const relativePath = path8.relative(outputRoot, filePath);
|
|
8596
|
+
if (relativePath === ".." || relativePath.startsWith(`..${path8.sep}`) || path8.isAbsolute(relativePath)) {
|
|
7263
8597
|
throw new Error(`Refusing to write benchmark artifact outside ${outputRoot}`);
|
|
7264
8598
|
}
|
|
7265
8599
|
return filePath;
|
|
@@ -7267,7 +8601,7 @@ function resolveContainedPath(root, ...segments) {
|
|
|
7267
8601
|
|
|
7268
8602
|
// src/leaderboard-export.ts
|
|
7269
8603
|
import { mkdir as mkdir6, writeFile as writeFile6 } from "fs/promises";
|
|
7270
|
-
import
|
|
8604
|
+
import path9 from "path";
|
|
7271
8605
|
async function writeLeaderboardArtifactsForResult(result, outputDir) {
|
|
7272
8606
|
if (result.meta.benchmark === "ama-bench") {
|
|
7273
8607
|
return writeAmaBenchLeaderboard(result, outputDir);
|
|
@@ -7282,7 +8616,7 @@ async function writeAmaBenchLeaderboard(result, outputDir) {
|
|
|
7282
8616
|
if (rows.length === 0) {
|
|
7283
8617
|
return [];
|
|
7284
8618
|
}
|
|
7285
|
-
const outputRoot =
|
|
8619
|
+
const outputRoot = path9.resolve(outputDir);
|
|
7286
8620
|
const leaderboardDir = resolveContainedPath(outputRoot, "leaderboard");
|
|
7287
8621
|
await mkdir6(leaderboardDir, { recursive: true });
|
|
7288
8622
|
const timestamp = sanitizeFilenameSegment(result.meta.timestamp.replace(/[:.]/g, "-"));
|
|
@@ -7300,7 +8634,7 @@ async function writeAmaBenchLeaderboard(result, outputDir) {
|
|
|
7300
8634
|
async function writeMemCorrectLeaderboard(result, outputDir) {
|
|
7301
8635
|
const row = buildMemCorrectLeaderboardRow(result);
|
|
7302
8636
|
if (!row) return [];
|
|
7303
|
-
const outputRoot =
|
|
8637
|
+
const outputRoot = path9.resolve(outputDir);
|
|
7304
8638
|
const leaderboardDir = resolveContainedPath(outputRoot, "leaderboard");
|
|
7305
8639
|
await mkdir6(leaderboardDir, { recursive: true });
|
|
7306
8640
|
const timestamp = sanitizeFilenameSegment(result.meta.timestamp.replace(/[:.]/g, "-"));
|
|
@@ -7668,7 +9002,7 @@ function replaceLoneSurrogates(value) {
|
|
|
7668
9002
|
return out;
|
|
7669
9003
|
}
|
|
7670
9004
|
async function writeBenchmarkResult(result, outputDir) {
|
|
7671
|
-
const outputRoot =
|
|
9005
|
+
const outputRoot = path10.resolve(outputDir);
|
|
7672
9006
|
await mkdir7(outputRoot, { recursive: true });
|
|
7673
9007
|
const safeBenchmark = sanitizeFilenameSegment(result.meta.benchmark);
|
|
7674
9008
|
const safeRemnicVersion = sanitizeFilenameSegment(result.meta.remnicVersion);
|
|
@@ -7704,7 +9038,7 @@ async function writeBenchmarkResult(result, outputDir) {
|
|
|
7704
9038
|
async function getRemnicVersion() {
|
|
7705
9039
|
try {
|
|
7706
9040
|
const packageJson = JSON.parse(
|
|
7707
|
-
await readFile7(
|
|
9041
|
+
await readFile7(path10.resolve(import.meta.dirname, "../../../package.json"), "utf8")
|
|
7708
9042
|
);
|
|
7709
9043
|
return typeof packageJson.version === "string" ? packageJson.version : "unknown";
|
|
7710
9044
|
} catch {
|
|
@@ -8210,6 +9544,8 @@ function createProvider(config) {
|
|
|
8210
9544
|
return createLocalLlmProvider(config);
|
|
8211
9545
|
case "codex-cli":
|
|
8212
9546
|
return createCodexCliProvider(config);
|
|
9547
|
+
case "claude-cli":
|
|
9548
|
+
return createClaudeCliProvider(config);
|
|
8213
9549
|
default: {
|
|
8214
9550
|
const exhaustive = config;
|
|
8215
9551
|
throw new Error(`Unknown provider: ${JSON.stringify(exhaustive)}`);
|
|
@@ -9752,7 +11088,7 @@ function clampNormalizedScore(value) {
|
|
|
9752
11088
|
}
|
|
9753
11089
|
|
|
9754
11090
|
// src/runtime-profiles.ts
|
|
9755
|
-
import
|
|
11091
|
+
import path11 from "path";
|
|
9756
11092
|
import { readFile as readFile9 } from "fs/promises";
|
|
9757
11093
|
import {
|
|
9758
11094
|
resolvePluginEntry,
|
|
@@ -10796,10 +12132,10 @@ async function loadOpenclawRuntimeConfig(filePath) {
|
|
|
10796
12132
|
};
|
|
10797
12133
|
}
|
|
10798
12134
|
function deriveOpenclawRuntimeContext(configPath) {
|
|
10799
|
-
const rootDir =
|
|
12135
|
+
const rootDir = path11.dirname(path11.resolve(configPath));
|
|
10800
12136
|
return {
|
|
10801
|
-
agentDir:
|
|
10802
|
-
workspaceDir:
|
|
12137
|
+
agentDir: path11.join(rootDir, "agents", "main", "agent"),
|
|
12138
|
+
workspaceDir: path11.join(rootDir, "workspace")
|
|
10803
12139
|
};
|
|
10804
12140
|
}
|
|
10805
12141
|
async function loadJsonObject(filePath, label) {
|
|
@@ -10836,6 +12172,11 @@ function resolveProviderConfig(kind, provider, model, baseUrl, requestTimeout, d
|
|
|
10836
12172
|
`${kind} provider "local-llm" requires a baseUrl (e.g. http://localhost:8080/v1 for llama.cpp).`
|
|
10837
12173
|
);
|
|
10838
12174
|
}
|
|
12175
|
+
if (kind === "internal" && provider === "claude-cli") {
|
|
12176
|
+
throw new Error(
|
|
12177
|
+
`${kind} provider "claude-cli" is not supported: it has no @remnic/core gateway wiring and is only available via --provider, --system-provider, or --judge-provider.`
|
|
12178
|
+
);
|
|
12179
|
+
}
|
|
10839
12180
|
if (reasoningEffort !== void 0 && provider !== "codex-cli") {
|
|
10840
12181
|
throw new Error(
|
|
10841
12182
|
`${kind} Codex reasoning effort requires provider "codex-cli"`
|
|
@@ -10958,6 +12299,11 @@ function gatewayProviderApi(provider) {
|
|
|
10958
12299
|
if (provider === "codex-cli") {
|
|
10959
12300
|
return "codex-cli";
|
|
10960
12301
|
}
|
|
12302
|
+
if (provider === "claude-cli") {
|
|
12303
|
+
throw new Error(
|
|
12304
|
+
"claude-cli has no @remnic/core gateway api mapping; it is only supported as a --provider / --system-provider / --judge-provider, not an internal provider."
|
|
12305
|
+
);
|
|
12306
|
+
}
|
|
10961
12307
|
if (provider === "ollama") {
|
|
10962
12308
|
return "ollama-chat";
|
|
10963
12309
|
}
|
|
@@ -10978,6 +12324,10 @@ function defaultInternalBaseUrl(provider) {
|
|
|
10978
12324
|
return "http://localhost:11434/api";
|
|
10979
12325
|
case "codex-cli":
|
|
10980
12326
|
return "codex-cli://local";
|
|
12327
|
+
case "claude-cli":
|
|
12328
|
+
throw new Error(
|
|
12329
|
+
"claude-cli has no internal-provider base URL; it is only supported as a --provider / --system-provider / --judge-provider, not an internal provider."
|
|
12330
|
+
);
|
|
10981
12331
|
case "local-llm":
|
|
10982
12332
|
return void 0;
|
|
10983
12333
|
default: {
|
|
@@ -11198,7 +12548,7 @@ async function resolveLocalLabRuntimeProfile(options) {
|
|
|
11198
12548
|
|
|
11199
12549
|
// src/benchmark.ts
|
|
11200
12550
|
import fs2 from "fs";
|
|
11201
|
-
import
|
|
12551
|
+
import path34 from "path";
|
|
11202
12552
|
import { createHash as createHash11 } from "crypto";
|
|
11203
12553
|
import { expandTildePath as expandTildePath3 } from "@remnic/core";
|
|
11204
12554
|
|
|
@@ -11208,10 +12558,10 @@ import {
|
|
|
11208
12558
|
mkdir as mkdir8,
|
|
11209
12559
|
readFile as readFile10,
|
|
11210
12560
|
rename as rename2,
|
|
11211
|
-
rm as
|
|
12561
|
+
rm as rm4,
|
|
11212
12562
|
writeFile as writeFile8
|
|
11213
12563
|
} from "fs/promises";
|
|
11214
|
-
import
|
|
12564
|
+
import path12 from "path";
|
|
11215
12565
|
var JUDGE_CACHE_PROTOCOL_VERSION = "judge-protocol-v1";
|
|
11216
12566
|
function stableStringify2(value) {
|
|
11217
12567
|
if (Array.isArray(value)) {
|
|
@@ -11242,7 +12592,7 @@ var JudgeCache = class {
|
|
|
11242
12592
|
inflight = /* @__PURE__ */ new Map();
|
|
11243
12593
|
cachedDirExists = false;
|
|
11244
12594
|
constructor(options) {
|
|
11245
|
-
this.dir =
|
|
12595
|
+
this.dir = path12.resolve(options.dir);
|
|
11246
12596
|
}
|
|
11247
12597
|
/** Compute the sha256-hex key for a set of parts. Pure, sync, side-effect-free. */
|
|
11248
12598
|
computeKey(parts) {
|
|
@@ -11317,7 +12667,7 @@ var JudgeCache = class {
|
|
|
11317
12667
|
this.cachedDirExists = true;
|
|
11318
12668
|
}
|
|
11319
12669
|
const filePath = this.entryPath(key);
|
|
11320
|
-
const tempPath =
|
|
12670
|
+
const tempPath = path12.join(
|
|
11321
12671
|
this.dir,
|
|
11322
12672
|
`.${key}.${randomBytes2(6).toString("hex")}.tmp`
|
|
11323
12673
|
);
|
|
@@ -11326,12 +12676,12 @@ var JudgeCache = class {
|
|
|
11326
12676
|
try {
|
|
11327
12677
|
await rename2(tempPath, filePath);
|
|
11328
12678
|
} catch (error) {
|
|
11329
|
-
await
|
|
12679
|
+
await rm4(tempPath, { force: true }).catch(() => void 0);
|
|
11330
12680
|
throw error;
|
|
11331
12681
|
}
|
|
11332
12682
|
}
|
|
11333
12683
|
entryPath(key) {
|
|
11334
|
-
return
|
|
12684
|
+
return path12.join(this.dir, `${key}.json`);
|
|
11335
12685
|
}
|
|
11336
12686
|
};
|
|
11337
12687
|
function runJudgeWithCache(options) {
|
|
@@ -11506,7 +12856,7 @@ function isBenchJudgeResult(value) {
|
|
|
11506
12856
|
// src/benchmarks/published/ama-bench/runner.ts
|
|
11507
12857
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
11508
12858
|
import { readFile as readFile11 } from "fs/promises";
|
|
11509
|
-
import
|
|
12859
|
+
import path13 from "path";
|
|
11510
12860
|
|
|
11511
12861
|
// src/benchmarks/published/ama-bench/fixture.ts
|
|
11512
12862
|
var AMA_BENCH_SMOKE_FIXTURE = [
|
|
@@ -12081,7 +13431,7 @@ async function loadDataset(mode, datasetDir, limit) {
|
|
|
12081
13431
|
return episodes;
|
|
12082
13432
|
};
|
|
12083
13433
|
if (datasetDir) {
|
|
12084
|
-
const filePath =
|
|
13434
|
+
const filePath = path13.join(datasetDir, "open_end_qa_set.jsonl");
|
|
12085
13435
|
let raw;
|
|
12086
13436
|
try {
|
|
12087
13437
|
raw = await readFile11(filePath, "utf8");
|
|
@@ -12377,7 +13727,7 @@ function isValidQaPairs(value) {
|
|
|
12377
13727
|
// src/benchmarks/published/amemgym/runner.ts
|
|
12378
13728
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
12379
13729
|
import { readFile as readFile12 } from "fs/promises";
|
|
12380
|
-
import
|
|
13730
|
+
import path14 from "path";
|
|
12381
13731
|
|
|
12382
13732
|
// src/benchmarks/published/amemgym/fixture.ts
|
|
12383
13733
|
var AMEMGYM_SMOKE_FIXTURE = [
|
|
@@ -12906,7 +14256,7 @@ async function loadDataset2(mode, datasetDir, limit) {
|
|
|
12906
14256
|
const datasetErrors = [];
|
|
12907
14257
|
for (const filename of DATASET_FILENAMES) {
|
|
12908
14258
|
try {
|
|
12909
|
-
const raw = await readFile12(
|
|
14259
|
+
const raw = await readFile12(path14.join(datasetDir, filename), "utf8");
|
|
12910
14260
|
const parsed = parseDataset(raw, filename, normalizedLimit);
|
|
12911
14261
|
return ensureDatasetProfiles(parsed);
|
|
12912
14262
|
} catch (error) {
|
|
@@ -13081,7 +14431,7 @@ function normalizeRole(role) {
|
|
|
13081
14431
|
// src/benchmarks/published/memory-arena/runner.ts
|
|
13082
14432
|
import { randomUUID as randomUUID4 } from "crypto";
|
|
13083
14433
|
import { readFile as readFile13, readdir as readdir5, stat as stat3 } from "fs/promises";
|
|
13084
|
-
import
|
|
14434
|
+
import path15 from "path";
|
|
13085
14435
|
import { expandTildePath as expandTildePath2 } from "@remnic/core";
|
|
13086
14436
|
|
|
13087
14437
|
// src/benchmarks/published/memory-arena/fixture.ts
|
|
@@ -13408,7 +14758,7 @@ async function loadDataset3(mode, datasetDir, limit) {
|
|
|
13408
14758
|
if (remainingLimit2 === 0) {
|
|
13409
14759
|
break;
|
|
13410
14760
|
}
|
|
13411
|
-
const raw = await readFile13(
|
|
14761
|
+
const raw = await readFile13(path15.join(datasetDir, filename), "utf8");
|
|
13412
14762
|
const parsedTasks = [];
|
|
13413
14763
|
raw.split("\n").forEach((line, lineIndex) => {
|
|
13414
14764
|
if (line.trim().length === 0) {
|
|
@@ -13764,14 +15114,14 @@ async function loadMemoryArenaWebshopProductCatalog(datasetDir) {
|
|
|
13764
15114
|
async function resolveMemoryArenaWebshopProductCatalogPath(datasetDir) {
|
|
13765
15115
|
const configuredPath = process.env[MEMORY_ARENA_WEBSHOP_PRODUCTS_ENV]?.trim();
|
|
13766
15116
|
if (configuredPath && configuredPath.length > 0) {
|
|
13767
|
-
return
|
|
15117
|
+
return path15.resolve(expandTildePath2(configuredPath));
|
|
13768
15118
|
}
|
|
13769
15119
|
if (datasetDir === void 0) {
|
|
13770
15120
|
return void 0;
|
|
13771
15121
|
}
|
|
13772
15122
|
const candidatePaths = [
|
|
13773
15123
|
...MEMORY_ARENA_WEBSHOP_PRODUCT_SIDECAR_FILENAMES
|
|
13774
|
-
].map((filename) =>
|
|
15124
|
+
].map((filename) => path15.join(datasetDir, filename));
|
|
13775
15125
|
for (const candidatePath of candidatePaths) {
|
|
13776
15126
|
try {
|
|
13777
15127
|
const candidateStat = await stat3(candidatePath);
|
|
@@ -15194,7 +16544,7 @@ import { collectTemporalLexicalCues } from "@remnic/core";
|
|
|
15194
16544
|
|
|
15195
16545
|
// src/benchmarks/published/dataset-loader.ts
|
|
15196
16546
|
import { readFile as readFile14 } from "fs/promises";
|
|
15197
|
-
import
|
|
16547
|
+
import path16 from "path";
|
|
15198
16548
|
|
|
15199
16549
|
// src/benchmarks/published/longmemeval/fixture.ts
|
|
15200
16550
|
var LONG_MEM_EVAL_SMOKE_FIXTURE = [
|
|
@@ -15297,7 +16647,7 @@ async function loadDataset4(options) {
|
|
|
15297
16647
|
const errors = [];
|
|
15298
16648
|
if (options.datasetDir) {
|
|
15299
16649
|
for (const filename of options.filenames) {
|
|
15300
|
-
const abs =
|
|
16650
|
+
const abs = path16.join(options.datasetDir, filename);
|
|
15301
16651
|
let raw;
|
|
15302
16652
|
try {
|
|
15303
16653
|
raw = await readFile14(abs, "utf8");
|
|
@@ -17037,7 +18387,7 @@ function normalizeQaArray(value, location) {
|
|
|
17037
18387
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
17038
18388
|
import { createReadStream as createReadStream2 } from "fs";
|
|
17039
18389
|
import { readdir as readdir6 } from "fs/promises";
|
|
17040
|
-
import
|
|
18390
|
+
import path17 from "path";
|
|
17041
18391
|
import { createInterface } from "readline/promises";
|
|
17042
18392
|
import {
|
|
17043
18393
|
asyncBufferFromFile,
|
|
@@ -17508,8 +18858,8 @@ async function listBeamDatasetFiles(datasetDir) {
|
|
|
17508
18858
|
return directFiles;
|
|
17509
18859
|
}
|
|
17510
18860
|
try {
|
|
17511
|
-
const nestedFilenames = await readdir6(
|
|
17512
|
-
return nestedFilenames.filter((filename) => isBeamDatasetFilename(filename)).map((filename) =>
|
|
18861
|
+
const nestedFilenames = await readdir6(path17.join(datasetDir, "data"));
|
|
18862
|
+
return nestedFilenames.filter((filename) => isBeamDatasetFilename(filename)).map((filename) => path17.join("data", filename));
|
|
17513
18863
|
} catch {
|
|
17514
18864
|
return [];
|
|
17515
18865
|
}
|
|
@@ -17536,7 +18886,7 @@ async function* iterateDatasetFiles(datasetDir, datasetFiles, limit) {
|
|
|
17536
18886
|
let remainingLimit = limit;
|
|
17537
18887
|
for (const filename of datasetFiles) {
|
|
17538
18888
|
const scale = inferScaleFromFilename(filename);
|
|
17539
|
-
const filePath =
|
|
18889
|
+
const filePath = path17.join(datasetDir, filename);
|
|
17540
18890
|
const conversations = filename.endsWith(".jsonl") ? streamJsonlDataset(filePath, filename, remainingLimit) : filename.endsWith(".parquet") ? streamParquetDataset(filePath, filename, remainingLimit) : streamJsonDataset(filePath, filename, remainingLimit);
|
|
17541
18891
|
for await (const conversation of conversations) {
|
|
17542
18892
|
yield {
|
|
@@ -18549,7 +19899,7 @@ var StructuredLiteralParser = class {
|
|
|
18549
19899
|
// src/benchmarks/published/personamem/runner.ts
|
|
18550
19900
|
import { createHash as createHash7, randomUUID as randomUUID7 } from "crypto";
|
|
18551
19901
|
import { readFile as readFile15, realpath as realpath4 } from "fs/promises";
|
|
18552
|
-
import
|
|
19902
|
+
import path18 from "path";
|
|
18553
19903
|
|
|
18554
19904
|
// src/benchmarks/published/personamem/fixture.ts
|
|
18555
19905
|
var PERSONAMEM_SMOKE_FIXTURE = [
|
|
@@ -18825,7 +20175,7 @@ async function loadDataset8(mode, datasetDir, limit) {
|
|
|
18825
20175
|
if (datasetDir) {
|
|
18826
20176
|
const datasetErrors = [];
|
|
18827
20177
|
for (const relativePath of DATASET_FILE_CANDIDATES) {
|
|
18828
|
-
const datasetPath =
|
|
20178
|
+
const datasetPath = path18.join(datasetDir, relativePath);
|
|
18829
20179
|
let raw;
|
|
18830
20180
|
try {
|
|
18831
20181
|
raw = await readFile15(datasetPath, "utf8");
|
|
@@ -19019,12 +20369,12 @@ function parseCsv(raw, limit) {
|
|
|
19019
20369
|
return rows;
|
|
19020
20370
|
}
|
|
19021
20371
|
async function resolveDatasetFilePath(datasetRoot, relativePath) {
|
|
19022
|
-
const rootPath =
|
|
20372
|
+
const rootPath = path18.resolve(datasetRoot);
|
|
19023
20373
|
const rootRealPath = await realpath4(rootPath);
|
|
19024
|
-
const candidatePath =
|
|
20374
|
+
const candidatePath = path18.resolve(rootPath, relativePath);
|
|
19025
20375
|
const candidateRealPath = await realpath4(candidatePath);
|
|
19026
|
-
const relativeToRoot =
|
|
19027
|
-
if (relativeToRoot.startsWith("..") ||
|
|
20376
|
+
const relativeToRoot = path18.relative(rootRealPath, candidateRealPath);
|
|
20377
|
+
if (relativeToRoot.startsWith("..") || path18.isAbsolute(relativeToRoot)) {
|
|
19028
20378
|
throw new Error(
|
|
19029
20379
|
`PersonaMem-v2 dataset file reference "${relativePath}" must stay within datasetDir.`
|
|
19030
20380
|
);
|
|
@@ -19353,7 +20703,7 @@ function applyLimit6(items, limit) {
|
|
|
19353
20703
|
// src/benchmarks/published/membench/runner.ts
|
|
19354
20704
|
import { randomUUID as randomUUID8 } from "crypto";
|
|
19355
20705
|
import { readFile as readFile16, readdir as readdir7 } from "fs/promises";
|
|
19356
|
-
import
|
|
20706
|
+
import path19 from "path";
|
|
19357
20707
|
|
|
19358
20708
|
// src/benchmarks/published/membench/fixture.ts
|
|
19359
20709
|
var MEMBENCH_SMOKE_FIXTURE = [
|
|
@@ -19614,7 +20964,7 @@ async function loadDataset9(mode, datasetDir, limit) {
|
|
|
19614
20964
|
let remainingLimit = normalizedLimit;
|
|
19615
20965
|
for (const filename of filenames) {
|
|
19616
20966
|
try {
|
|
19617
|
-
const raw = await readFile16(
|
|
20967
|
+
const raw = await readFile16(path19.join(datasetDir, filename), "utf8");
|
|
19618
20968
|
const parsed = filename.endsWith(".jsonl") ? parseJsonlDataset(raw, filename) : parseJsonDataset(raw, filename);
|
|
19619
20969
|
const limitedCases = remainingLimit === 0 ? [] : applyLimit7(parsed, remainingLimit);
|
|
19620
20970
|
if (limitedCases.length > 0) {
|
|
@@ -20482,7 +21832,7 @@ function isPlainObject4(value) {
|
|
|
20482
21832
|
// src/benchmarks/published/memoryagentbench/runner.ts
|
|
20483
21833
|
import { randomUUID as randomUUID9 } from "crypto";
|
|
20484
21834
|
import { access, readFile as readFile17 } from "fs/promises";
|
|
20485
|
-
import
|
|
21835
|
+
import path20 from "path";
|
|
20486
21836
|
|
|
20487
21837
|
// src/benchmarks/published/memoryagentbench/fixture.ts
|
|
20488
21838
|
var MEMORY_AGENT_BENCH_SMOKE_FIXTURE = [
|
|
@@ -21561,21 +22911,21 @@ function recsysEntityMappingCandidates(datasetDir) {
|
|
|
21561
22911
|
if (!datasetDir) {
|
|
21562
22912
|
return [];
|
|
21563
22913
|
}
|
|
21564
|
-
const absoluteDatasetDir =
|
|
22914
|
+
const absoluteDatasetDir = path20.resolve(datasetDir);
|
|
21565
22915
|
const roots = [
|
|
21566
22916
|
absoluteDatasetDir,
|
|
21567
|
-
|
|
22917
|
+
path20.dirname(absoluteDatasetDir)
|
|
21568
22918
|
];
|
|
21569
22919
|
const canonicalSuffixes = [
|
|
21570
|
-
|
|
21571
|
-
|
|
22920
|
+
path20.join("processed_data", "Recsys_Redial", "entity2id.json"),
|
|
22921
|
+
path20.join("Recsys_Redial", "entity2id.json")
|
|
21572
22922
|
];
|
|
21573
22923
|
const looseSuffixes = ["entity2id.json"];
|
|
21574
22924
|
return [
|
|
21575
22925
|
...roots.flatMap(
|
|
21576
|
-
(root) => canonicalSuffixes.map((suffix) =>
|
|
22926
|
+
(root) => canonicalSuffixes.map((suffix) => path20.join(root, suffix))
|
|
21577
22927
|
),
|
|
21578
|
-
...looseSuffixes.map((suffix) =>
|
|
22928
|
+
...looseSuffixes.map((suffix) => path20.join(absoluteDatasetDir, suffix))
|
|
21579
22929
|
];
|
|
21580
22930
|
}
|
|
21581
22931
|
async function fileExists(filePath) {
|
|
@@ -21612,7 +22962,7 @@ async function loadDataset10(mode, datasetDir, limit) {
|
|
|
21612
22962
|
const datasetErrors = [];
|
|
21613
22963
|
for (const filename of DATASET_BUNDLE_CANDIDATES) {
|
|
21614
22964
|
const parsed = await tryReadDatasetFile(
|
|
21615
|
-
|
|
22965
|
+
path20.join(datasetDir, filename),
|
|
21616
22966
|
filename,
|
|
21617
22967
|
datasetErrors
|
|
21618
22968
|
);
|
|
@@ -21629,7 +22979,7 @@ async function loadDataset10(mode, datasetDir, limit) {
|
|
|
21629
22979
|
let splitData;
|
|
21630
22980
|
for (const filename of splitConfig.candidates) {
|
|
21631
22981
|
try {
|
|
21632
|
-
splitData = await readDatasetFile(
|
|
22982
|
+
splitData = await readDatasetFile(path20.join(datasetDir, filename), filename);
|
|
21633
22983
|
break;
|
|
21634
22984
|
} catch (error) {
|
|
21635
22985
|
if (!isFileNotFoundError2(error)) {
|
|
@@ -22277,8 +23627,8 @@ function loadCases(mode, limit) {
|
|
|
22277
23627
|
|
|
22278
23628
|
// src/benchmarks/remnic/extraction-judge-calibration/runner.ts
|
|
22279
23629
|
import { randomUUID as randomUUID11 } from "crypto";
|
|
22280
|
-
import
|
|
22281
|
-
import
|
|
23630
|
+
import os5 from "os";
|
|
23631
|
+
import path21 from "path";
|
|
22282
23632
|
import {
|
|
22283
23633
|
createVerdictCache,
|
|
22284
23634
|
judgeFactDurability,
|
|
@@ -22388,8 +23738,8 @@ var extractionJudgeCalibrationDefinition = {
|
|
|
22388
23738
|
async function runExtractionJudgeCalibrationBenchmark(options) {
|
|
22389
23739
|
const cases = loadCases2(options.mode, options.limit);
|
|
22390
23740
|
const config = parseConfig2({
|
|
22391
|
-
memoryDir:
|
|
22392
|
-
workspaceDir:
|
|
23741
|
+
memoryDir: path21.join(os5.tmpdir(), "remnic-bench-extraction-judge"),
|
|
23742
|
+
workspaceDir: path21.join(os5.tmpdir(), "remnic-bench-extraction-judge-workspace"),
|
|
22393
23743
|
openaiApiKey: "bench-test-key",
|
|
22394
23744
|
extractionJudgeEnabled: true,
|
|
22395
23745
|
extractionJudgeBatchSize: 4,
|
|
@@ -22937,10 +24287,10 @@ function constantAggregate2(value) {
|
|
|
22937
24287
|
}
|
|
22938
24288
|
|
|
22939
24289
|
// src/benchmarks/remnic/entity-consolidation/runner.ts
|
|
22940
|
-
import
|
|
22941
|
-
import
|
|
24290
|
+
import os6 from "os";
|
|
24291
|
+
import path22 from "path";
|
|
22942
24292
|
import { randomUUID as randomUUID13 } from "crypto";
|
|
22943
|
-
import { mkdtemp as
|
|
24293
|
+
import { mkdtemp as mkdtemp4, rm as rm5 } from "fs/promises";
|
|
22944
24294
|
import { StorageManager } from "@remnic/core";
|
|
22945
24295
|
|
|
22946
24296
|
// src/benchmarks/remnic/entity-consolidation/fixture.ts
|
|
@@ -23101,7 +24451,7 @@ function loadCases4(mode, limit) {
|
|
|
23101
24451
|
return limited;
|
|
23102
24452
|
}
|
|
23103
24453
|
async function executeCase(sample) {
|
|
23104
|
-
const tmpDir = await
|
|
24454
|
+
const tmpDir = await mkdtemp4(path22.join(os6.tmpdir(), "remnic-bench-entity-consolidation-"));
|
|
23105
24455
|
try {
|
|
23106
24456
|
const storage = new StorageManager(tmpDir);
|
|
23107
24457
|
await storage.ensureDirectories();
|
|
@@ -23109,7 +24459,7 @@ async function executeCase(sample) {
|
|
|
23109
24459
|
const rawEntity = await storage.readEntity(canonicalName);
|
|
23110
24460
|
return summarizeEntity(rawEntity, canonicalName);
|
|
23111
24461
|
} finally {
|
|
23112
|
-
await
|
|
24462
|
+
await rm5(tmpDir, { recursive: true, force: true });
|
|
23113
24463
|
}
|
|
23114
24464
|
}
|
|
23115
24465
|
async function applyScenario(storage, sample) {
|
|
@@ -23280,9 +24630,9 @@ function parseNonNegativeInt(rawValue) {
|
|
|
23280
24630
|
|
|
23281
24631
|
// src/benchmarks/remnic/page-versioning/runner.ts
|
|
23282
24632
|
import { randomUUID as randomUUID14 } from "crypto";
|
|
23283
|
-
import { mkdir as mkdir9, mkdtemp as
|
|
23284
|
-
import
|
|
23285
|
-
import
|
|
24633
|
+
import { mkdir as mkdir9, mkdtemp as mkdtemp5, readFile as readFile18, rm as rm6, writeFile as writeFile9 } from "fs/promises";
|
|
24634
|
+
import os7 from "os";
|
|
24635
|
+
import path23 from "path";
|
|
23286
24636
|
import {
|
|
23287
24637
|
createVersion,
|
|
23288
24638
|
diffVersions,
|
|
@@ -23446,10 +24796,10 @@ function loadCases5(mode, limit) {
|
|
|
23446
24796
|
return limited;
|
|
23447
24797
|
}
|
|
23448
24798
|
async function executeCase2(sample, dependencies) {
|
|
23449
|
-
const tmpDir = await
|
|
24799
|
+
const tmpDir = await mkdtemp5(path23.join(os7.tmpdir(), "remnic-bench-page-versioning-"));
|
|
23450
24800
|
try {
|
|
23451
|
-
const factsDir =
|
|
23452
|
-
const pagePath =
|
|
24801
|
+
const factsDir = path23.join(tmpDir, "facts");
|
|
24802
|
+
const pagePath = path23.join(factsDir, `${sample.id}.md`);
|
|
23453
24803
|
await mkdir9(factsDir, { recursive: true });
|
|
23454
24804
|
const config = versioningConfig();
|
|
23455
24805
|
switch (sample.scenario) {
|
|
@@ -23530,7 +24880,7 @@ async function executeCase2(sample, dependencies) {
|
|
|
23530
24880
|
}
|
|
23531
24881
|
}
|
|
23532
24882
|
} finally {
|
|
23533
|
-
await
|
|
24883
|
+
await rm6(tmpDir, { recursive: true, force: true });
|
|
23534
24884
|
}
|
|
23535
24885
|
}
|
|
23536
24886
|
function isMissingPageVersionError(error, pagePath, versionId) {
|
|
@@ -25803,9 +27153,9 @@ function loadCases9(mode, limit) {
|
|
|
25803
27153
|
|
|
25804
27154
|
// src/benchmarks/remnic/procedural-recall/runner.ts
|
|
25805
27155
|
import { randomUUID as randomUUID21 } from "crypto";
|
|
25806
|
-
import { mkdtemp as
|
|
25807
|
-
import
|
|
25808
|
-
import
|
|
27156
|
+
import { mkdtemp as mkdtemp6, rm as rm7 } from "fs/promises";
|
|
27157
|
+
import os8 from "os";
|
|
27158
|
+
import path24 from "path";
|
|
25809
27159
|
import {
|
|
25810
27160
|
StorageManager as StorageManager2,
|
|
25811
27161
|
parseConfig as parseConfig3,
|
|
@@ -25935,7 +27285,7 @@ async function runProceduralRecallBenchmark(options) {
|
|
|
25935
27285
|
}
|
|
25936
27286
|
for (const sample of e2eCases) {
|
|
25937
27287
|
const startedAt = performance.now();
|
|
25938
|
-
const dir = await
|
|
27288
|
+
const dir = await mkdtemp6(path24.join(os8.tmpdir(), "remnic-bench-procedural-recall-"));
|
|
25939
27289
|
let section = null;
|
|
25940
27290
|
try {
|
|
25941
27291
|
const storage = new StorageManager2(dir);
|
|
@@ -25950,7 +27300,7 @@ ${body}`,
|
|
|
25950
27300
|
);
|
|
25951
27301
|
const config = parseConfig3({
|
|
25952
27302
|
memoryDir: dir,
|
|
25953
|
-
workspaceDir:
|
|
27303
|
+
workspaceDir: path24.join(dir, "ws"),
|
|
25954
27304
|
openaiApiKey: "bench-key",
|
|
25955
27305
|
procedural: {
|
|
25956
27306
|
enabled: sample.proceduralEnabled !== false,
|
|
@@ -25959,7 +27309,7 @@ ${body}`,
|
|
|
25959
27309
|
});
|
|
25960
27310
|
section = await buildProcedureRecallSection(storage, sample.prompt, config);
|
|
25961
27311
|
} finally {
|
|
25962
|
-
await
|
|
27312
|
+
await rm7(dir, { recursive: true, force: true });
|
|
25963
27313
|
}
|
|
25964
27314
|
const latencyMs = Math.round(performance.now() - startedAt);
|
|
25965
27315
|
const nonNull = section !== null && section.length > 0;
|
|
@@ -26020,9 +27370,9 @@ ${body}`,
|
|
|
26020
27370
|
|
|
26021
27371
|
// src/benchmarks/remnic/ingestion-entity-recall/runner.ts
|
|
26022
27372
|
import { randomUUID as randomUUID22 } from "crypto";
|
|
26023
|
-
import { mkdtemp as
|
|
27373
|
+
import { mkdtemp as mkdtemp7, writeFile as writeFile10, rm as rm8, mkdir as mkdir10, realpath as realpath5 } from "fs/promises";
|
|
26024
27374
|
import { tmpdir as tmpdir2 } from "os";
|
|
26025
|
-
import
|
|
27375
|
+
import path25 from "path";
|
|
26026
27376
|
|
|
26027
27377
|
// src/ingestion-scorer.ts
|
|
26028
27378
|
function normalize(value) {
|
|
@@ -26524,12 +27874,12 @@ async function runIngestionEntityRecallBenchmark(options) {
|
|
|
26524
27874
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
26525
27875
|
}
|
|
26526
27876
|
const fixture = emailFixture.generate();
|
|
26527
|
-
const fixtureDir = await
|
|
27877
|
+
const fixtureDir = await mkdtemp7(path25.join(tmpdir2(), "bench-email-"));
|
|
26528
27878
|
try {
|
|
26529
27879
|
await options.ingestionAdapter.reset();
|
|
26530
27880
|
for (const file of fixture.files) {
|
|
26531
|
-
const filePath =
|
|
26532
|
-
await mkdir10(
|
|
27881
|
+
const filePath = path25.join(fixtureDir, file.relativePath);
|
|
27882
|
+
await mkdir10(path25.dirname(filePath), { recursive: true });
|
|
26533
27883
|
await writeFile10(filePath, file.content, "utf8");
|
|
26534
27884
|
}
|
|
26535
27885
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -26611,7 +27961,7 @@ async function runIngestionEntityRecallBenchmark(options) {
|
|
|
26611
27961
|
];
|
|
26612
27962
|
return buildResult(options, tasks, durationMs);
|
|
26613
27963
|
} finally {
|
|
26614
|
-
await
|
|
27964
|
+
await rm8(fixtureDir, { recursive: true, force: true });
|
|
26615
27965
|
}
|
|
26616
27966
|
}
|
|
26617
27967
|
async function buildResult(options, tasks, totalLatencyMs) {
|
|
@@ -26657,9 +28007,9 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
26657
28007
|
|
|
26658
28008
|
// src/benchmarks/remnic/ingestion-schema-completeness/runner.ts
|
|
26659
28009
|
import { randomUUID as randomUUID23 } from "crypto";
|
|
26660
|
-
import { mkdtemp as
|
|
28010
|
+
import { mkdtemp as mkdtemp8, writeFile as writeFile11, rm as rm9, mkdir as mkdir11, realpath as realpath6 } from "fs/promises";
|
|
26661
28011
|
import { tmpdir as tmpdir3 } from "os";
|
|
26662
|
-
import
|
|
28012
|
+
import path26 from "path";
|
|
26663
28013
|
var ingestionSchemaCompletenessDefinition = {
|
|
26664
28014
|
id: "ingestion-schema-completeness",
|
|
26665
28015
|
title: "Ingestion: Schema Completeness",
|
|
@@ -26678,12 +28028,12 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
26678
28028
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
26679
28029
|
}
|
|
26680
28030
|
const fixture = emailFixture.generate();
|
|
26681
|
-
const fixtureDir = await
|
|
28031
|
+
const fixtureDir = await mkdtemp8(path26.join(tmpdir3(), "bench-email-"));
|
|
26682
28032
|
try {
|
|
26683
28033
|
await options.ingestionAdapter.reset();
|
|
26684
28034
|
for (const file of fixture.files) {
|
|
26685
|
-
const filePath =
|
|
26686
|
-
await mkdir11(
|
|
28035
|
+
const filePath = path26.join(fixtureDir, file.relativePath);
|
|
28036
|
+
await mkdir11(path26.dirname(filePath), { recursive: true });
|
|
26687
28037
|
await writeFile11(filePath, file.content, "utf8");
|
|
26688
28038
|
}
|
|
26689
28039
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -26824,15 +28174,15 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
26824
28174
|
}
|
|
26825
28175
|
};
|
|
26826
28176
|
} finally {
|
|
26827
|
-
await
|
|
28177
|
+
await rm9(fixtureDir, { recursive: true, force: true });
|
|
26828
28178
|
}
|
|
26829
28179
|
}
|
|
26830
28180
|
|
|
26831
28181
|
// src/benchmarks/remnic/ingestion-backlink-f1/runner.ts
|
|
26832
28182
|
import { randomUUID as randomUUID24 } from "crypto";
|
|
26833
|
-
import { mkdtemp as
|
|
28183
|
+
import { mkdtemp as mkdtemp9, writeFile as writeFile12, rm as rm10, mkdir as mkdir12, realpath as realpath7 } from "fs/promises";
|
|
26834
28184
|
import { tmpdir as tmpdir4 } from "os";
|
|
26835
|
-
import
|
|
28185
|
+
import path27 from "path";
|
|
26836
28186
|
var ingestionBacklinkF1Definition = {
|
|
26837
28187
|
id: "ingestion-backlink-f1",
|
|
26838
28188
|
title: "Ingestion: Backlink F1",
|
|
@@ -26851,12 +28201,12 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
26851
28201
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
26852
28202
|
}
|
|
26853
28203
|
const fixture = emailFixture.generate();
|
|
26854
|
-
const fixtureDir = await
|
|
28204
|
+
const fixtureDir = await mkdtemp9(path27.join(tmpdir4(), "bench-email-"));
|
|
26855
28205
|
try {
|
|
26856
28206
|
await options.ingestionAdapter.reset();
|
|
26857
28207
|
for (const file of fixture.files) {
|
|
26858
|
-
const filePath =
|
|
26859
|
-
await mkdir12(
|
|
28208
|
+
const filePath = path27.join(fixtureDir, file.relativePath);
|
|
28209
|
+
await mkdir12(path27.dirname(filePath), { recursive: true });
|
|
26860
28210
|
await writeFile12(filePath, file.content, "utf8");
|
|
26861
28211
|
}
|
|
26862
28212
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -26925,15 +28275,15 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
26925
28275
|
}
|
|
26926
28276
|
};
|
|
26927
28277
|
} finally {
|
|
26928
|
-
await
|
|
28278
|
+
await rm10(fixtureDir, { recursive: true, force: true });
|
|
26929
28279
|
}
|
|
26930
28280
|
}
|
|
26931
28281
|
|
|
26932
28282
|
// src/benchmarks/remnic/ingestion-setup-friction/runner.ts
|
|
26933
28283
|
import { randomUUID as randomUUID25 } from "crypto";
|
|
26934
|
-
import { mkdtemp as
|
|
28284
|
+
import { mkdtemp as mkdtemp10, writeFile as writeFile13, rm as rm11, mkdir as mkdir13, realpath as realpath8 } from "fs/promises";
|
|
26935
28285
|
import { tmpdir as tmpdir5 } from "os";
|
|
26936
|
-
import
|
|
28286
|
+
import path28 from "path";
|
|
26937
28287
|
var INGESTION_SETUP_FRICTION_LOWER_IS_BETTER = /* @__PURE__ */ new Set(["setup_friction", "commands_count", "prompts_count", "errors_count"]);
|
|
26938
28288
|
var ingestionSetupFrictionDefinition = {
|
|
26939
28289
|
id: "ingestion-setup-friction",
|
|
@@ -26953,12 +28303,12 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
26953
28303
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
26954
28304
|
}
|
|
26955
28305
|
const fixture = emailFixture.generate();
|
|
26956
|
-
const fixtureDir = await
|
|
28306
|
+
const fixtureDir = await mkdtemp10(path28.join(tmpdir5(), "bench-friction-"));
|
|
26957
28307
|
try {
|
|
26958
28308
|
await options.ingestionAdapter.reset();
|
|
26959
28309
|
for (const file of fixture.files) {
|
|
26960
|
-
const filePath =
|
|
26961
|
-
await mkdir13(
|
|
28310
|
+
const filePath = path28.join(fixtureDir, file.relativePath);
|
|
28311
|
+
await mkdir13(path28.dirname(filePath), { recursive: true });
|
|
26962
28312
|
await writeFile13(filePath, file.content, "utf8");
|
|
26963
28313
|
}
|
|
26964
28314
|
const { result: ingestionLog, durationMs } = await timed(
|
|
@@ -27031,15 +28381,15 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
27031
28381
|
}
|
|
27032
28382
|
};
|
|
27033
28383
|
} finally {
|
|
27034
|
-
await
|
|
28384
|
+
await rm11(fixtureDir, { recursive: true, force: true });
|
|
27035
28385
|
}
|
|
27036
28386
|
}
|
|
27037
28387
|
|
|
27038
28388
|
// src/benchmarks/remnic/ingestion-citation-accuracy/runner.ts
|
|
27039
28389
|
import { randomUUID as randomUUID26 } from "crypto";
|
|
27040
|
-
import { mkdtemp as
|
|
28390
|
+
import { mkdtemp as mkdtemp11, writeFile as writeFile14, rm as rm12, mkdir as mkdir14, realpath as realpath9 } from "fs/promises";
|
|
27041
28391
|
import { tmpdir as tmpdir6 } from "os";
|
|
27042
|
-
import
|
|
28392
|
+
import path29 from "path";
|
|
27043
28393
|
var CITATION_SUPPORT_THRESHOLD = 0.72;
|
|
27044
28394
|
var ingestionCitationAccuracyDefinition = {
|
|
27045
28395
|
id: "ingestion-citation-accuracy",
|
|
@@ -27098,10 +28448,10 @@ function resolveCitedSources(sourceRefs, seeAlso, pageRef, sourceContentMap) {
|
|
|
27098
28448
|
return "";
|
|
27099
28449
|
}
|
|
27100
28450
|
for (const ref of normalizedRefs) {
|
|
27101
|
-
const refBase =
|
|
28451
|
+
const refBase = path29.basename(ref).toLowerCase();
|
|
27102
28452
|
let matched = false;
|
|
27103
28453
|
for (const [relativePath, content] of sourceContentMap) {
|
|
27104
|
-
if (relativePath === ref || relativePath.endsWith(ref) ||
|
|
28454
|
+
if (relativePath === ref || relativePath.endsWith(ref) || path29.basename(relativePath).toLowerCase() === refBase) {
|
|
27105
28455
|
resolved.push(content);
|
|
27106
28456
|
matched = true;
|
|
27107
28457
|
break;
|
|
@@ -27117,9 +28467,9 @@ function resolveCitedSources(sourceRefs, seeAlso, pageRef, sourceContentMap) {
|
|
|
27117
28467
|
if (normalizedRefs.length > 0) {
|
|
27118
28468
|
return "";
|
|
27119
28469
|
}
|
|
27120
|
-
const pageBase =
|
|
28470
|
+
const pageBase = path29.basename(pageRef).toLowerCase();
|
|
27121
28471
|
for (const [relativePath, content] of sourceContentMap) {
|
|
27122
|
-
if (
|
|
28472
|
+
if (path29.basename(relativePath).toLowerCase() === pageBase) {
|
|
27123
28473
|
return content;
|
|
27124
28474
|
}
|
|
27125
28475
|
}
|
|
@@ -27130,12 +28480,12 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
27130
28480
|
throw new Error("ingestionAdapter is required for ingestion benchmarks");
|
|
27131
28481
|
}
|
|
27132
28482
|
const fixture = emailFixture.generate();
|
|
27133
|
-
const fixtureDir = await
|
|
28483
|
+
const fixtureDir = await mkdtemp11(path29.join(tmpdir6(), "bench-citation-"));
|
|
27134
28484
|
try {
|
|
27135
28485
|
await options.ingestionAdapter.reset();
|
|
27136
28486
|
for (const file of fixture.files) {
|
|
27137
|
-
const filePath =
|
|
27138
|
-
await mkdir14(
|
|
28487
|
+
const filePath = path29.join(fixtureDir, file.relativePath);
|
|
28488
|
+
await mkdir14(path29.dirname(filePath), { recursive: true });
|
|
27139
28489
|
await writeFile14(filePath, file.content, "utf8");
|
|
27140
28490
|
}
|
|
27141
28491
|
const benchmarkStart = performance.now();
|
|
@@ -27333,7 +28683,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
27333
28683
|
}
|
|
27334
28684
|
};
|
|
27335
28685
|
} finally {
|
|
27336
|
-
await
|
|
28686
|
+
await rm12(fixtureDir, { recursive: true, force: true });
|
|
27337
28687
|
}
|
|
27338
28688
|
}
|
|
27339
28689
|
function citationSupportScore(claim, citedSources) {
|
|
@@ -27521,7 +28871,7 @@ var ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS = ASSISTANT_MORNING_BRIEF_SCENARIOS.
|
|
|
27521
28871
|
|
|
27522
28872
|
// src/benchmarks/remnic/_assistant-common/runner.ts
|
|
27523
28873
|
import { randomUUID as randomUUID27 } from "crypto";
|
|
27524
|
-
import
|
|
28874
|
+
import path31 from "path";
|
|
27525
28875
|
|
|
27526
28876
|
// src/run-seeds.ts
|
|
27527
28877
|
function buildBenchmarkRunSeeds(runCount, baseSeed) {
|
|
@@ -27613,7 +28963,7 @@ function pairedDeltaConfidenceInterval(candidateValues, baselineValues, options
|
|
|
27613
28963
|
// src/judges/sealed-rubric.ts
|
|
27614
28964
|
import { createHash as createHash8 } from "crypto";
|
|
27615
28965
|
import { appendFileSync, mkdirSync } from "fs";
|
|
27616
|
-
import
|
|
28966
|
+
import path30 from "path";
|
|
27617
28967
|
|
|
27618
28968
|
// src/judges/sealed-prompts/assistant-rubric-v1.ts
|
|
27619
28969
|
var ASSISTANT_RUBRIC_V1 = `# Assistant rubric v1 (sealed)
|
|
@@ -27893,7 +29243,7 @@ function createSpotCheckFileLogger(options) {
|
|
|
27893
29243
|
return { log() {
|
|
27894
29244
|
} };
|
|
27895
29245
|
}
|
|
27896
|
-
const logPath =
|
|
29246
|
+
const logPath = path30.join(directory, `${runId}.jsonl`);
|
|
27897
29247
|
let written = 0;
|
|
27898
29248
|
let warnedOnWriteFailure = false;
|
|
27899
29249
|
const cap = typeof sampleSize === "number" && sampleSize > 0 ? sampleSize : 5;
|
|
@@ -27982,7 +29332,7 @@ async function runAssistantBenchmark(definition, scenarios, resolved, runnerOpti
|
|
|
27982
29332
|
const runId = buildRunId(definition.id);
|
|
27983
29333
|
const spotCheckLogger = createSpotCheckFileLogger({
|
|
27984
29334
|
runId,
|
|
27985
|
-
directory: runnerOptions.spotCheckDir ??
|
|
29335
|
+
directory: runnerOptions.spotCheckDir ?? path31.join(process.cwd(), "benchmarks", "results", "spot-checks"),
|
|
27986
29336
|
sampleRate: 0.35,
|
|
27987
29337
|
sampleSize: 5
|
|
27988
29338
|
});
|
|
@@ -28631,9 +29981,9 @@ async function runAssistantSynthesisBenchmark(options) {
|
|
|
28631
29981
|
|
|
28632
29982
|
// src/benchmarks/remnic/buffer-surprise-trigger/runner.ts
|
|
28633
29983
|
import { randomUUID as randomUUID28 } from "crypto";
|
|
28634
|
-
import
|
|
28635
|
-
import
|
|
28636
|
-
import { mkdir as mkdir15, rm as
|
|
29984
|
+
import path32 from "path";
|
|
29985
|
+
import os9 from "os";
|
|
29986
|
+
import { mkdir as mkdir15, rm as rm13 } from "fs/promises";
|
|
28637
29987
|
import {
|
|
28638
29988
|
SmartBuffer,
|
|
28639
29989
|
computeSurprise,
|
|
@@ -28862,8 +30212,8 @@ function hasExplicitTopicPivotCue(text) {
|
|
|
28862
30212
|
}
|
|
28863
30213
|
async function runBufferSurpriseTriggerBenchmark(options) {
|
|
28864
30214
|
const cases = loadCases10(options.mode, options.limit);
|
|
28865
|
-
const tmpRoot =
|
|
28866
|
-
|
|
30215
|
+
const tmpRoot = path32.join(
|
|
30216
|
+
os9.tmpdir(),
|
|
28867
30217
|
`remnic-bench-buffer-surprise-${randomUUID28()}`
|
|
28868
30218
|
);
|
|
28869
30219
|
await mkdir15(tmpRoot, { recursive: true });
|
|
@@ -28884,7 +30234,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
28884
30234
|
tasks.push(buildTaskResult(caseDef, control, candidate));
|
|
28885
30235
|
}
|
|
28886
30236
|
} finally {
|
|
28887
|
-
await
|
|
30237
|
+
await rm13(tmpRoot, { recursive: true, force: true });
|
|
28888
30238
|
}
|
|
28889
30239
|
const totalLatencyMs = Math.round(performance.now() - startedAt);
|
|
28890
30240
|
const aggregates = buildAggregates2(tasks);
|
|
@@ -28931,11 +30281,11 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
28931
30281
|
};
|
|
28932
30282
|
}
|
|
28933
30283
|
async function runSingleCase(caseDef, options) {
|
|
28934
|
-
const memoryDir =
|
|
30284
|
+
const memoryDir = path32.join(
|
|
28935
30285
|
options.tmpRoot,
|
|
28936
30286
|
`${caseDef.id}-${options.label}`
|
|
28937
30287
|
);
|
|
28938
|
-
const workspaceDir =
|
|
30288
|
+
const workspaceDir = path32.join(memoryDir, "workspace");
|
|
28939
30289
|
await mkdir15(workspaceDir, { recursive: true });
|
|
28940
30290
|
const config = parseConfig4({
|
|
28941
30291
|
memoryDir,
|
|
@@ -30978,7 +32328,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
30978
32328
|
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
30979
32329
|
import { randomUUID as randomUUID32 } from "crypto";
|
|
30980
32330
|
import { mkdir as mkdir16, writeFile as writeFile15 } from "fs/promises";
|
|
30981
|
-
import
|
|
32331
|
+
import path33 from "path";
|
|
30982
32332
|
|
|
30983
32333
|
// src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
|
|
30984
32334
|
import { createHash as createHash10 } from "crypto";
|
|
@@ -31463,17 +32813,7 @@ var BOUNDED_MEMORY_SMOKE_FIXTURE = [
|
|
|
31463
32813
|
];
|
|
31464
32814
|
function fixtureHash(tasks) {
|
|
31465
32815
|
const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
|
|
31466
|
-
const payload =
|
|
31467
|
-
(t) => [
|
|
31468
|
-
t.id,
|
|
31469
|
-
t.family,
|
|
31470
|
-
t.expectedAnswer,
|
|
31471
|
-
t.scope,
|
|
31472
|
-
t.shouldAsk === void 0 ? "-" : String(t.shouldAsk),
|
|
31473
|
-
t.memoryItems.map((m) => `${m.id}:${m.status}:${m.scope}`).join(","),
|
|
31474
|
-
t.skills.map((s) => s.id).join(",")
|
|
31475
|
-
].join("|")
|
|
31476
|
-
).join("\n");
|
|
32816
|
+
const payload = JSON.stringify(source);
|
|
31477
32817
|
return createHash10("sha256").update(payload, "utf8").digest("hex");
|
|
31478
32818
|
}
|
|
31479
32819
|
|
|
@@ -31869,7 +33209,7 @@ function aggregateCondition(condition, scored, skillLog) {
|
|
|
31869
33209
|
const tp = injected.filter((e) => e.outcome === "helped").length;
|
|
31870
33210
|
const fp = injected.filter((e) => e.outcome === "harmed").length;
|
|
31871
33211
|
const notInjected = considered.filter((e) => !e.injected);
|
|
31872
|
-
const fn =
|
|
33212
|
+
const fn = skillLog.filter((e) => !e.injected && e.outcome === "harmed").length;
|
|
31873
33213
|
const tn = notInjected.filter((e) => e.outcome === "irrelevant").length;
|
|
31874
33214
|
return {
|
|
31875
33215
|
condition,
|
|
@@ -32220,23 +33560,23 @@ async function runBoundedMemoryContractsBenchmark(options) {
|
|
|
32220
33560
|
};
|
|
32221
33561
|
}
|
|
32222
33562
|
async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks) {
|
|
32223
|
-
const root =
|
|
32224
|
-
await mkdir16(
|
|
32225
|
-
await mkdir16(
|
|
32226
|
-
await mkdir16(
|
|
32227
|
-
await mkdir16(
|
|
33563
|
+
const root = path33.resolve(outputDir);
|
|
33564
|
+
await mkdir16(path33.join(root, "conditions"), { recursive: true });
|
|
33565
|
+
await mkdir16(path33.join(root, "prompts"), { recursive: true });
|
|
33566
|
+
await mkdir16(path33.join(root, "retrieval"), { recursive: true });
|
|
33567
|
+
await mkdir16(path33.join(root, "scores"), { recursive: true });
|
|
32228
33568
|
const csvRows = [
|
|
32229
33569
|
"task_id,condition,family,scope,task_success,should_ask_accuracy,relevant_memory_recall,stale_memory_harm_rate,wrong_scope_retrieval_rate,supersession_respected_rate,citation_coverage,memory_tokens_injected,retrieved_item_count,compression_ratio_vs_raw_transcript"
|
|
32230
33570
|
];
|
|
32231
33571
|
for (const condition of BOUNDED_MEMORY_CONDITIONS) {
|
|
32232
33572
|
const results = byCondition.get(condition);
|
|
32233
|
-
const condDir =
|
|
33573
|
+
const condDir = path33.join(root, "conditions", condition);
|
|
32234
33574
|
await mkdir16(condDir, { recursive: true });
|
|
32235
33575
|
for (const { task, pack, decision } of results) {
|
|
32236
33576
|
const scores = scoreTaskPair(task, pack, decision);
|
|
32237
33577
|
const promptMd = renderPromptPack(task, condition, pack);
|
|
32238
|
-
const promptPath =
|
|
32239
|
-
await mkdir16(
|
|
33578
|
+
const promptPath = path33.join(root, "prompts", `${task.id}.${condition}.md`);
|
|
33579
|
+
await mkdir16(path33.dirname(promptPath), { recursive: true });
|
|
32240
33580
|
await writeFile15(promptPath, promptMd, "utf8");
|
|
32241
33581
|
const retrievalJson = `${JSON.stringify(
|
|
32242
33582
|
{
|
|
@@ -32260,7 +33600,7 @@ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks
|
|
|
32260
33600
|
2
|
|
32261
33601
|
)}
|
|
32262
33602
|
`;
|
|
32263
|
-
const retrievalPath =
|
|
33603
|
+
const retrievalPath = path33.join(root, "retrieval", `${task.id}.${condition}.json`);
|
|
32264
33604
|
await writeFile15(retrievalPath, retrievalJson, "utf8");
|
|
32265
33605
|
csvRows.push(
|
|
32266
33606
|
[
|
|
@@ -32282,23 +33622,23 @@ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks
|
|
|
32282
33622
|
);
|
|
32283
33623
|
}
|
|
32284
33624
|
await writeFile15(
|
|
32285
|
-
|
|
33625
|
+
path33.join(condDir, "summary.json"),
|
|
32286
33626
|
`${JSON.stringify(conditionAggregates[condition], null, 2)}
|
|
32287
33627
|
`,
|
|
32288
33628
|
"utf8"
|
|
32289
33629
|
);
|
|
32290
33630
|
}
|
|
32291
|
-
await writeFile15(
|
|
33631
|
+
await writeFile15(path33.join(root, "scores", "per-task.csv"), `${csvRows.join("\n")}
|
|
32292
33632
|
`, "utf8");
|
|
32293
33633
|
await writeFile15(
|
|
32294
|
-
|
|
33634
|
+
path33.join(root, "scores", "aggregate.json"),
|
|
32295
33635
|
`${JSON.stringify(conditionAggregates, null, 2)}
|
|
32296
33636
|
`,
|
|
32297
33637
|
"utf8"
|
|
32298
33638
|
);
|
|
32299
33639
|
const report = renderReportMarkdown(tasks, conditionAggregates);
|
|
32300
|
-
await writeFile15(
|
|
32301
|
-
return
|
|
33640
|
+
await writeFile15(path33.join(root, "report.md"), report, "utf8");
|
|
33641
|
+
return path33.join(root, "report.md");
|
|
32302
33642
|
}
|
|
32303
33643
|
function renderPromptPack(task, condition, pack) {
|
|
32304
33644
|
const lines = [];
|
|
@@ -32523,8 +33863,8 @@ function finalizeBenchmarkResultConfig(result, options) {
|
|
|
32523
33863
|
}
|
|
32524
33864
|
|
|
32525
33865
|
// src/benchmark.ts
|
|
32526
|
-
var DEFAULT_BASELINE_PATH =
|
|
32527
|
-
var DEFAULT_REPORT_PATH =
|
|
33866
|
+
var DEFAULT_BASELINE_PATH = path34.join(process.cwd(), "benchmarks", "baseline.json");
|
|
33867
|
+
var DEFAULT_REPORT_PATH = path34.join(process.cwd(), "benchmarks", "report.json");
|
|
32528
33868
|
var BASELINE_VERSION = 1;
|
|
32529
33869
|
var DEFAULT_TOLERANCE = 10;
|
|
32530
33870
|
var DEFAULT_FULL_RUN_COUNT = 5;
|
|
@@ -32602,7 +33942,7 @@ async function runBenchmark(benchmarkId, options) {
|
|
|
32602
33942
|
if (!willWrapPrimary && !willWrapCross) {
|
|
32603
33943
|
return void 0;
|
|
32604
33944
|
}
|
|
32605
|
-
const cacheDir = options.judgeCacheDir ?
|
|
33945
|
+
const cacheDir = options.judgeCacheDir ? path34.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path34.join(path34.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
|
|
32606
33946
|
if (cacheDir === void 0) {
|
|
32607
33947
|
return void 0;
|
|
32608
33948
|
}
|
|
@@ -32791,7 +34131,7 @@ function loadBaseline(baselinePath) {
|
|
|
32791
34131
|
return raw;
|
|
32792
34132
|
}
|
|
32793
34133
|
function saveBaseline(baselinePath, baseline) {
|
|
32794
|
-
fs2.mkdirSync(
|
|
34134
|
+
fs2.mkdirSync(path34.dirname(baselinePath), { recursive: true });
|
|
32795
34135
|
fs2.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}
|
|
32796
34136
|
`);
|
|
32797
34137
|
}
|
|
@@ -33021,7 +34361,7 @@ function generateReport(results, reportPath) {
|
|
|
33021
34361
|
totalDurationMs: results.reduce((sum, result) => sum + result.totalDurationMs, 0)
|
|
33022
34362
|
};
|
|
33023
34363
|
if (reportPath) {
|
|
33024
|
-
fs2.mkdirSync(
|
|
34364
|
+
fs2.mkdirSync(path34.dirname(reportPath), { recursive: true });
|
|
33025
34365
|
fs2.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}
|
|
33026
34366
|
`);
|
|
33027
34367
|
}
|
|
@@ -33487,7 +34827,7 @@ function formatError(error) {
|
|
|
33487
34827
|
|
|
33488
34828
|
// src/benchmarks/custom/runner.ts
|
|
33489
34829
|
import { randomUUID as randomUUID33 } from "crypto";
|
|
33490
|
-
import
|
|
34830
|
+
import path35 from "path";
|
|
33491
34831
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
33492
34832
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
33493
34833
|
const spec = await loadCustomBenchmarkFile(filePath);
|
|
@@ -33500,7 +34840,7 @@ async function runCustomBenchmarkFile(filePath, options) {
|
|
|
33500
34840
|
let cacheRestore;
|
|
33501
34841
|
let cacheCounters;
|
|
33502
34842
|
if (spec.scoring === "llm_judge" && runOptions.system.judge !== void 0 && !runOptions.noJudgeCache && (runOptions.judgeProvider ?? null) !== null) {
|
|
33503
|
-
const cacheDir = runOptions.judgeCacheDir ?
|
|
34843
|
+
const cacheDir = runOptions.judgeCacheDir ? path35.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path35.join(path35.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
|
|
33504
34844
|
if (cacheDir !== void 0) {
|
|
33505
34845
|
const originalJudge = runOptions.system.judge;
|
|
33506
34846
|
const wrapped = wrapJudgeWithCache({
|
|
@@ -33701,7 +35041,7 @@ async function scoreTask(scoring, options, question, actual, expected) {
|
|
|
33701
35041
|
}
|
|
33702
35042
|
}
|
|
33703
35043
|
function createCustomBenchmarkDefinition(benchmark, filePath) {
|
|
33704
|
-
const id = `custom:${slugify(
|
|
35044
|
+
const id = `custom:${slugify(path35.basename(filePath, path35.extname(filePath)) || benchmark.name)}`;
|
|
33705
35045
|
return {
|
|
33706
35046
|
id,
|
|
33707
35047
|
title: benchmark.name,
|
|
@@ -34573,7 +35913,7 @@ var chatFixture = {
|
|
|
34573
35913
|
// src/judges/calibration-slice.ts
|
|
34574
35914
|
import { createHash as createHash12, randomBytes as randomBytes3 } from "crypto";
|
|
34575
35915
|
import { mkdir as mkdir17, readFile as readFile21, rename as rename3, unlink as unlink3, writeFile as writeFile16 } from "fs/promises";
|
|
34576
|
-
import
|
|
35916
|
+
import path36 from "path";
|
|
34577
35917
|
|
|
34578
35918
|
// src/judges/cohen-kappa.ts
|
|
34579
35919
|
function computeCohensKappa(raterA, raterB) {
|
|
@@ -34710,7 +36050,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
|
|
|
34710
36050
|
warning: result.warning,
|
|
34711
36051
|
...identities ? identities : {}
|
|
34712
36052
|
};
|
|
34713
|
-
const filePath =
|
|
36053
|
+
const filePath = path36.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
|
|
34714
36054
|
const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
|
|
34715
36055
|
await writeFile16(tempPath, `${JSON.stringify(state, null, 2)}
|
|
34716
36056
|
`, "utf8");
|
|
@@ -34723,7 +36063,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
|
|
|
34723
36063
|
return filePath;
|
|
34724
36064
|
}
|
|
34725
36065
|
async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
|
|
34726
|
-
const filePath =
|
|
36066
|
+
const filePath = path36.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
|
|
34727
36067
|
let raw;
|
|
34728
36068
|
try {
|
|
34729
36069
|
raw = await readFile21(filePath, "utf8");
|
|
@@ -34797,9 +36137,9 @@ function sanitizeCalibrationSegment(value) {
|
|
|
34797
36137
|
}
|
|
34798
36138
|
|
|
34799
36139
|
// src/benchmarks/remnic/procedural-recall/ablation.ts
|
|
34800
|
-
import { mkdir as mkdir18, mkdtemp as
|
|
34801
|
-
import
|
|
34802
|
-
import
|
|
36140
|
+
import { mkdir as mkdir18, mkdtemp as mkdtemp12, rm as rm14, writeFile as writeFile17, readFile as readFile22 } from "fs/promises";
|
|
36141
|
+
import os10 from "os";
|
|
36142
|
+
import path37 from "path";
|
|
34803
36143
|
import {
|
|
34804
36144
|
StorageManager as StorageManager3,
|
|
34805
36145
|
parseConfig as parseConfig5,
|
|
@@ -34829,8 +36169,8 @@ function fixtureToAblationScenarios(fixture) {
|
|
|
34829
36169
|
async function runSide(scenarios, proceduralEnabled) {
|
|
34830
36170
|
const observed = [];
|
|
34831
36171
|
for (const scenario of scenarios) {
|
|
34832
|
-
const dir = await
|
|
34833
|
-
|
|
36172
|
+
const dir = await mkdtemp12(
|
|
36173
|
+
path37.join(os10.tmpdir(), "remnic-bench-proc-ablation-")
|
|
34834
36174
|
);
|
|
34835
36175
|
try {
|
|
34836
36176
|
const storage = new StorageManager3(dir);
|
|
@@ -34845,7 +36185,7 @@ ${body}`,
|
|
|
34845
36185
|
);
|
|
34846
36186
|
const config = parseConfig5({
|
|
34847
36187
|
memoryDir: dir,
|
|
34848
|
-
workspaceDir:
|
|
36188
|
+
workspaceDir: path37.join(dir, "ws"),
|
|
34849
36189
|
openaiApiKey: "bench-key",
|
|
34850
36190
|
procedural: {
|
|
34851
36191
|
enabled: proceduralEnabled,
|
|
@@ -34859,7 +36199,7 @@ ${body}`,
|
|
|
34859
36199
|
);
|
|
34860
36200
|
observed.push(section !== null && section.length > 0);
|
|
34861
36201
|
} finally {
|
|
34862
|
-
await
|
|
36202
|
+
await rm14(dir, { recursive: true, force: true });
|
|
34863
36203
|
}
|
|
34864
36204
|
}
|
|
34865
36205
|
return observed;
|
|
@@ -35016,12 +36356,61 @@ async function runProceduralAblationCli(args) {
|
|
|
35016
36356
|
random: args.random,
|
|
35017
36357
|
seed: args.seed
|
|
35018
36358
|
});
|
|
35019
|
-
const outDir =
|
|
36359
|
+
const outDir = path37.dirname(path37.resolve(args.outPath));
|
|
35020
36360
|
await mkdir18(outDir, { recursive: true });
|
|
35021
36361
|
await writeFile17(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
|
|
35022
36362
|
return artifact;
|
|
35023
36363
|
}
|
|
35024
36364
|
|
|
36365
|
+
// src/ablations/single-flag-matrix.ts
|
|
36366
|
+
var SINGLE_FLAG_ABLATION_MATRIX = [
|
|
36367
|
+
{
|
|
36368
|
+
id: "memory-worth-off",
|
|
36369
|
+
label: "Memory Worth multiplier OFF",
|
|
36370
|
+
axis: "memory-worth",
|
|
36371
|
+
description: "Disables the Memory Worth recall multiplier (recallMemoryWorthFilterEnabled=false). Baseline (#1574) ran with it ON via the config.ts default; this cell measures the cost of removing it.",
|
|
36372
|
+
baselineState: "recallMemoryWorthFilterEnabled defaults true in config.ts; baseline ran ON.",
|
|
36373
|
+
configOverrides: {
|
|
36374
|
+
recallMemoryWorthFilterEnabled: false
|
|
36375
|
+
},
|
|
36376
|
+
primaryFlag: "recallMemoryWorthFilterEnabled"
|
|
36377
|
+
},
|
|
36378
|
+
{
|
|
36379
|
+
id: "contradiction-scan-on",
|
|
36380
|
+
label: "Contradiction scan ON",
|
|
36381
|
+
axis: "contradiction-scan",
|
|
36382
|
+
description: "Enables inline contradiction detection on the write path (contradictionDetectionEnabled=true) so supersessions land at ingest time, before answering. The batch contradictionScan cron is a no-op during a bench replay (it only registers a scheduled job), so the measurable axis is the inline write-path gate (orchestrator \xA715667); contradictionAutoResolve defaults true so detected contradictions are applied. Baseline (#1574) ran with it OFF.",
|
|
36383
|
+
baselineState: "contradictionDetectionEnabled defaults false (config.ts \xA72078); baseline ran OFF.",
|
|
36384
|
+
configOverrides: {
|
|
36385
|
+
contradictionDetectionEnabled: true
|
|
36386
|
+
},
|
|
36387
|
+
primaryFlag: "contradictionDetectionEnabled"
|
|
36388
|
+
},
|
|
36389
|
+
{
|
|
36390
|
+
id: "graph-recall-on",
|
|
36391
|
+
label: "Graph / temporal recall ON",
|
|
36392
|
+
axis: "graph-recall",
|
|
36393
|
+
description: "Enables graph recall + full-mode graph assist (graphRecallEnabled=true, graphAssistInFullModeEnabled=true, multiGraphMemoryEnabled=true). orchestrator.ts \xA71379 skips graph_mode unless BOTH graphRecallEnabled AND multiGraphMemoryEnabled are set, so the cell carries both gates (plus the full-mode assist) or it measures nothing. Baseline (#1574) ran with graph recall OFF; this cell measures the benefit of causal/timeline expansion.",
|
|
36394
|
+
baselineState: "graphRecallEnabled, graphAssistInFullModeEnabled, multiGraphMemoryEnabled all default false; baseline ran OFF.",
|
|
36395
|
+
configOverrides: {
|
|
36396
|
+
graphRecallEnabled: true,
|
|
36397
|
+
graphAssistInFullModeEnabled: true,
|
|
36398
|
+
multiGraphMemoryEnabled: true
|
|
36399
|
+
},
|
|
36400
|
+
primaryFlag: "graphRecallEnabled"
|
|
36401
|
+
}
|
|
36402
|
+
];
|
|
36403
|
+
var DEFAULT_ABLATION_BENCHMARK = "locomo";
|
|
36404
|
+
function getAblationCell(id) {
|
|
36405
|
+
const cell = SINGLE_FLAG_ABLATION_MATRIX.find((c) => c.id === id);
|
|
36406
|
+
if (!cell) {
|
|
36407
|
+
throw new Error(
|
|
36408
|
+
`Unknown ablation cell id "${id}". Known ids: ${SINGLE_FLAG_ABLATION_MATRIX.map((c) => c.id).join(", ")}.`
|
|
36409
|
+
);
|
|
36410
|
+
}
|
|
36411
|
+
return cell;
|
|
36412
|
+
}
|
|
36413
|
+
|
|
35025
36414
|
// src/benchmarks/remnic/procedural-recall/real-scenarios.ts
|
|
35026
36415
|
var PROCEDURAL_REAL_SCENARIOS = [
|
|
35027
36416
|
// --- exact-re-run (5) ---------------------------------------------------
|
|
@@ -36207,11 +37596,11 @@ function pickStableQualifiedName(repo, index) {
|
|
|
36207
37596
|
|
|
36208
37597
|
// src/coding-graph/harness.ts
|
|
36209
37598
|
import { performance as performance2 } from "perf_hooks";
|
|
36210
|
-
import { mkdtemp as
|
|
37599
|
+
import { mkdtemp as mkdtemp13, rm as rm15 } from "fs/promises";
|
|
36211
37600
|
import { statSync } from "fs";
|
|
36212
37601
|
import { tmpdir as tmpdir7 } from "os";
|
|
36213
|
-
import
|
|
36214
|
-
import
|
|
37602
|
+
import path38 from "path";
|
|
37603
|
+
import os11 from "os";
|
|
36215
37604
|
import {
|
|
36216
37605
|
GraphStore
|
|
36217
37606
|
} from "@remnic/coding-graph";
|
|
@@ -36237,14 +37626,14 @@ var CODING_GRAPH_BENCH_SCHEMA_VERSION = 2;
|
|
|
36237
37626
|
|
|
36238
37627
|
// src/coding-graph/harness.ts
|
|
36239
37628
|
function captureMachineFingerprint() {
|
|
36240
|
-
const cpus =
|
|
37629
|
+
const cpus = os11.cpus();
|
|
36241
37630
|
return {
|
|
36242
37631
|
arch: process.arch,
|
|
36243
37632
|
platform: process.platform,
|
|
36244
37633
|
nodeVersion: process.version,
|
|
36245
37634
|
cpuModel: cpus.length > 0 ? cpus[0].model : null,
|
|
36246
37635
|
cpuCores: cpus.length,
|
|
36247
|
-
totalMemoryMb: Math.round(
|
|
37636
|
+
totalMemoryMb: Math.round(os11.totalmem() / (1024 * 1024))
|
|
36248
37637
|
};
|
|
36249
37638
|
}
|
|
36250
37639
|
function percentile2(sorted, p) {
|
|
@@ -36307,8 +37696,8 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
36307
37696
|
const sampleRss = () => {
|
|
36308
37697
|
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
|
36309
37698
|
};
|
|
36310
|
-
const dir = await
|
|
36311
|
-
const dbPath =
|
|
37699
|
+
const dir = await mkdtemp13(path38.join(tmpdir7(), "coding-graph-bench-"));
|
|
37700
|
+
const dbPath = path38.join(dir, "bench.sqlite");
|
|
36312
37701
|
try {
|
|
36313
37702
|
const store = await GraphStore.open({ dbPath });
|
|
36314
37703
|
try {
|
|
@@ -36444,7 +37833,7 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
36444
37833
|
await store.close();
|
|
36445
37834
|
}
|
|
36446
37835
|
} finally {
|
|
36447
|
-
await
|
|
37836
|
+
await rm15(dir, { recursive: true, force: true });
|
|
36448
37837
|
}
|
|
36449
37838
|
}
|
|
36450
37839
|
|
|
@@ -36652,6 +38041,7 @@ export {
|
|
|
36652
38041
|
DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE,
|
|
36653
38042
|
MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS,
|
|
36654
38043
|
DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE,
|
|
38044
|
+
DEFAULT_ABLATION_BENCHMARK,
|
|
36655
38045
|
DEFAULT_ABLATION_BOOTSTRAP_SEED,
|
|
36656
38046
|
DEFAULT_ASSISTANT_RUBRIC_ID,
|
|
36657
38047
|
DEFAULT_BASELINE_SCENARIOS,
|
|
@@ -36664,11 +38054,14 @@ export {
|
|
|
36664
38054
|
LOCAL_LAB_PROVIDER_KINDS,
|
|
36665
38055
|
LOCOMO_DATASET_FILENAMES,
|
|
36666
38056
|
LONG_MEM_EVAL_DATASET_FILENAMES,
|
|
38057
|
+
LettaMemCorrectAdapter,
|
|
36667
38058
|
LocalLabPreflightError,
|
|
36668
38059
|
MEMORY_EVAL_DIMENSIONS,
|
|
36669
38060
|
MEMORY_EVAL_PUBLIC_LINE,
|
|
36670
38061
|
MIN_CALIBRATION_SOURCE_TASKS,
|
|
36671
38062
|
MITIGATED_BASELINE_SCENARIOS,
|
|
38063
|
+
Mem0MemCorrectAdapter,
|
|
38064
|
+
MissingCredentialError,
|
|
36672
38065
|
OTHER_NAMESPACE_MEMORIES,
|
|
36673
38066
|
PROCEDURAL_REAL_SCENARIOS,
|
|
36674
38067
|
PROCEDURAL_REAL_SCENARIOS_SMOKE,
|
|
@@ -36677,7 +38070,9 @@ export {
|
|
|
36677
38070
|
SCHEMA_TIER_FIXTURE,
|
|
36678
38071
|
SCHEMA_TIER_SMOKE_FIXTURE,
|
|
36679
38072
|
SEALED_PROMPT_REGISTRY,
|
|
38073
|
+
SINGLE_FLAG_ABLATION_MATRIX,
|
|
36680
38074
|
SYNTHETIC_MEMORIES,
|
|
38075
|
+
ZepMemCorrectAdapter,
|
|
36681
38076
|
addContaminationEntry,
|
|
36682
38077
|
aggregateTaskScores,
|
|
36683
38078
|
answerBenchmarkQuestion,
|
|
@@ -36722,6 +38117,7 @@ export {
|
|
|
36722
38117
|
createAmaBenchDiagnosticAdapter,
|
|
36723
38118
|
createAnthropicProvider,
|
|
36724
38119
|
createCanaryAdapter,
|
|
38120
|
+
createClaudeCliProvider,
|
|
36725
38121
|
createCodexCliProvider,
|
|
36726
38122
|
createSeededRng3 as createCodingGraphSeededRng,
|
|
36727
38123
|
createDeterministicSpotCheckLogger,
|
|
@@ -36762,6 +38158,7 @@ export {
|
|
|
36762
38158
|
formatMissingDatasetError,
|
|
36763
38159
|
generateReport,
|
|
36764
38160
|
generateSyntheticRepo,
|
|
38161
|
+
getAblationCell,
|
|
36765
38162
|
getBenchmark,
|
|
36766
38163
|
getBenchmarkLowerIsBetter,
|
|
36767
38164
|
getMemoryEvalDimension,
|