@remnic/bench 9.3.729 → 9.3.731

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.d.ts +403 -2
  2. package/dist/index.js +1702 -305
  3. package/package.json +3 -3
package/dist/index.js CHANGED
@@ -1,3 +1,598 @@
1
+ // src/benchmarks/remnic/memcorrect/third-party/shared.ts
2
+ var MissingCredentialError = class extends Error {
3
+ reason;
4
+ constructor(system, missing) {
5
+ const reason = `${system} MemCorrect adapter is not runnable: missing ${missing.join(", ")}. Provide the required credentials (env vars or config) to exercise this adapter; skipped \u2014 no keys in CI.`;
6
+ super(reason);
7
+ this.name = "MissingCredentialError";
8
+ this.reason = reason;
9
+ }
10
+ };
11
+ function requireCredentials(system, missing) {
12
+ if (missing.length > 0) {
13
+ throw new MissingCredentialError(system, missing);
14
+ }
15
+ }
16
+ function resolveFetch(override) {
17
+ if (override) return override;
18
+ if (typeof fetch === "function") return fetch;
19
+ throw new Error(
20
+ "No fetch implementation available. Provide `fetch` in the adapter config or run in a runtime with a global fetch (Node 18+)."
21
+ );
22
+ }
23
+ async function httpJson(fetchImpl, method, url, options = {}) {
24
+ const { headers = {}, body, timeoutMs } = options;
25
+ const controller = new AbortController();
26
+ const { promise: timeoutPromise, resolve: resolveTimeout } = Promise.withResolvers();
27
+ const timer = timeoutMs !== void 0 ? setTimeout(() => resolveTimeout(), timeoutMs) : void 0;
28
+ if (timer) timer.unref?.();
29
+ const fetchPromise = fetchImpl(url, {
30
+ method,
31
+ headers: { "Content-Type": "application/json", ...headers },
32
+ body: body !== void 0 ? JSON.stringify(body) : void 0,
33
+ signal: controller.signal
34
+ });
35
+ fetchPromise.catch(() => {
36
+ });
37
+ let response;
38
+ try {
39
+ if (timer) {
40
+ const winner = await Promise.race([
41
+ fetchPromise.then((r) => r),
42
+ timeoutPromise.then(() => "timeout")
43
+ ]);
44
+ if (winner === "timeout") {
45
+ controller.abort();
46
+ throw new Error(`Request timed out after ${timeoutMs}ms: ${method} ${url}`);
47
+ }
48
+ response = winner;
49
+ } else {
50
+ response = await fetchPromise;
51
+ }
52
+ } finally {
53
+ if (timer) clearTimeout(timer);
54
+ }
55
+ if (response.status === 204) return null;
56
+ const text = await response.text();
57
+ if (!response.ok) {
58
+ const excerpt = text.length > 500 ? `${text.slice(0, 500)}\u2026` : text;
59
+ throw new HttpError(method, url, response.status, excerpt);
60
+ }
61
+ if (text.length === 0) return null;
62
+ try {
63
+ return JSON.parse(text);
64
+ } catch {
65
+ return text;
66
+ }
67
+ }
68
+ var HttpError = class extends Error {
69
+ status;
70
+ method;
71
+ url;
72
+ bodyExcerpt;
73
+ constructor(method, url, status, bodyExcerpt) {
74
+ super(`HTTP ${status} from ${method} ${url}: ${bodyExcerpt}`);
75
+ this.name = "HttpError";
76
+ this.method = method;
77
+ this.url = url;
78
+ this.status = status;
79
+ this.bodyExcerpt = bodyExcerpt;
80
+ }
81
+ };
82
+ function delay(ms) {
83
+ if (ms <= 0) return Promise.resolve();
84
+ const { promise, resolve } = Promise.withResolvers();
85
+ setTimeout(() => resolve(), ms);
86
+ return promise;
87
+ }
88
+ var runSuffixCounter = 0;
89
+ function uniqueRunSuffix() {
90
+ runSuffixCounter += 1;
91
+ const pid = typeof process !== "undefined" && typeof process.pid === "number" ? process.pid : 0;
92
+ return `${pid.toString(36)}-${Date.now().toString(36)}-${runSuffixCounter.toString(36)}`;
93
+ }
94
+ function isNotFoundDelete(err) {
95
+ return err instanceof HttpError && (err.status === 404 || err.status === 422);
96
+ }
97
+ function isConflict(err) {
98
+ return err instanceof HttpError && err.status === 409;
99
+ }
100
+ async function resetTrackedIds(system, ids, deleteFn) {
101
+ const snapshot = [...ids];
102
+ const failed = [];
103
+ for (const id of snapshot) {
104
+ try {
105
+ await deleteFn(id);
106
+ ids.delete(id);
107
+ } catch (err) {
108
+ if (isNotFoundDelete(err)) {
109
+ ids.delete(id);
110
+ } else {
111
+ failed.push(id);
112
+ }
113
+ }
114
+ }
115
+ if (failed.length > 0) {
116
+ throw new Error(
117
+ `${system} reset could not clean ${failed.length} session(s): ${failed.join(", ")}. Remote data may remain; the failed ids are retained for the next reset() retry.`
118
+ );
119
+ }
120
+ }
121
+
122
+ // src/benchmarks/remnic/memcorrect/third-party/mem0-adapter.ts
123
+ var Mem0MemCorrectAdapter = class {
124
+ label;
125
+ mode;
126
+ baseUrl;
127
+ apiKey;
128
+ userIdPrefix;
129
+ ossAuthMode;
130
+ pollIntervalMs;
131
+ maxPolls;
132
+ fetchImpl;
133
+ timeoutMs;
134
+ /** Session-scoped user_ids we have ingested under, for precise reset. */
135
+ knownSessions = /* @__PURE__ */ new Set();
136
+ constructor(config = {}) {
137
+ this.mode = config.mode ?? (config.baseUrl && !config.baseUrl.includes("api.mem0.ai") ? "oss" : "hosted");
138
+ this.baseUrl = config.baseUrl?.replace(/\/+$/, "") ?? (this.mode === "hosted" ? "https://api.mem0.ai" : "");
139
+ this.apiKey = config.apiKey;
140
+ this.userIdPrefix = config.userIdPrefix ?? `memcorrect-${uniqueRunSuffix()}`;
141
+ this.ossAuthMode = config.ossAuthMode ?? "x-api-key";
142
+ this.pollIntervalMs = config.pollIntervalMs ?? 500;
143
+ this.maxPolls = config.maxPolls ?? 120;
144
+ this.fetchImpl = resolveFetch(config.fetch);
145
+ this.timeoutMs = config.timeoutMs;
146
+ this.label = `mem0-${this.mode}`;
147
+ }
148
+ /** Whether this adapter has the credentials needed to run. */
149
+ isConfigured() {
150
+ if (!this.apiKey) return false;
151
+ if (this.mode === "oss" && !this.baseUrl) return false;
152
+ return true;
153
+ }
154
+ userIdFor(sessionKey) {
155
+ return `${this.userIdPrefix}:${sessionKey}`;
156
+ }
157
+ ensureReady() {
158
+ const missing = [];
159
+ if (!this.apiKey) missing.push("apiKey (MEM0_API_KEY)");
160
+ if (this.mode === "oss" && !this.baseUrl)
161
+ missing.push("baseUrl (MEM0_BASE_URL)");
162
+ requireCredentials("Mem0", missing);
163
+ }
164
+ authHeaders() {
165
+ if (this.mode === "hosted") {
166
+ return { Authorization: `Token ${this.apiKey}` };
167
+ }
168
+ if (this.ossAuthMode === "bearer") {
169
+ return { Authorization: `Bearer ${this.apiKey}` };
170
+ }
171
+ return { "X-API-Key": this.apiKey };
172
+ }
173
+ async reset() {
174
+ this.ensureReady();
175
+ await resetTrackedIds("Mem0", this.knownSessions, async (sessionKey) => {
176
+ const userId = this.userIdFor(sessionKey);
177
+ if (this.mode === "oss") {
178
+ await httpJson(
179
+ this.fetchImpl,
180
+ "DELETE",
181
+ `${this.baseUrl}/memories?user_id=${encodeURIComponent(userId)}`,
182
+ { headers: this.authHeaders(), timeoutMs: this.timeoutMs }
183
+ );
184
+ } else {
185
+ await httpJson(
186
+ this.fetchImpl,
187
+ "DELETE",
188
+ `${this.baseUrl}/v1/memories/?user_id=${encodeURIComponent(userId)}`,
189
+ { headers: this.authHeaders(), timeoutMs: this.timeoutMs }
190
+ );
191
+ }
192
+ });
193
+ }
194
+ async ingestTurn(sessionKey, role, text, _at) {
195
+ this.ensureReady();
196
+ const userId = this.userIdFor(sessionKey);
197
+ this.knownSessions.add(sessionKey);
198
+ if (this.mode === "oss") {
199
+ await httpJson(this.fetchImpl, "POST", `${this.baseUrl}/memories`, {
200
+ headers: this.authHeaders(),
201
+ body: {
202
+ messages: [{ role, content: text }],
203
+ user_id: userId
204
+ },
205
+ timeoutMs: this.timeoutMs
206
+ });
207
+ } else {
208
+ const addResponse = await httpJson(
209
+ this.fetchImpl,
210
+ "POST",
211
+ `${this.baseUrl}/v3/memories/add/`,
212
+ {
213
+ headers: this.authHeaders(),
214
+ body: {
215
+ messages: [{ role, content: text }],
216
+ user_id: userId
217
+ },
218
+ timeoutMs: this.timeoutMs
219
+ }
220
+ );
221
+ if (addResponse?.status === "FAILED") {
222
+ throw new Error(
223
+ `Mem0 hosted add failed immediately: ${addResponse.error ?? addResponse.status}`
224
+ );
225
+ }
226
+ const eventId = addResponse?.event_id;
227
+ if (eventId) {
228
+ await this.pollEvent(eventId);
229
+ }
230
+ }
231
+ }
232
+ async recall(query, sessionKey) {
233
+ this.ensureReady();
234
+ const userId = this.userIdFor(sessionKey);
235
+ let results;
236
+ if (this.mode === "oss") {
237
+ const body = await httpJson(
238
+ this.fetchImpl,
239
+ "POST",
240
+ `${this.baseUrl}/search`,
241
+ {
242
+ headers: this.authHeaders(),
243
+ // Mem0 OSS v3 aligns search with the platform API: entity IDs go
244
+ // inside `filters` and the count parameter is `top_k`, not
245
+ // top-level `user_id`/`limit` (oss-v2-to-v3 migration).
246
+ body: { query, filters: { user_id: userId }, top_k: 10 },
247
+ timeoutMs: this.timeoutMs
248
+ }
249
+ );
250
+ results = normalizeSearchResults(body);
251
+ } else {
252
+ const body = await httpJson(
253
+ this.fetchImpl,
254
+ "POST",
255
+ `${this.baseUrl}/v3/memories/search/`,
256
+ {
257
+ headers: this.authHeaders(),
258
+ body: { query, filters: { user_id: userId }, top_k: 10 },
259
+ timeoutMs: this.timeoutMs
260
+ }
261
+ );
262
+ results = normalizeSearchResults(body);
263
+ }
264
+ return results.map((r) => r.memory).filter((m) => typeof m === "string" && m.length > 0);
265
+ }
266
+ async correct(text, sessionKey, _at) {
267
+ await this.ingestTurn(sessionKey, "user", text, _at ?? (/* @__PURE__ */ new Date()).toISOString());
268
+ }
269
+ async runMaintenance() {
270
+ }
271
+ /** Poll the hosted event endpoint until the add is processed. */
272
+ async pollEvent(eventId) {
273
+ for (let i = 0; i < this.maxPolls; i++) {
274
+ await delay(this.pollIntervalMs);
275
+ const status = await httpJson(
276
+ this.fetchImpl,
277
+ "GET",
278
+ `${this.baseUrl}/v1/event/${eventId}/`,
279
+ {
280
+ headers: this.authHeaders(),
281
+ timeoutMs: this.timeoutMs
282
+ }
283
+ );
284
+ if (status?.status === "SUCCEEDED") return;
285
+ if (status?.status === "FAILED") {
286
+ throw new Error(
287
+ `Mem0 add event ${eventId} failed: ${status.error ?? "unknown"}`
288
+ );
289
+ }
290
+ }
291
+ throw new Error(
292
+ `Mem0 add event ${eventId} did not complete after ${this.maxPolls} polls`
293
+ );
294
+ }
295
+ };
296
+ function normalizeSearchResults(body) {
297
+ if (!body) return [];
298
+ if (Array.isArray(body)) return body;
299
+ return body.results ?? [];
300
+ }
301
+
302
+ // src/benchmarks/remnic/memcorrect/third-party/zep-adapter.ts
303
+ var ZepMemCorrectAdapter = class {
304
+ label = "zep";
305
+ baseUrl;
306
+ apiKey;
307
+ sessionPrefix;
308
+ settleMs;
309
+ fetchImpl;
310
+ timeoutMs;
311
+ /** Sessions we have ensured exist, to avoid redundant POST /sessions calls. */
312
+ knownSessions = /* @__PURE__ */ new Set();
313
+ /** True when turns have been ingested since the last settle, so recall() can
314
+ * wait for Zep's async graph pipeline before a scored read. */
315
+ pendingIngest = false;
316
+ constructor(config = {}) {
317
+ this.baseUrl = config.baseUrl?.replace(/\/+$/, "") ?? "https://api.getzep.com/api/v2";
318
+ this.apiKey = config.apiKey;
319
+ this.sessionPrefix = config.sessionPrefix ?? `memcorrect-${uniqueRunSuffix()}`;
320
+ this.settleMs = config.settleMs ?? 0;
321
+ this.fetchImpl = resolveFetch(config.fetch);
322
+ this.timeoutMs = config.timeoutMs;
323
+ }
324
+ isConfigured() {
325
+ return !!this.apiKey;
326
+ }
327
+ sessionIdFor(sessionKey) {
328
+ return `${this.sessionPrefix}:${sessionKey}`;
329
+ }
330
+ ensureReady() {
331
+ requireCredentials("Zep", this.apiKey ? [] : ["apiKey (ZEP_API_KEY)"]);
332
+ }
333
+ authHeaders() {
334
+ return { Authorization: `Api-Key ${this.apiKey}` };
335
+ }
336
+ async reset() {
337
+ this.ensureReady();
338
+ await resetTrackedIds("Zep", this.knownSessions, async (sessionId) => {
339
+ try {
340
+ await httpJson(
341
+ this.fetchImpl,
342
+ "DELETE",
343
+ `${this.baseUrl}/sessions/${encodeURIComponent(sessionId)}`,
344
+ { headers: this.authHeaders(), timeoutMs: this.timeoutMs }
345
+ );
346
+ } catch (err) {
347
+ if (!isNotFoundDelete(err)) throw err;
348
+ }
349
+ await httpJson(
350
+ this.fetchImpl,
351
+ "DELETE",
352
+ `${this.baseUrl}/users/${encodeURIComponent(sessionId)}`,
353
+ { headers: this.authHeaders(), timeoutMs: this.timeoutMs }
354
+ );
355
+ });
356
+ this.pendingIngest = false;
357
+ }
358
+ /** Ensure the Zep session (and its user) exist before adding memory. */
359
+ async ensureSession(sessionId) {
360
+ if (this.knownSessions.has(sessionId)) return;
361
+ try {
362
+ await httpJson(this.fetchImpl, "POST", `${this.baseUrl}/users`, {
363
+ headers: this.authHeaders(),
364
+ body: { user_id: sessionId },
365
+ timeoutMs: this.timeoutMs
366
+ });
367
+ } catch (err) {
368
+ if (!isConflict(err)) throw err;
369
+ }
370
+ try {
371
+ await httpJson(
372
+ this.fetchImpl,
373
+ "POST",
374
+ `${this.baseUrl}/sessions`,
375
+ {
376
+ headers: this.authHeaders(),
377
+ body: { session_id: sessionId, user_id: sessionId },
378
+ timeoutMs: this.timeoutMs
379
+ }
380
+ );
381
+ } catch (err) {
382
+ if (!isConflict(err)) throw err;
383
+ }
384
+ this.knownSessions.add(sessionId);
385
+ }
386
+ async ingestTurn(sessionKey, role, text, _at) {
387
+ this.ensureReady();
388
+ const sessionId = this.sessionIdFor(sessionKey);
389
+ await this.ensureSession(sessionId);
390
+ const roleType = role === "user" ? "user" : "assistant";
391
+ await httpJson(
392
+ this.fetchImpl,
393
+ "POST",
394
+ `${this.baseUrl}/sessions/${encodeURIComponent(sessionId)}/memory`,
395
+ {
396
+ headers: this.authHeaders(),
397
+ body: {
398
+ messages: [{ role, role_type: roleType, content: text }]
399
+ },
400
+ timeoutMs: this.timeoutMs
401
+ }
402
+ );
403
+ this.pendingIngest = true;
404
+ }
405
+ async recall(query, sessionKey) {
406
+ this.ensureReady();
407
+ if (this.pendingIngest && this.settleMs > 0) {
408
+ await delay(this.settleMs);
409
+ this.pendingIngest = false;
410
+ }
411
+ const sessionId = this.sessionIdFor(sessionKey);
412
+ await this.ensureSession(sessionId);
413
+ const results = await httpJson(
414
+ this.fetchImpl,
415
+ "POST",
416
+ `${this.baseUrl}/graph/search`,
417
+ {
418
+ headers: this.authHeaders(),
419
+ body: { user_id: sessionId, query, scope: "edges", limit: 10 },
420
+ timeoutMs: this.timeoutMs
421
+ }
422
+ );
423
+ if (!results || !results.edges) return [];
424
+ const strings = [];
425
+ for (const edge of results.edges) {
426
+ const fact3 = edge.fact;
427
+ if (fact3 && fact3.trim().length > 0) {
428
+ strings.push(fact3.trim());
429
+ }
430
+ }
431
+ return strings;
432
+ }
433
+ async correct(text, sessionKey, _at) {
434
+ await this.ingestTurn(
435
+ sessionKey,
436
+ "user",
437
+ text,
438
+ _at ?? (/* @__PURE__ */ new Date()).toISOString()
439
+ );
440
+ }
441
+ async runMaintenance() {
442
+ if (this.settleMs > 0 && this.pendingIngest) {
443
+ await delay(this.settleMs);
444
+ this.pendingIngest = false;
445
+ }
446
+ }
447
+ };
448
+
449
+ // src/benchmarks/remnic/memcorrect/third-party/letta-adapter.ts
450
+ var LettaMemCorrectAdapter = class {
451
+ label = "letta";
452
+ baseUrl;
453
+ apiKey;
454
+ model;
455
+ agentNamePrefix;
456
+ personaBlock;
457
+ fetchImpl;
458
+ timeoutMs;
459
+ /** Maps MemCorrect sessionKey → Letta agent_id. */
460
+ agentsBySession = /* @__PURE__ */ new Map();
461
+ constructor(config = {}) {
462
+ this.baseUrl = config.baseUrl?.replace(/\/+$/, "") ?? "";
463
+ this.apiKey = config.apiKey;
464
+ this.model = config.model;
465
+ this.agentNamePrefix = config.agentNamePrefix ?? "memcorrect";
466
+ this.personaBlock = config.personaBlock ?? "You are a memory benchmark agent. Store every fact the user states in your human memory block. When the user corrects a fact, use memory_replace to update it.";
467
+ this.fetchImpl = resolveFetch(config.fetch);
468
+ this.timeoutMs = config.timeoutMs;
469
+ }
470
+ isConfigured() {
471
+ return !!(this.apiKey && this.baseUrl && this.model);
472
+ }
473
+ ensureReady() {
474
+ const missing = [];
475
+ if (!this.apiKey) missing.push("apiKey (LETTA_API_KEY)");
476
+ if (!this.baseUrl) missing.push("baseUrl (LETTA_BASE_URL)");
477
+ if (!this.model) missing.push("model (LETTA_MODEL)");
478
+ requireCredentials("Letta", missing);
479
+ }
480
+ authHeaders() {
481
+ return this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {};
482
+ }
483
+ async reset() {
484
+ this.ensureReady();
485
+ const failed = [];
486
+ for (const [sessionKey, agentId] of [...this.agentsBySession]) {
487
+ try {
488
+ await httpJson(
489
+ this.fetchImpl,
490
+ "DELETE",
491
+ `${this.baseUrl}/v1/agents/${encodeURIComponent(agentId)}`,
492
+ { headers: this.authHeaders(), timeoutMs: this.timeoutMs }
493
+ );
494
+ this.agentsBySession.delete(sessionKey);
495
+ } catch (err) {
496
+ if (isNotFoundDelete(err)) {
497
+ this.agentsBySession.delete(sessionKey);
498
+ } else {
499
+ failed.push(agentId);
500
+ }
501
+ }
502
+ }
503
+ if (failed.length > 0) {
504
+ throw new Error(
505
+ `Letta reset could not clean ${failed.length} agent(s): ${failed.join(", ")}. The failed agents are retained in agentsBySession for the next reset() retry.`
506
+ );
507
+ }
508
+ }
509
+ /** Create a Letta agent for the session if one does not yet exist. */
510
+ async ensureAgent(sessionKey) {
511
+ const existing = this.agentsBySession.get(sessionKey);
512
+ if (existing) return existing;
513
+ const agent = await httpJson(
514
+ this.fetchImpl,
515
+ "POST",
516
+ `${this.baseUrl}/v1/agents/`,
517
+ {
518
+ headers: this.authHeaders(),
519
+ body: {
520
+ name: `${this.agentNamePrefix}-${sessionKey}`,
521
+ agent_type: "memgpt_agent",
522
+ model: this.model,
523
+ memory_blocks: [
524
+ { label: "human", value: "", limit: 5e3 },
525
+ { label: "persona", value: this.personaBlock, limit: 5e3 }
526
+ ]
527
+ },
528
+ timeoutMs: this.timeoutMs
529
+ }
530
+ );
531
+ const agentId = agent?.id;
532
+ if (!agentId) {
533
+ throw new Error(
534
+ `Letta agent creation did not return an agent id for session ${sessionKey}`
535
+ );
536
+ }
537
+ this.agentsBySession.set(sessionKey, agentId);
538
+ return agentId;
539
+ }
540
+ async ingestTurn(sessionKey, role, text, _at) {
541
+ this.ensureReady();
542
+ const agentId = await this.ensureAgent(sessionKey);
543
+ await httpJson(
544
+ this.fetchImpl,
545
+ "POST",
546
+ `${this.baseUrl}/v1/agents/${encodeURIComponent(agentId)}/messages`,
547
+ {
548
+ headers: this.authHeaders(),
549
+ body: {
550
+ messages: [{ role, content: text }],
551
+ // Letta's /messages endpoint defaults to streaming (SSE). Request a
552
+ // single JSON response so httpJson parses it and ingestTurn resolves
553
+ // only after the agent (and its memory tools) finish processing.
554
+ stream: false
555
+ },
556
+ timeoutMs: this.timeoutMs
557
+ }
558
+ );
559
+ }
560
+ async recall(_query, sessionKey) {
561
+ this.ensureReady();
562
+ const agentId = await this.ensureAgent(sessionKey);
563
+ const response = await httpJson(
564
+ this.fetchImpl,
565
+ "GET",
566
+ `${this.baseUrl}/v1/agents/${encodeURIComponent(agentId)}/core-memory/blocks`,
567
+ { headers: this.authHeaders(), timeoutMs: this.timeoutMs }
568
+ );
569
+ if (!response) return [];
570
+ const blocks = Array.isArray(response) ? response : response.memory ?? response.blocks ?? [];
571
+ const strings = [];
572
+ for (const block of blocks) {
573
+ if (block.label === "persona") continue;
574
+ const value = block.value ?? block.text ?? block.content;
575
+ if (value && value.trim().length > 0) {
576
+ for (const line of value.split(/\n+/)) {
577
+ const trimmed = line.trim();
578
+ if (trimmed) strings.push(trimmed);
579
+ }
580
+ }
581
+ }
582
+ return strings;
583
+ }
584
+ async correct(text, sessionKey, _at) {
585
+ await this.ingestTurn(
586
+ sessionKey,
587
+ "user",
588
+ text,
589
+ _at ?? (/* @__PURE__ */ new Date()).toISOString()
590
+ );
591
+ }
592
+ async runMaintenance() {
593
+ }
594
+ };
595
+
1
596
  // src/ingestion-types.ts
2
597
  var REQUIRED_FRONTMATTER_FIELDS = ["title", "type", "state", "created", "see-also"];
3
598
 
@@ -2945,7 +3540,7 @@ async function withTimeout(label, timeoutMs, fn, onTimeout) {
2945
3540
  () => void 0,
2946
3541
  () => void 0
2947
3542
  ),
2948
- delay(BENCHMARK_TIMEOUT_ABORT_GRACE_MS)
3543
+ delay2(BENCHMARK_TIMEOUT_ABORT_GRACE_MS)
2949
3544
  ]);
2950
3545
  }
2951
3546
  throw error;
@@ -2955,7 +3550,7 @@ async function withTimeout(label, timeoutMs, fn, onTimeout) {
2955
3550
  }
2956
3551
  }
2957
3552
  }
2958
- function delay(ms) {
3553
+ function delay2(ms) {
2959
3554
  return new Promise((resolve) => {
2960
3555
  setTimeout(resolve, ms);
2961
3556
  });
@@ -6285,34 +6880,20 @@ function createAnthropicProvider(config) {
6285
6880
  return new AnthropicProvider(config);
6286
6881
  }
6287
6882
 
6288
- // src/providers/codex-cli.ts
6883
+ // src/providers/claude-cli.ts
6289
6884
  import { spawn } from "child_process";
6290
- import { createHash as createHash5, randomUUID } from "crypto";
6291
- import { mkdir as mkdir5, mkdtemp as mkdtemp2, readFile as readFile6, rm as rm2, writeFile as writeFile5 } from "fs/promises";
6885
+ import { mkdtemp as mkdtemp2, rm as rm2 } from "fs/promises";
6292
6886
  import os3 from "os";
6293
6887
  import path6 from "path";
6294
- var DEFAULT_REASONING_EFFORT = "xhigh";
6295
- var DEFAULT_SERVICE_TIER = "fast";
6296
- var CODEX_CLI_STDIO_LIMIT = 64e3;
6297
- var CODEX_CLI_PARENT_SIGNALS = [
6298
- "SIGHUP",
6299
- "SIGINT",
6300
- "SIGTERM"
6301
- ];
6302
- var CODEX_CLI_FORCED_PARENT_EXIT_MS = 1e3;
6303
- var CODEX_CLI_DIAGNOSTICS_DIR_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_DIR";
6304
- var CODEX_CLI_DIAGNOSTICS_MODE_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_MODE";
6305
- var CODEX_CLI_EXECUTABLE_ENV = "REMNIC_BENCH_CODEX_CLI_EXECUTABLE";
6306
- var CODEX_CLI_TRANSPORT_ENV = "REMNIC_BENCH_CODEX_CLI_TRANSPORT";
6307
- var CODEX_CLI_VERSION_TIMEOUT_MS = 5e3;
6308
- var CODEX_CLI_HEALTH_CACHE_TTL_MS = 3e4;
6309
- var OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
6310
- var OPENAI_BASE_URL_ENV = "OPENAI_BASE_URL";
6311
- var OPENAI_RESPONSES_BASE_URL = "https://api.openai.com/v1";
6312
- var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
6888
+ var CLAUDE_CLI_STDIO_LIMIT = 64e3;
6889
+ var CLAUDE_CLI_PARENT_SIGNALS = ["SIGHUP", "SIGINT", "SIGTERM"];
6890
+ var CLAUDE_CLI_FORCED_PARENT_EXIT_MS = 1e3;
6891
+ var CLAUDE_CLI_EXECUTABLE_ENV = "REMNIC_BENCH_CLAUDE_CLI_EXECUTABLE";
6892
+ var CLAUDE_CLI_VERSION_TIMEOUT_MS = 5e3;
6893
+ var CLAUDE_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
6313
6894
  "ALL_PROXY",
6314
6895
  "APPDATA",
6315
- "CODEX_HOME",
6896
+ "CLAUDE_CODE_OAUTH_TOKEN",
6316
6897
  "COLORTERM",
6317
6898
  "COMSPEC",
6318
6899
  "FORCE_COLOR",
@@ -6326,9 +6907,6 @@ var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
6326
6907
  "NODE_EXTRA_CA_CERTS",
6327
6908
  "NO_PROXY",
6328
6909
  "NUMBER_OF_PROCESSORS",
6329
- OPENAI_BASE_URL_ENV,
6330
- "OPENAI_ORGANIZATION",
6331
- "OPENAI_PROJECT",
6332
6910
  "OS",
6333
6911
  "PATH",
6334
6912
  "PATHEXT",
@@ -6354,17 +6932,84 @@ var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
6354
6932
  "XDG_DATA_HOME",
6355
6933
  "XDG_RUNTIME_DIR"
6356
6934
  ]);
6357
- var activeCodexCliChildPids = /* @__PURE__ */ new Set();
6358
- var codexCliParentCleanupInstalled = false;
6359
- var codexCliHealthCache = /* @__PURE__ */ new Map();
6360
- var CodexCliProvider = class {
6361
- provider = "codex-cli";
6935
+ var DEFAULT_USAGE_LIMIT_MAX_WAIT_MS = 30 * 60 * 1e3;
6936
+ var USAGE_LIMIT_BASE_BACKOFF_MS = 6e4;
6937
+ var USAGE_LIMIT_MAX_STEP_MS = 10 * 60 * 1e3;
6938
+ var activeClaudeCliChildPids = /* @__PURE__ */ new Set();
6939
+ var claudeCliParentCleanupInstalled = false;
6940
+ var ClaudeCliConcurrencyGate = class {
6941
+ active = 0;
6942
+ limit;
6943
+ waiters = [];
6944
+ constructor(limit) {
6945
+ this.limit = normalizeGateLimit(limit);
6946
+ }
6947
+ async run(fn) {
6948
+ await this.acquire();
6949
+ try {
6950
+ return await fn();
6951
+ } finally {
6952
+ this.release();
6953
+ }
6954
+ }
6955
+ /** Raises the gate's limit if `limit` is higher than the current one;
6956
+ * never lowers it. Immediately wakes any queued waiters that fit under
6957
+ * the new, higher limit — see `getSharedClaudeCliGate` for the policy
6958
+ * this implements (shared max across all constructed instances). */
6959
+ raiseLimit(limit) {
6960
+ const normalized = normalizeGateLimit(limit);
6961
+ if (normalized <= this.limit) {
6962
+ return;
6963
+ }
6964
+ this.limit = normalized;
6965
+ while (this.active < this.limit) {
6966
+ const next = this.waiters.shift();
6967
+ if (!next) {
6968
+ break;
6969
+ }
6970
+ next();
6971
+ }
6972
+ }
6973
+ acquire() {
6974
+ if (this.active < this.limit) {
6975
+ this.active += 1;
6976
+ return Promise.resolve();
6977
+ }
6978
+ return new Promise((resolve) => {
6979
+ this.waiters.push(() => {
6980
+ this.active += 1;
6981
+ resolve();
6982
+ });
6983
+ });
6984
+ }
6985
+ release() {
6986
+ this.active = Math.max(0, this.active - 1);
6987
+ const next = this.waiters.shift();
6988
+ if (next) {
6989
+ next();
6990
+ }
6991
+ }
6992
+ };
6993
+ function normalizeGateLimit(limit) {
6994
+ return Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1;
6995
+ }
6996
+ var sharedClaudeCliGate;
6997
+ function getSharedClaudeCliGate(requestedLimit) {
6998
+ if (!sharedClaudeCliGate) {
6999
+ sharedClaudeCliGate = new ClaudeCliConcurrencyGate(requestedLimit);
7000
+ return sharedClaudeCliGate;
7001
+ }
7002
+ sharedClaudeCliGate.raiseLimit(requestedLimit);
7003
+ return sharedClaudeCliGate;
7004
+ }
7005
+ var ClaudeCliProvider = class {
7006
+ provider = "claude-cli";
6362
7007
  id;
6363
7008
  name;
6364
7009
  config;
6365
- runCodexCli;
6366
- runCodexVersion;
6367
- shouldProbeCliHealth;
7010
+ runClaudeCli;
7011
+ runClaudeVersion;
7012
+ gate;
6368
7013
  usage = {
6369
7014
  inputTokens: 0,
6370
7015
  outputTokens: 0,
@@ -6372,112 +7017,801 @@ var CodexCliProvider = class {
6372
7017
  };
6373
7018
  constructor(config, deps = {}) {
6374
7019
  this.config = config;
6375
- this.runCodexCli = deps.runCodexCli ?? runCodexCliCommand;
6376
- this.runCodexVersion = deps.runCodexVersion ?? runCodexVersionCommand;
6377
- this.shouldProbeCliHealth = deps.runCodexCli === void 0;
6378
- this.id = `codex-cli:${config.model}`;
7020
+ this.runClaudeCli = deps.runClaudeCli ?? runClaudeCliCommand;
7021
+ this.runClaudeVersion = deps.runClaudeVersion ?? runClaudeVersionCommand;
7022
+ this.gate = getSharedClaudeCliGate(config.concurrency ?? 1);
7023
+ this.id = `claude-cli:${config.model}`;
6379
7024
  this.name = config.model;
6380
7025
  }
6381
7026
  async complete(prompt, opts = {}) {
6382
- const startedAt = performance.now();
6383
- if (await this.shouldUseResponsesFallback()) {
6384
- return this.completeViaResponsesApi(prompt, opts, startedAt);
7027
+ return this.gate.run(() => this.completeSerialized(prompt, opts));
7028
+ }
7029
+ async completeSerialized(prompt, opts) {
7030
+ if (opts.signal?.aborted) {
7031
+ throw claudeCliAbortError(opts.signal);
6385
7032
  }
6386
- const maxAttempts = normalizeCodexCliMaxAttempts(
6387
- this.config.retryOptions?.maxAttempts
6388
- );
6389
- let lastError;
6390
- for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
6391
- const tempDir = await mkdtemp2(path6.join(os3.tmpdir(), "remnic-codex-cli-"));
6392
- const workspacePath = path6.join(tempDir, "workspace");
6393
- const outputPath = path6.join(tempDir, "last-message.txt");
6394
- let diagnostics;
6395
- let diagnosticsFinished = false;
6396
- const finishDiagnostics = async (outcome) => {
6397
- if (diagnosticsFinished) {
6398
- return;
6399
- }
6400
- diagnosticsFinished = true;
6401
- await finishCodexCliDiagnostics(diagnostics, startedAt, outcome);
6402
- };
7033
+ const startedAt = performance.now();
7034
+ const maxAttempts = normalizeClaudeCliMaxAttempts(this.config.retryOptions?.maxAttempts);
7035
+ const usageLimitBudgetMs = normalizeUsageLimitMaxWaitMs(this.config.retryOptions?.max429WaitMs);
7036
+ const loopStartedAt = performance.now();
7037
+ let transientAttempt = 1;
7038
+ let usageLimitAttempt = 1;
7039
+ while (true) {
7040
+ const tempDir = await mkdtemp2(path6.join(os3.tmpdir(), "remnic-claude-cli-"));
6403
7041
  try {
6404
- await mkdir5(workspacePath, { recursive: true });
6405
- const request = this.buildRunRequest(prompt, opts, workspacePath, outputPath);
6406
- diagnostics = await startCodexCliDiagnostics({
6407
- config: this.config,
6408
- request,
6409
- reasoningEffort: this.config.reasoningEffort ?? DEFAULT_REASONING_EFFORT,
6410
- serviceTier: DEFAULT_SERVICE_TIER,
6411
- retry: { attempt, maxAttempts }
6412
- });
6413
- const result = await this.runCodexCli(request);
7042
+ const request = this.buildRunRequest(prompt, opts, tempDir);
7043
+ const result = await this.runClaudeCli(request);
6414
7044
  if (result.status !== 0) {
7045
+ if (isClaudeUsageLimitSignal(`${result.stderr}
7046
+ ${result.stdout}`)) {
7047
+ await sleepBeforeUsageLimitRetry({
7048
+ attempt: usageLimitAttempt,
7049
+ loopStartedAt,
7050
+ budgetMs: usageLimitBudgetMs,
7051
+ failureSummary: summarizeProcessOutput(result.stderr, result.stdout),
7052
+ signal: opts.signal
7053
+ });
7054
+ usageLimitAttempt += 1;
7055
+ continue;
7056
+ }
6415
7057
  const exitLabel = result.signal ? `signal ${result.signal}` : `exit ${result.status ?? "unknown"}`;
6416
7058
  const error = new Error(
6417
- `Codex CLI completion failed (${exitLabel}): ${summarizeProcessOutput(result.stderr, result.stdout)}`
7059
+ `Claude CLI completion failed (${exitLabel}): ${summarizeProcessOutput(result.stderr, result.stdout)}`
6418
7060
  );
6419
- if (attempt < maxAttempts && isRetryableCodexCliResult(result)) {
6420
- lastError = error;
6421
- await finishDiagnostics({
6422
- result,
6423
- error,
6424
- transientFailure: true
7061
+ if (transientAttempt < maxAttempts && isRetryableClaudeCliResult(result)) {
7062
+ await sleepBeforeClaudeCliRetry({
7063
+ attempt: transientAttempt,
7064
+ baseBackoffMs: this.config.retryOptions?.baseBackoffMs,
7065
+ maxStepMs: 3e4,
7066
+ capMs: Number.POSITIVE_INFINITY,
7067
+ signal: opts.signal
6425
7068
  });
6426
- await sleepBeforeCodexCliRetry(
6427
- attempt,
6428
- this.config.retryOptions?.baseBackoffMs,
6429
- opts.signal
6430
- );
7069
+ transientAttempt += 1;
6431
7070
  continue;
6432
7071
  }
6433
- await finishDiagnostics({ result, error });
6434
7072
  throw error;
6435
7073
  }
6436
- const text = result.outputText.trim();
7074
+ const payload = parseClaudeCliJsonResult(result.stdout);
7075
+ if (isClaudeCliErrorFlagSet(payload.is_error)) {
7076
+ if (isClaudeUsageLimitSignal(
7077
+ `${result.stderr}
7078
+ ${payload.error ?? ""}
7079
+ ${payload.result ?? ""}`
7080
+ )) {
7081
+ await sleepBeforeUsageLimitRetry({
7082
+ attempt: usageLimitAttempt,
7083
+ loopStartedAt,
7084
+ budgetMs: usageLimitBudgetMs,
7085
+ failureSummary: summarizeProcessOutput(result.stderr, result.stdout),
7086
+ signal: opts.signal
7087
+ });
7088
+ usageLimitAttempt += 1;
7089
+ continue;
7090
+ }
7091
+ throw new Error(
7092
+ `Claude CLI reported is_error: ${payload.error?.trim() || payload.result?.trim() || summarizeProcessOutput(result.stderr, result.stdout)}`
7093
+ );
7094
+ }
7095
+ const text = typeof payload.result === "string" ? payload.result.trim() : "";
6437
7096
  if (text.length === 0) {
6438
- const error = new Error(
6439
- `Codex CLI completion returned no final message: ${summarizeProcessOutput(result.stderr, result.stdout)}`
7097
+ if (isClaudeUsageLimitSignal(`${result.stderr}
7098
+ ${payload.error ?? ""}`)) {
7099
+ await sleepBeforeUsageLimitRetry({
7100
+ attempt: usageLimitAttempt,
7101
+ loopStartedAt,
7102
+ budgetMs: usageLimitBudgetMs,
7103
+ failureSummary: summarizeProcessOutput(result.stderr, result.stdout),
7104
+ signal: opts.signal
7105
+ });
7106
+ usageLimitAttempt += 1;
7107
+ continue;
7108
+ }
7109
+ throw new Error(
7110
+ `Claude CLI completion returned no result text: ${summarizeProcessOutput(result.stderr, result.stdout)}`
6440
7111
  );
6441
- await finishDiagnostics({ result, error });
6442
- throw error;
6443
7112
  }
6444
- await finishDiagnostics({ result });
6445
- const tokens = parseCodexTokenUsage(
6446
- `${result.stderr}
6447
- ${result.stdout}`,
6448
- text
6449
- );
6450
- this.recordUsage(tokens.input, tokens.output);
7113
+ const inputTokens = nonNegativeInt(payload.usage?.input_tokens);
7114
+ const outputTokens = nonNegativeInt(payload.usage?.output_tokens);
7115
+ this.recordUsage(inputTokens, outputTokens);
6451
7116
  return {
6452
7117
  text,
6453
- tokens,
7118
+ tokens: { input: inputTokens, output: outputTokens },
6454
7119
  latencyMs: Math.round(performance.now() - startedAt),
6455
7120
  model: this.config.model
6456
7121
  };
6457
- } catch (error) {
6458
- lastError = error;
6459
- await finishDiagnostics({ error });
6460
- throw error;
6461
7122
  } finally {
6462
7123
  await rm2(tempDir, { force: true, recursive: true });
6463
7124
  }
6464
7125
  }
6465
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
6466
7126
  }
6467
7127
  async discover() {
6468
- const version = await this.runCodexVersion(
6469
- resolveCodexCliExecutable(this.config),
6470
- buildIsolatedCodexEnv()
7128
+ const version = await this.runClaudeVersion(
7129
+ resolveClaudeCliExecutable(this.config),
7130
+ buildIsolatedClaudeEnv(this.config)
6471
7131
  );
6472
7132
  if (version.status !== 0) {
6473
7133
  throw new Error(
6474
- `Codex CLI discovery failed: ${version.stderr.trim() || `exit ${version.status ?? "unknown"}`}`
7134
+ `Claude CLI discovery failed: ${version.stderr.trim() || `exit ${version.status ?? "unknown"}`}`
6475
7135
  );
6476
7136
  }
6477
7137
  return [
6478
7138
  {
6479
7139
  id: this.config.model,
6480
- name: `${this.config.model} (Codex CLI)`,
7140
+ name: `${this.config.model} (Claude CLI)`,
7141
+ contextLength: 0,
7142
+ capabilities: ["completion"]
7143
+ }
7144
+ ];
7145
+ }
7146
+ getUsage() {
7147
+ return { ...this.usage };
7148
+ }
7149
+ resetUsage() {
7150
+ this.usage = {
7151
+ inputTokens: 0,
7152
+ outputTokens: 0,
7153
+ totalTokens: 0
7154
+ };
7155
+ }
7156
+ recordUsage(inputTokens, outputTokens) {
7157
+ this.usage = {
7158
+ inputTokens: this.usage.inputTokens + inputTokens,
7159
+ outputTokens: this.usage.outputTokens + outputTokens,
7160
+ totalTokens: this.usage.totalTokens + inputTokens + outputTokens
7161
+ };
7162
+ }
7163
+ buildRunRequest(prompt, opts, cwd) {
7164
+ return {
7165
+ executable: resolveClaudeCliExecutable(this.config),
7166
+ args: buildClaudeCliArgs(this.config, opts.systemPrompt),
7167
+ input: buildClaudeCompletionPrompt(prompt),
7168
+ cwd,
7169
+ timeoutMs: this.config.retryOptions?.timeoutMs,
7170
+ signal: opts.signal,
7171
+ env: buildIsolatedClaudeEnv(this.config, opts.maxTokens)
7172
+ };
7173
+ }
7174
+ };
7175
+ function buildClaudeCliArgs(config, systemPrompt) {
7176
+ const args = [
7177
+ "--print",
7178
+ "--model",
7179
+ config.model,
7180
+ "--output-format",
7181
+ "json",
7182
+ "--input-format",
7183
+ "text",
7184
+ "--safe-mode",
7185
+ "--strict-mcp-config",
7186
+ "--tools",
7187
+ "",
7188
+ "--no-session-persistence"
7189
+ ];
7190
+ const trimmedSystemPrompt = systemPrompt?.trim();
7191
+ if (trimmedSystemPrompt) {
7192
+ args.push("--system-prompt", trimmedSystemPrompt);
7193
+ }
7194
+ return args;
7195
+ }
7196
+ function buildClaudeCompletionPrompt(userPrompt) {
7197
+ const payload = { userPrompt };
7198
+ return [
7199
+ "You are a benchmark evaluation endpoint, not a coding agent.",
7200
+ "Use only the explicit JSON payload below.",
7201
+ "Any additional scoring/format protocol instructions are provided via the",
7202
+ "system prompt, which takes priority over this user payload.",
7203
+ "Do not use tools, do not read or write files, do not browse, and do not use persisted memory.",
7204
+ "Return only the final answer text. If the request asks for JSON, return raw JSON only.",
7205
+ "",
7206
+ "BENCHMARK_REQUEST_JSON:",
7207
+ JSON.stringify(payload, null, 2)
7208
+ ].join("\n");
7209
+ }
7210
+ var CLAUDE_CLI_MAX_OUTPUT_TOKENS_ENV = "CLAUDE_CODE_MAX_OUTPUT_TOKENS";
7211
+ function buildIsolatedClaudeEnv(config, maxTokens) {
7212
+ const env = {};
7213
+ for (const [key, value] of Object.entries(process.env)) {
7214
+ if (value !== void 0 && isAllowedClaudeRuntimeEnvKey(key)) {
7215
+ env[key] = value;
7216
+ }
7217
+ }
7218
+ if (config.apiKey && config.apiKey.trim().length > 0) {
7219
+ env.ANTHROPIC_API_KEY = config.apiKey.trim();
7220
+ }
7221
+ if (config.baseUrl && config.baseUrl.trim().length > 0) {
7222
+ env.ANTHROPIC_BASE_URL = config.baseUrl.trim();
7223
+ }
7224
+ if (typeof maxTokens === "number" && Number.isFinite(maxTokens) && maxTokens > 0) {
7225
+ env[CLAUDE_CLI_MAX_OUTPUT_TOKENS_ENV] = String(Math.floor(maxTokens));
7226
+ }
7227
+ return env;
7228
+ }
7229
+ function isAllowedClaudeRuntimeEnvKey(key) {
7230
+ const normalized = key.toUpperCase();
7231
+ return CLAUDE_CLI_RUNTIME_ENV_ALLOWLIST.has(normalized) || normalized.startsWith("LC_");
7232
+ }
7233
+ function resolveClaudeCliExecutable(config) {
7234
+ const configured = config.executable ?? process.env[CLAUDE_CLI_EXECUTABLE_ENV];
7235
+ if (configured === void 0) {
7236
+ return "claude";
7237
+ }
7238
+ const trimmed = configured.trim();
7239
+ if (trimmed.length === 0) {
7240
+ throw new Error(`${CLAUDE_CLI_EXECUTABLE_ENV} / claude-cli executable must not be empty`);
7241
+ }
7242
+ return expandHomeRelativePath(trimmed);
7243
+ }
7244
+ function expandHomeRelativePath(value) {
7245
+ if (value === "~") {
7246
+ return os3.homedir();
7247
+ }
7248
+ if (value.startsWith("~/") || value.startsWith("~\\")) {
7249
+ return path6.join(os3.homedir(), value.slice(2));
7250
+ }
7251
+ return value;
7252
+ }
7253
+ function parseClaudeCliJsonResult(stdout) {
7254
+ const trimmed = stdout.trim();
7255
+ if (trimmed.length === 0) {
7256
+ return { is_error: true, error: "Claude CLI produced no stdout." };
7257
+ }
7258
+ try {
7259
+ const parsed = JSON.parse(trimmed);
7260
+ if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
7261
+ return { is_error: true, error: trimmed.slice(-1e3) };
7262
+ }
7263
+ return parsed;
7264
+ } catch {
7265
+ return { is_error: true, error: trimmed.slice(-1e3) };
7266
+ }
7267
+ }
7268
+ function nonNegativeInt(value) {
7269
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0;
7270
+ }
7271
+ function isClaudeCliErrorFlagSet(value) {
7272
+ return value === true;
7273
+ }
7274
+ var USAGE_LIMIT_SIGNAL_REGEX = /\b(?:usage limit|session limit|weekly limit|opus limit|sonnet limit|rate[- ]limit(?:ed)?|too many requests|quota exceeded|429)\b/i;
7275
+ function isClaudeUsageLimitSignal(combinedOutput) {
7276
+ return USAGE_LIMIT_SIGNAL_REGEX.test(combinedOutput);
7277
+ }
7278
+ function isRetryableClaudeCliResult(result) {
7279
+ if (!result.signal) {
7280
+ return false;
7281
+ }
7282
+ const stderr = result.stderr.toLowerCase();
7283
+ if (stderr.includes("timed out after") || stderr.includes("aborted by benchmark timeout")) {
7284
+ return false;
7285
+ }
7286
+ return true;
7287
+ }
7288
+ function normalizeClaudeCliMaxAttempts(value) {
7289
+ if (value === void 0) {
7290
+ return 3;
7291
+ }
7292
+ if (!Number.isFinite(value) || value < 1) {
7293
+ return 1;
7294
+ }
7295
+ return Math.min(10, Math.floor(value));
7296
+ }
7297
+ function normalizeUsageLimitMaxWaitMs(value) {
7298
+ if (value === void 0) {
7299
+ return DEFAULT_USAGE_LIMIT_MAX_WAIT_MS;
7300
+ }
7301
+ if (!Number.isFinite(value) || value < 0) {
7302
+ return DEFAULT_USAGE_LIMIT_MAX_WAIT_MS;
7303
+ }
7304
+ return value;
7305
+ }
7306
+ async function sleepBeforeUsageLimitRetry(options) {
7307
+ const delayMs = Math.min(
7308
+ USAGE_LIMIT_BASE_BACKOFF_MS * Math.pow(2, options.attempt - 1),
7309
+ USAGE_LIMIT_MAX_STEP_MS
7310
+ );
7311
+ const remainingBudgetMs = options.budgetMs - (performance.now() - options.loopStartedAt);
7312
+ if (delayMs > remainingBudgetMs) {
7313
+ throw new Error(
7314
+ `Claude CLI usage-limit backoff budget (${options.budgetMs}ms) exhausted: ${options.failureSummary}`
7315
+ );
7316
+ }
7317
+ await sleepBeforeClaudeCliRetry({
7318
+ attempt: options.attempt,
7319
+ baseBackoffMs: USAGE_LIMIT_BASE_BACKOFF_MS,
7320
+ maxStepMs: USAGE_LIMIT_MAX_STEP_MS,
7321
+ capMs: remainingBudgetMs,
7322
+ signal: options.signal
7323
+ });
7324
+ }
7325
+ async function sleepBeforeClaudeCliRetry(options) {
7326
+ const baseBackoffMs = options.baseBackoffMs !== void 0 && Number.isFinite(options.baseBackoffMs) && options.baseBackoffMs > 0 ? options.baseBackoffMs : 1e3;
7327
+ const uncappedDelayMs = baseBackoffMs * Math.pow(2, options.attempt - 1);
7328
+ const delayMs = Math.max(0, Math.min(uncappedDelayMs, options.maxStepMs, options.capMs));
7329
+ const signal = options.signal;
7330
+ if (!signal) {
7331
+ await new Promise((resolve) => {
7332
+ setTimeout(resolve, delayMs);
7333
+ });
7334
+ return;
7335
+ }
7336
+ if (signal.aborted) {
7337
+ throw claudeCliAbortError(signal);
7338
+ }
7339
+ await new Promise((resolve, reject) => {
7340
+ const cleanup = () => {
7341
+ signal.removeEventListener("abort", onAbort);
7342
+ };
7343
+ const onAbort = () => {
7344
+ clearTimeout(timeout);
7345
+ cleanup();
7346
+ reject(claudeCliAbortError(signal));
7347
+ };
7348
+ const timeout = setTimeout(() => {
7349
+ cleanup();
7350
+ resolve();
7351
+ }, delayMs);
7352
+ signal.addEventListener("abort", onAbort, { once: true });
7353
+ });
7354
+ }
7355
+ function claudeCliAbortError(signal) {
7356
+ if (signal.reason instanceof Error) {
7357
+ return signal.reason;
7358
+ }
7359
+ if (signal.reason !== void 0) {
7360
+ return new Error(String(signal.reason));
7361
+ }
7362
+ return new DOMException("The operation was aborted.", "AbortError");
7363
+ }
7364
+ function summarizeProcessOutput(stderr, stdout) {
7365
+ const summary = [stderr.trim(), stdout.trim()].filter((value) => value.length > 0).join("\n").trim();
7366
+ return summary.length > 0 ? summary.slice(-1e3) : "no process output";
7367
+ }
7368
+ function runClaudeVersionCommand(executable, env) {
7369
+ return new Promise((resolve, reject) => {
7370
+ const child = spawn(executable, ["--version"], {
7371
+ env,
7372
+ stdio: ["ignore", "ignore", "pipe"],
7373
+ detached: process.platform !== "win32",
7374
+ windowsHide: true
7375
+ });
7376
+ let stderr = "";
7377
+ let timedOut = false;
7378
+ let killTimeout;
7379
+ const terminateChild = (signal) => {
7380
+ if (child.pid && process.platform !== "win32") {
7381
+ try {
7382
+ process.kill(-child.pid, signal);
7383
+ return;
7384
+ } catch {
7385
+ }
7386
+ }
7387
+ child.kill(signal);
7388
+ };
7389
+ const timeout = setTimeout(() => {
7390
+ timedOut = true;
7391
+ terminateChild("SIGTERM");
7392
+ killTimeout = setTimeout(() => {
7393
+ terminateChild("SIGKILL");
7394
+ }, 1e3);
7395
+ killTimeout.unref();
7396
+ }, CLAUDE_CLI_VERSION_TIMEOUT_MS);
7397
+ timeout.unref();
7398
+ child.stderr?.setEncoding("utf8");
7399
+ child.stderr?.on("data", (chunk) => {
7400
+ stderr = appendBounded(stderr, chunk);
7401
+ });
7402
+ child.on("error", (error) => {
7403
+ clearTimeout(timeout);
7404
+ if (killTimeout) {
7405
+ clearTimeout(killTimeout);
7406
+ }
7407
+ reject(error);
7408
+ });
7409
+ child.on("close", (status) => {
7410
+ clearTimeout(timeout);
7411
+ if (killTimeout) {
7412
+ clearTimeout(killTimeout);
7413
+ }
7414
+ resolve({
7415
+ status: timedOut ? status ?? 124 : status,
7416
+ stderr: timedOut ? appendBounded(stderr, `
7417
+ Claude CLI --version timed out after ${CLAUDE_CLI_VERSION_TIMEOUT_MS}ms.`) : stderr
7418
+ });
7419
+ });
7420
+ });
7421
+ }
7422
+ function runClaudeCliCommand(request) {
7423
+ return new Promise((resolve, reject) => {
7424
+ if (request.signal?.aborted) {
7425
+ resolve({
7426
+ status: 124,
7427
+ signal: null,
7428
+ stdout: "",
7429
+ stderr: "Claude CLI aborted before start."
7430
+ });
7431
+ return;
7432
+ }
7433
+ const child = spawn(request.executable, request.args, {
7434
+ cwd: request.cwd,
7435
+ env: request.env,
7436
+ stdio: ["pipe", "pipe", "pipe"],
7437
+ detached: process.platform !== "win32",
7438
+ windowsHide: true
7439
+ });
7440
+ if (child.pid) {
7441
+ registerActiveClaudeCliChild(child.pid);
7442
+ }
7443
+ let stdout = "";
7444
+ let stderr = "";
7445
+ let timedOut = false;
7446
+ let aborted = false;
7447
+ let killTimeout;
7448
+ const clearKillTimeout = () => {
7449
+ if (killTimeout) {
7450
+ clearTimeout(killTimeout);
7451
+ killTimeout = void 0;
7452
+ }
7453
+ };
7454
+ const terminateChild = (signal) => {
7455
+ if (child.pid && process.platform !== "win32") {
7456
+ try {
7457
+ process.kill(-child.pid, signal);
7458
+ return;
7459
+ } catch {
7460
+ }
7461
+ }
7462
+ child.kill(signal);
7463
+ };
7464
+ const scheduleForcedKill = () => {
7465
+ clearKillTimeout();
7466
+ killTimeout = setTimeout(() => {
7467
+ terminateChild("SIGKILL");
7468
+ }, 1e3);
7469
+ killTimeout.unref();
7470
+ };
7471
+ const onAbort = () => {
7472
+ if (aborted) {
7473
+ return;
7474
+ }
7475
+ aborted = true;
7476
+ stderr = appendBounded(stderr, "\nClaude CLI aborted by benchmark timeout.");
7477
+ terminateChild("SIGTERM");
7478
+ scheduleForcedKill();
7479
+ };
7480
+ request.signal?.addEventListener("abort", onAbort, { once: true });
7481
+ if (request.signal?.aborted) {
7482
+ onAbort();
7483
+ }
7484
+ const timeout = request.timeoutMs ? setTimeout(() => {
7485
+ timedOut = true;
7486
+ terminateChild("SIGTERM");
7487
+ scheduleForcedKill();
7488
+ }, request.timeoutMs) : void 0;
7489
+ timeout?.unref();
7490
+ child.stdout?.setEncoding("utf8");
7491
+ child.stderr?.setEncoding("utf8");
7492
+ child.stdout?.on("data", (chunk) => {
7493
+ stdout = appendBounded(stdout, chunk);
7494
+ });
7495
+ child.stderr?.on("data", (chunk) => {
7496
+ stderr = appendBounded(stderr, chunk);
7497
+ });
7498
+ child.stdin?.on("error", (error) => {
7499
+ stderr = appendBounded(stderr, `
7500
+ Claude CLI stdin error: ${error.code ?? error.message}`);
7501
+ });
7502
+ child.on("error", (error) => {
7503
+ if (timeout) {
7504
+ clearTimeout(timeout);
7505
+ }
7506
+ clearKillTimeout();
7507
+ if (child.pid) {
7508
+ unregisterActiveClaudeCliChild(child.pid);
7509
+ }
7510
+ request.signal?.removeEventListener("abort", onAbort);
7511
+ reject(error);
7512
+ });
7513
+ child.on("close", (status, signal) => {
7514
+ if (timeout) {
7515
+ clearTimeout(timeout);
7516
+ }
7517
+ clearKillTimeout();
7518
+ if (child.pid) {
7519
+ unregisterActiveClaudeCliChild(child.pid);
7520
+ }
7521
+ request.signal?.removeEventListener("abort", onAbort);
7522
+ if (timedOut) {
7523
+ resolve({
7524
+ status: status ?? 124,
7525
+ signal,
7526
+ stdout,
7527
+ stderr: appendBounded(stderr, `
7528
+ Claude CLI timed out after ${request.timeoutMs}ms.`)
7529
+ });
7530
+ return;
7531
+ }
7532
+ if (aborted) {
7533
+ resolve({
7534
+ status: status ?? 124,
7535
+ signal,
7536
+ stdout,
7537
+ stderr
7538
+ });
7539
+ return;
7540
+ }
7541
+ resolve({ status, signal, stdout, stderr });
7542
+ });
7543
+ try {
7544
+ child.stdin?.end(request.input);
7545
+ } catch (error) {
7546
+ stderr = appendBounded(stderr, `
7547
+ Claude CLI stdin error: ${error instanceof Error ? error.message : String(error)}`);
7548
+ }
7549
+ });
7550
+ }
7551
+ function registerActiveClaudeCliChild(pid) {
7552
+ installClaudeCliParentCleanup();
7553
+ activeClaudeCliChildPids.add(pid);
7554
+ }
7555
+ function unregisterActiveClaudeCliChild(pid) {
7556
+ activeClaudeCliChildPids.delete(pid);
7557
+ }
7558
+ function installClaudeCliParentCleanup() {
7559
+ if (claudeCliParentCleanupInstalled) {
7560
+ return;
7561
+ }
7562
+ claudeCliParentCleanupInstalled = true;
7563
+ process.once("exit", () => {
7564
+ terminateActiveClaudeCliChildren("SIGTERM");
7565
+ });
7566
+ for (const signal of CLAUDE_CLI_PARENT_SIGNALS) {
7567
+ process.once(signal, () => {
7568
+ const activeChildren = activeClaudeCliChildPids.size;
7569
+ terminateActiveClaudeCliChildren(signal === "SIGINT" ? "SIGINT" : "SIGTERM");
7570
+ process.exitCode = signalExitCode(signal);
7571
+ setTimeout(
7572
+ () => {
7573
+ terminateActiveClaudeCliChildren("SIGKILL");
7574
+ process.exit(signalExitCode(signal));
7575
+ },
7576
+ activeChildren > 0 ? CLAUDE_CLI_FORCED_PARENT_EXIT_MS : 0
7577
+ );
7578
+ });
7579
+ }
7580
+ }
7581
+ function terminateActiveClaudeCliChildren(signal) {
7582
+ for (const pid of activeClaudeCliChildPids) {
7583
+ terminateClaudeCliChildPid(pid, signal);
7584
+ }
7585
+ }
7586
+ function terminateClaudeCliChildPid(pid, signal) {
7587
+ if (process.platform !== "win32") {
7588
+ try {
7589
+ process.kill(-pid, signal);
7590
+ return;
7591
+ } catch {
7592
+ }
7593
+ }
7594
+ try {
7595
+ process.kill(pid, signal);
7596
+ } catch {
7597
+ }
7598
+ }
7599
+ function signalExitCode(signal) {
7600
+ switch (signal) {
7601
+ case "SIGHUP":
7602
+ return 129;
7603
+ case "SIGINT":
7604
+ return 130;
7605
+ case "SIGTERM":
7606
+ return 143;
7607
+ default:
7608
+ return 1;
7609
+ }
7610
+ }
7611
+ function appendBounded(existing, next) {
7612
+ const combined = existing + next;
7613
+ if (combined.length <= CLAUDE_CLI_STDIO_LIMIT) {
7614
+ return combined;
7615
+ }
7616
+ return combined.slice(combined.length - CLAUDE_CLI_STDIO_LIMIT);
7617
+ }
7618
+ function createClaudeCliProvider(config, deps) {
7619
+ return new ClaudeCliProvider(config, deps);
7620
+ }
7621
+
7622
+ // src/providers/codex-cli.ts
7623
+ import { spawn as spawn2 } from "child_process";
7624
+ import { createHash as createHash5, randomUUID } from "crypto";
7625
+ import { mkdir as mkdir5, mkdtemp as mkdtemp3, readFile as readFile6, rm as rm3, writeFile as writeFile5 } from "fs/promises";
7626
+ import os4 from "os";
7627
+ import path7 from "path";
7628
+ var DEFAULT_REASONING_EFFORT = "xhigh";
7629
+ var DEFAULT_SERVICE_TIER = "fast";
7630
+ var CODEX_CLI_STDIO_LIMIT = 64e3;
7631
+ var CODEX_CLI_PARENT_SIGNALS = [
7632
+ "SIGHUP",
7633
+ "SIGINT",
7634
+ "SIGTERM"
7635
+ ];
7636
+ var CODEX_CLI_FORCED_PARENT_EXIT_MS = 1e3;
7637
+ var CODEX_CLI_DIAGNOSTICS_DIR_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_DIR";
7638
+ var CODEX_CLI_DIAGNOSTICS_MODE_ENV = "REMNIC_BENCH_CODEX_CLI_DIAGNOSTICS_MODE";
7639
+ var CODEX_CLI_EXECUTABLE_ENV = "REMNIC_BENCH_CODEX_CLI_EXECUTABLE";
7640
+ var CODEX_CLI_TRANSPORT_ENV = "REMNIC_BENCH_CODEX_CLI_TRANSPORT";
7641
+ var CODEX_CLI_VERSION_TIMEOUT_MS = 5e3;
7642
+ var CODEX_CLI_HEALTH_CACHE_TTL_MS = 3e4;
7643
+ var OPENAI_API_KEY_ENV = "OPENAI_API_KEY";
7644
+ var OPENAI_BASE_URL_ENV = "OPENAI_BASE_URL";
7645
+ var OPENAI_RESPONSES_BASE_URL = "https://api.openai.com/v1";
7646
+ var CODEX_CLI_RUNTIME_ENV_ALLOWLIST = /* @__PURE__ */ new Set([
7647
+ "ALL_PROXY",
7648
+ "APPDATA",
7649
+ "CODEX_HOME",
7650
+ "COLORTERM",
7651
+ "COMSPEC",
7652
+ "FORCE_COLOR",
7653
+ "HOME",
7654
+ "HOMEDRIVE",
7655
+ "HOMEPATH",
7656
+ "LANG",
7657
+ "LOCALAPPDATA",
7658
+ "LOGNAME",
7659
+ "NO_COLOR",
7660
+ "NODE_EXTRA_CA_CERTS",
7661
+ "NO_PROXY",
7662
+ "NUMBER_OF_PROCESSORS",
7663
+ OPENAI_BASE_URL_ENV,
7664
+ "OPENAI_ORGANIZATION",
7665
+ "OPENAI_PROJECT",
7666
+ "OS",
7667
+ "PATH",
7668
+ "PATHEXT",
7669
+ "PROGRAMDATA",
7670
+ "PROCESSOR_ARCHITECTURE",
7671
+ "HTTP_PROXY",
7672
+ "HTTPS_PROXY",
7673
+ "SHELL",
7674
+ "SSL_CERT_DIR",
7675
+ "SSL_CERT_FILE",
7676
+ "SYSTEMDRIVE",
7677
+ "SYSTEMROOT",
7678
+ "TEMP",
7679
+ "TERM",
7680
+ "TMP",
7681
+ "TMPDIR",
7682
+ "USER",
7683
+ "USERNAME",
7684
+ "USERPROFILE",
7685
+ "WINDIR",
7686
+ "XDG_CACHE_HOME",
7687
+ "XDG_CONFIG_HOME",
7688
+ "XDG_DATA_HOME",
7689
+ "XDG_RUNTIME_DIR"
7690
+ ]);
7691
+ var activeCodexCliChildPids = /* @__PURE__ */ new Set();
7692
+ var codexCliParentCleanupInstalled = false;
7693
+ var codexCliHealthCache = /* @__PURE__ */ new Map();
7694
+ var CodexCliProvider = class {
7695
+ provider = "codex-cli";
7696
+ id;
7697
+ name;
7698
+ config;
7699
+ runCodexCli;
7700
+ runCodexVersion;
7701
+ shouldProbeCliHealth;
7702
+ usage = {
7703
+ inputTokens: 0,
7704
+ outputTokens: 0,
7705
+ totalTokens: 0
7706
+ };
7707
+ constructor(config, deps = {}) {
7708
+ this.config = config;
7709
+ this.runCodexCli = deps.runCodexCli ?? runCodexCliCommand;
7710
+ this.runCodexVersion = deps.runCodexVersion ?? runCodexVersionCommand;
7711
+ this.shouldProbeCliHealth = deps.runCodexCli === void 0;
7712
+ this.id = `codex-cli:${config.model}`;
7713
+ this.name = config.model;
7714
+ }
7715
+ async complete(prompt, opts = {}) {
7716
+ const startedAt = performance.now();
7717
+ if (await this.shouldUseResponsesFallback()) {
7718
+ return this.completeViaResponsesApi(prompt, opts, startedAt);
7719
+ }
7720
+ const maxAttempts = normalizeCodexCliMaxAttempts(
7721
+ this.config.retryOptions?.maxAttempts
7722
+ );
7723
+ let lastError;
7724
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
7725
+ const tempDir = await mkdtemp3(path7.join(os4.tmpdir(), "remnic-codex-cli-"));
7726
+ const workspacePath = path7.join(tempDir, "workspace");
7727
+ const outputPath = path7.join(tempDir, "last-message.txt");
7728
+ let diagnostics;
7729
+ let diagnosticsFinished = false;
7730
+ const finishDiagnostics = async (outcome) => {
7731
+ if (diagnosticsFinished) {
7732
+ return;
7733
+ }
7734
+ diagnosticsFinished = true;
7735
+ await finishCodexCliDiagnostics(diagnostics, startedAt, outcome);
7736
+ };
7737
+ try {
7738
+ await mkdir5(workspacePath, { recursive: true });
7739
+ const request = this.buildRunRequest(prompt, opts, workspacePath, outputPath);
7740
+ diagnostics = await startCodexCliDiagnostics({
7741
+ config: this.config,
7742
+ request,
7743
+ reasoningEffort: this.config.reasoningEffort ?? DEFAULT_REASONING_EFFORT,
7744
+ serviceTier: DEFAULT_SERVICE_TIER,
7745
+ retry: { attempt, maxAttempts }
7746
+ });
7747
+ const result = await this.runCodexCli(request);
7748
+ if (result.status !== 0) {
7749
+ const exitLabel = result.signal ? `signal ${result.signal}` : `exit ${result.status ?? "unknown"}`;
7750
+ const error = new Error(
7751
+ `Codex CLI completion failed (${exitLabel}): ${summarizeProcessOutput2(result.stderr, result.stdout)}`
7752
+ );
7753
+ if (attempt < maxAttempts && isRetryableCodexCliResult(result)) {
7754
+ lastError = error;
7755
+ await finishDiagnostics({
7756
+ result,
7757
+ error,
7758
+ transientFailure: true
7759
+ });
7760
+ await sleepBeforeCodexCliRetry(
7761
+ attempt,
7762
+ this.config.retryOptions?.baseBackoffMs,
7763
+ opts.signal
7764
+ );
7765
+ continue;
7766
+ }
7767
+ await finishDiagnostics({ result, error });
7768
+ throw error;
7769
+ }
7770
+ const text = result.outputText.trim();
7771
+ if (text.length === 0) {
7772
+ const error = new Error(
7773
+ `Codex CLI completion returned no final message: ${summarizeProcessOutput2(result.stderr, result.stdout)}`
7774
+ );
7775
+ await finishDiagnostics({ result, error });
7776
+ throw error;
7777
+ }
7778
+ await finishDiagnostics({ result });
7779
+ const tokens = parseCodexTokenUsage(
7780
+ `${result.stderr}
7781
+ ${result.stdout}`,
7782
+ text
7783
+ );
7784
+ this.recordUsage(tokens.input, tokens.output);
7785
+ return {
7786
+ text,
7787
+ tokens,
7788
+ latencyMs: Math.round(performance.now() - startedAt),
7789
+ model: this.config.model
7790
+ };
7791
+ } catch (error) {
7792
+ lastError = error;
7793
+ await finishDiagnostics({ error });
7794
+ throw error;
7795
+ } finally {
7796
+ await rm3(tempDir, { force: true, recursive: true });
7797
+ }
7798
+ }
7799
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
7800
+ }
7801
+ async discover() {
7802
+ const version = await this.runCodexVersion(
7803
+ resolveCodexCliExecutable(this.config),
7804
+ buildIsolatedCodexEnv()
7805
+ );
7806
+ if (version.status !== 0) {
7807
+ throw new Error(
7808
+ `Codex CLI discovery failed: ${version.stderr.trim() || `exit ${version.status ?? "unknown"}`}`
7809
+ );
7810
+ }
7811
+ return [
7812
+ {
7813
+ id: this.config.model,
7814
+ name: `${this.config.model} (Codex CLI)`,
6481
7815
  contextLength: 0,
6482
7816
  capabilities: ["completion"]
6483
7817
  }
@@ -6686,7 +8020,7 @@ function extractResponsesOutputText(payload) {
6686
8020
  }
6687
8021
  function runCodexVersionCommand(executable, env) {
6688
8022
  return new Promise((resolve, reject) => {
6689
- const child = spawn(executable, ["--version"], {
8023
+ const child = spawn2(executable, ["--version"], {
6690
8024
  env,
6691
8025
  stdio: ["ignore", "ignore", "pipe"],
6692
8026
  detached: process.platform !== "win32",
@@ -6716,7 +8050,7 @@ function runCodexVersionCommand(executable, env) {
6716
8050
  timeout.unref();
6717
8051
  child.stderr?.setEncoding("utf8");
6718
8052
  child.stderr?.on("data", (chunk) => {
6719
- stderr = appendBounded(stderr, chunk);
8053
+ stderr = appendBounded2(stderr, chunk);
6720
8054
  });
6721
8055
  child.on("error", (error) => {
6722
8056
  clearTimeout(timeout);
@@ -6732,7 +8066,7 @@ function runCodexVersionCommand(executable, env) {
6732
8066
  }
6733
8067
  resolve({
6734
8068
  status: timedOut ? status ?? 124 : status,
6735
- stderr: timedOut ? appendBounded(
8069
+ stderr: timedOut ? appendBounded2(
6736
8070
  stderr,
6737
8071
  `
6738
8072
  Codex CLI --version timed out after ${CODEX_CLI_VERSION_TIMEOUT_MS}ms.`
@@ -6752,7 +8086,7 @@ function resolveCodexCliExecutable(config) {
6752
8086
  `${CODEX_CLI_EXECUTABLE_ENV} / codex-cli executable must not be empty`
6753
8087
  );
6754
8088
  }
6755
- return expandHomeRelativePath(trimmed);
8089
+ return expandHomeRelativePath2(trimmed);
6756
8090
  }
6757
8091
  function buildCodexCompletionPrompt(userPrompt, systemPrompt) {
6758
8092
  const payload = {
@@ -6810,10 +8144,10 @@ async function startCodexCliDiagnostics(args) {
6810
8144
  model: args.config.model,
6811
8145
  reasoningEffort: args.reasoningEffort,
6812
8146
  serviceTier: args.serviceTier,
6813
- executable: path6.basename(args.request.executable),
8147
+ executable: path7.basename(args.request.executable),
6814
8148
  ...args.request.timeoutMs ? { timeoutMs: args.request.timeoutMs } : {},
6815
- workspaceBasename: path6.basename(args.request.workspacePath),
6816
- outputBasename: path6.basename(args.request.outputPath),
8149
+ workspaceBasename: path7.basename(args.request.workspacePath),
8150
+ outputBasename: path7.basename(args.request.outputPath),
6817
8151
  prompt: promptStats,
6818
8152
  command: {
6819
8153
  args: redactCodexCliArgs(args.request.args)
@@ -6821,7 +8155,7 @@ async function startCodexCliDiagnostics(args) {
6821
8155
  retry: args.retry,
6822
8156
  ...mode === "full" ? { fullPrompt: args.request.input } : {}
6823
8157
  };
6824
- const filePath = path6.join(diagnosticsDir, `${id}.json`);
8158
+ const filePath = path7.join(diagnosticsDir, `${id}.json`);
6825
8159
  await writeCodexCliDiagnosticRecord(filePath, record);
6826
8160
  return { path: filePath, record };
6827
8161
  } catch {
@@ -6873,14 +8207,14 @@ async function writeCodexCliDiagnosticRecord(filePath, record) {
6873
8207
  function resolveCodexCliDiagnosticsDir(config) {
6874
8208
  const dir = config.diagnosticsDir ?? process.env[CODEX_CLI_DIAGNOSTICS_DIR_ENV];
6875
8209
  const trimmed = typeof dir === "string" ? dir.trim() : "";
6876
- return trimmed.length > 0 ? path6.resolve(expandHomeRelativePath(trimmed)) : void 0;
8210
+ return trimmed.length > 0 ? path7.resolve(expandHomeRelativePath2(trimmed)) : void 0;
6877
8211
  }
6878
- function expandHomeRelativePath(value) {
8212
+ function expandHomeRelativePath2(value) {
6879
8213
  if (value === "~") {
6880
- return os3.homedir();
8214
+ return os4.homedir();
6881
8215
  }
6882
8216
  if (value.startsWith("~/") || value.startsWith("~\\")) {
6883
- return path6.join(os3.homedir(), value.slice(2));
8217
+ return path7.join(os4.homedir(), value.slice(2));
6884
8218
  }
6885
8219
  return value;
6886
8220
  }
@@ -6939,7 +8273,7 @@ function runCodexCliCommand(request) {
6939
8273
  });
6940
8274
  return;
6941
8275
  }
6942
- const child = spawn(request.executable, request.args, {
8276
+ const child = spawn2(request.executable, request.args, {
6943
8277
  cwd: request.workspacePath,
6944
8278
  env: request.env,
6945
8279
  stdio: ["pipe", "pipe", "pipe"],
@@ -6982,7 +8316,7 @@ function runCodexCliCommand(request) {
6982
8316
  return;
6983
8317
  }
6984
8318
  aborted = true;
6985
- stderr = appendBounded(stderr, "\nCodex CLI aborted by benchmark timeout.");
8319
+ stderr = appendBounded2(stderr, "\nCodex CLI aborted by benchmark timeout.");
6986
8320
  terminateChild("SIGTERM");
6987
8321
  scheduleForcedKill();
6988
8322
  };
@@ -6999,13 +8333,13 @@ function runCodexCliCommand(request) {
6999
8333
  child.stdout?.setEncoding("utf8");
7000
8334
  child.stderr?.setEncoding("utf8");
7001
8335
  child.stdout?.on("data", (chunk) => {
7002
- stdout = appendBounded(stdout, chunk);
8336
+ stdout = appendBounded2(stdout, chunk);
7003
8337
  });
7004
8338
  child.stderr?.on("data", (chunk) => {
7005
- stderr = appendBounded(stderr, chunk);
8339
+ stderr = appendBounded2(stderr, chunk);
7006
8340
  });
7007
8341
  child.stdin?.on("error", (error) => {
7008
- stderr = appendBounded(
8342
+ stderr = appendBounded2(
7009
8343
  stderr,
7010
8344
  `
7011
8345
  Codex CLI stdin error: ${error.code ?? error.message}`
@@ -7036,7 +8370,7 @@ Codex CLI stdin error: ${error.code ?? error.message}`
7036
8370
  status: status ?? 124,
7037
8371
  signal,
7038
8372
  stdout,
7039
- stderr: appendBounded(
8373
+ stderr: appendBounded2(
7040
8374
  stderr,
7041
8375
  `
7042
8376
  Codex CLI timed out after ${request.timeoutMs}ms.`
@@ -7065,7 +8399,7 @@ Codex CLI timed out after ${request.timeoutMs}ms.`
7065
8399
  try {
7066
8400
  child.stdin?.end(request.input);
7067
8401
  } catch (error) {
7068
- stderr = appendBounded(
8402
+ stderr = appendBounded2(
7069
8403
  stderr,
7070
8404
  `
7071
8405
  Codex CLI stdin error: ${error instanceof Error ? error.message : String(error)}`
@@ -7092,11 +8426,11 @@ function installCodexCliParentCleanup() {
7092
8426
  process.once(signal, () => {
7093
8427
  const activeChildren = activeCodexCliChildPids.size;
7094
8428
  terminateActiveCodexCliChildren(signal === "SIGINT" ? "SIGINT" : "SIGTERM");
7095
- process.exitCode = signalExitCode(signal);
8429
+ process.exitCode = signalExitCode2(signal);
7096
8430
  setTimeout(
7097
8431
  () => {
7098
8432
  terminateActiveCodexCliChildren("SIGKILL");
7099
- process.exit(signalExitCode(signal));
8433
+ process.exit(signalExitCode2(signal));
7100
8434
  },
7101
8435
  activeChildren > 0 ? CODEX_CLI_FORCED_PARENT_EXIT_MS : 0
7102
8436
  );
@@ -7121,7 +8455,7 @@ function terminateCodexCliChildPid(pid, signal) {
7121
8455
  } catch {
7122
8456
  }
7123
8457
  }
7124
- function signalExitCode(signal) {
8458
+ function signalExitCode2(signal) {
7125
8459
  switch (signal) {
7126
8460
  case "SIGHUP":
7127
8461
  return 129;
@@ -7140,7 +8474,7 @@ async function readCodexOutput(outputPath, stdout) {
7140
8474
  return stdout;
7141
8475
  }
7142
8476
  }
7143
- function appendBounded(existing, next) {
8477
+ function appendBounded2(existing, next) {
7144
8478
  const combined = existing + next;
7145
8479
  if (combined.length <= CODEX_CLI_STDIO_LIMIT) {
7146
8480
  return combined;
@@ -7210,7 +8544,7 @@ function codexCliAbortError(signal) {
7210
8544
  }
7211
8545
  return new DOMException("The operation was aborted.", "AbortError");
7212
8546
  }
7213
- function summarizeProcessOutput(stderr, stdout) {
8547
+ function summarizeProcessOutput2(stderr, stdout) {
7214
8548
  const summary = [stderr.trim(), stdout.trim()].filter((value) => value.length > 0).join("\n").trim();
7215
8549
  return summary.length > 0 ? summary.slice(-1e3) : "no process output";
7216
8550
  }
@@ -7247,19 +8581,19 @@ function createCodexCliProvider(config, deps) {
7247
8581
  // src/reporter.ts
7248
8582
  import { execSync } from "child_process";
7249
8583
  import { mkdir as mkdir7, readFile as readFile7, writeFile as writeFile7 } from "fs/promises";
7250
- import path9 from "path";
8584
+ import path10 from "path";
7251
8585
 
7252
8586
  // src/filename-safety.ts
7253
- import path7 from "path";
8587
+ import path8 from "path";
7254
8588
  function sanitizeFilenameSegment(value) {
7255
8589
  const sanitized = value.trim().replace(/[^a-zA-Z0-9._-]/g, "_");
7256
8590
  return sanitized.length > 0 ? sanitized : "unknown";
7257
8591
  }
7258
8592
  function resolveContainedPath(root, ...segments) {
7259
- const outputRoot = path7.resolve(root);
7260
- const filePath = path7.resolve(outputRoot, ...segments);
7261
- const relativePath = path7.relative(outputRoot, filePath);
7262
- if (relativePath === ".." || relativePath.startsWith(`..${path7.sep}`) || path7.isAbsolute(relativePath)) {
8593
+ const outputRoot = path8.resolve(root);
8594
+ const filePath = path8.resolve(outputRoot, ...segments);
8595
+ const relativePath = path8.relative(outputRoot, filePath);
8596
+ if (relativePath === ".." || relativePath.startsWith(`..${path8.sep}`) || path8.isAbsolute(relativePath)) {
7263
8597
  throw new Error(`Refusing to write benchmark artifact outside ${outputRoot}`);
7264
8598
  }
7265
8599
  return filePath;
@@ -7267,7 +8601,7 @@ function resolveContainedPath(root, ...segments) {
7267
8601
 
7268
8602
  // src/leaderboard-export.ts
7269
8603
  import { mkdir as mkdir6, writeFile as writeFile6 } from "fs/promises";
7270
- import path8 from "path";
8604
+ import path9 from "path";
7271
8605
  async function writeLeaderboardArtifactsForResult(result, outputDir) {
7272
8606
  if (result.meta.benchmark === "ama-bench") {
7273
8607
  return writeAmaBenchLeaderboard(result, outputDir);
@@ -7282,7 +8616,7 @@ async function writeAmaBenchLeaderboard(result, outputDir) {
7282
8616
  if (rows.length === 0) {
7283
8617
  return [];
7284
8618
  }
7285
- const outputRoot = path8.resolve(outputDir);
8619
+ const outputRoot = path9.resolve(outputDir);
7286
8620
  const leaderboardDir = resolveContainedPath(outputRoot, "leaderboard");
7287
8621
  await mkdir6(leaderboardDir, { recursive: true });
7288
8622
  const timestamp = sanitizeFilenameSegment(result.meta.timestamp.replace(/[:.]/g, "-"));
@@ -7300,7 +8634,7 @@ async function writeAmaBenchLeaderboard(result, outputDir) {
7300
8634
  async function writeMemCorrectLeaderboard(result, outputDir) {
7301
8635
  const row = buildMemCorrectLeaderboardRow(result);
7302
8636
  if (!row) return [];
7303
- const outputRoot = path8.resolve(outputDir);
8637
+ const outputRoot = path9.resolve(outputDir);
7304
8638
  const leaderboardDir = resolveContainedPath(outputRoot, "leaderboard");
7305
8639
  await mkdir6(leaderboardDir, { recursive: true });
7306
8640
  const timestamp = sanitizeFilenameSegment(result.meta.timestamp.replace(/[:.]/g, "-"));
@@ -7668,7 +9002,7 @@ function replaceLoneSurrogates(value) {
7668
9002
  return out;
7669
9003
  }
7670
9004
  async function writeBenchmarkResult(result, outputDir) {
7671
- const outputRoot = path9.resolve(outputDir);
9005
+ const outputRoot = path10.resolve(outputDir);
7672
9006
  await mkdir7(outputRoot, { recursive: true });
7673
9007
  const safeBenchmark = sanitizeFilenameSegment(result.meta.benchmark);
7674
9008
  const safeRemnicVersion = sanitizeFilenameSegment(result.meta.remnicVersion);
@@ -7704,7 +9038,7 @@ async function writeBenchmarkResult(result, outputDir) {
7704
9038
  async function getRemnicVersion() {
7705
9039
  try {
7706
9040
  const packageJson = JSON.parse(
7707
- await readFile7(path9.resolve(import.meta.dirname, "../../../package.json"), "utf8")
9041
+ await readFile7(path10.resolve(import.meta.dirname, "../../../package.json"), "utf8")
7708
9042
  );
7709
9043
  return typeof packageJson.version === "string" ? packageJson.version : "unknown";
7710
9044
  } catch {
@@ -8210,6 +9544,8 @@ function createProvider(config) {
8210
9544
  return createLocalLlmProvider(config);
8211
9545
  case "codex-cli":
8212
9546
  return createCodexCliProvider(config);
9547
+ case "claude-cli":
9548
+ return createClaudeCliProvider(config);
8213
9549
  default: {
8214
9550
  const exhaustive = config;
8215
9551
  throw new Error(`Unknown provider: ${JSON.stringify(exhaustive)}`);
@@ -9752,7 +11088,7 @@ function clampNormalizedScore(value) {
9752
11088
  }
9753
11089
 
9754
11090
  // src/runtime-profiles.ts
9755
- import path10 from "path";
11091
+ import path11 from "path";
9756
11092
  import { readFile as readFile9 } from "fs/promises";
9757
11093
  import {
9758
11094
  resolvePluginEntry,
@@ -10796,10 +12132,10 @@ async function loadOpenclawRuntimeConfig(filePath) {
10796
12132
  };
10797
12133
  }
10798
12134
  function deriveOpenclawRuntimeContext(configPath) {
10799
- const rootDir = path10.dirname(path10.resolve(configPath));
12135
+ const rootDir = path11.dirname(path11.resolve(configPath));
10800
12136
  return {
10801
- agentDir: path10.join(rootDir, "agents", "main", "agent"),
10802
- workspaceDir: path10.join(rootDir, "workspace")
12137
+ agentDir: path11.join(rootDir, "agents", "main", "agent"),
12138
+ workspaceDir: path11.join(rootDir, "workspace")
10803
12139
  };
10804
12140
  }
10805
12141
  async function loadJsonObject(filePath, label) {
@@ -10836,6 +12172,11 @@ function resolveProviderConfig(kind, provider, model, baseUrl, requestTimeout, d
10836
12172
  `${kind} provider "local-llm" requires a baseUrl (e.g. http://localhost:8080/v1 for llama.cpp).`
10837
12173
  );
10838
12174
  }
12175
+ if (kind === "internal" && provider === "claude-cli") {
12176
+ throw new Error(
12177
+ `${kind} provider "claude-cli" is not supported: it has no @remnic/core gateway wiring and is only available via --provider, --system-provider, or --judge-provider.`
12178
+ );
12179
+ }
10839
12180
  if (reasoningEffort !== void 0 && provider !== "codex-cli") {
10840
12181
  throw new Error(
10841
12182
  `${kind} Codex reasoning effort requires provider "codex-cli"`
@@ -10958,6 +12299,11 @@ function gatewayProviderApi(provider) {
10958
12299
  if (provider === "codex-cli") {
10959
12300
  return "codex-cli";
10960
12301
  }
12302
+ if (provider === "claude-cli") {
12303
+ throw new Error(
12304
+ "claude-cli has no @remnic/core gateway api mapping; it is only supported as a --provider / --system-provider / --judge-provider, not an internal provider."
12305
+ );
12306
+ }
10961
12307
  if (provider === "ollama") {
10962
12308
  return "ollama-chat";
10963
12309
  }
@@ -10978,6 +12324,10 @@ function defaultInternalBaseUrl(provider) {
10978
12324
  return "http://localhost:11434/api";
10979
12325
  case "codex-cli":
10980
12326
  return "codex-cli://local";
12327
+ case "claude-cli":
12328
+ throw new Error(
12329
+ "claude-cli has no internal-provider base URL; it is only supported as a --provider / --system-provider / --judge-provider, not an internal provider."
12330
+ );
10981
12331
  case "local-llm":
10982
12332
  return void 0;
10983
12333
  default: {
@@ -11198,7 +12548,7 @@ async function resolveLocalLabRuntimeProfile(options) {
11198
12548
 
11199
12549
  // src/benchmark.ts
11200
12550
  import fs2 from "fs";
11201
- import path33 from "path";
12551
+ import path34 from "path";
11202
12552
  import { createHash as createHash11 } from "crypto";
11203
12553
  import { expandTildePath as expandTildePath3 } from "@remnic/core";
11204
12554
 
@@ -11208,10 +12558,10 @@ import {
11208
12558
  mkdir as mkdir8,
11209
12559
  readFile as readFile10,
11210
12560
  rename as rename2,
11211
- rm as rm3,
12561
+ rm as rm4,
11212
12562
  writeFile as writeFile8
11213
12563
  } from "fs/promises";
11214
- import path11 from "path";
12564
+ import path12 from "path";
11215
12565
  var JUDGE_CACHE_PROTOCOL_VERSION = "judge-protocol-v1";
11216
12566
  function stableStringify2(value) {
11217
12567
  if (Array.isArray(value)) {
@@ -11242,7 +12592,7 @@ var JudgeCache = class {
11242
12592
  inflight = /* @__PURE__ */ new Map();
11243
12593
  cachedDirExists = false;
11244
12594
  constructor(options) {
11245
- this.dir = path11.resolve(options.dir);
12595
+ this.dir = path12.resolve(options.dir);
11246
12596
  }
11247
12597
  /** Compute the sha256-hex key for a set of parts. Pure, sync, side-effect-free. */
11248
12598
  computeKey(parts) {
@@ -11317,7 +12667,7 @@ var JudgeCache = class {
11317
12667
  this.cachedDirExists = true;
11318
12668
  }
11319
12669
  const filePath = this.entryPath(key);
11320
- const tempPath = path11.join(
12670
+ const tempPath = path12.join(
11321
12671
  this.dir,
11322
12672
  `.${key}.${randomBytes2(6).toString("hex")}.tmp`
11323
12673
  );
@@ -11326,12 +12676,12 @@ var JudgeCache = class {
11326
12676
  try {
11327
12677
  await rename2(tempPath, filePath);
11328
12678
  } catch (error) {
11329
- await rm3(tempPath, { force: true }).catch(() => void 0);
12679
+ await rm4(tempPath, { force: true }).catch(() => void 0);
11330
12680
  throw error;
11331
12681
  }
11332
12682
  }
11333
12683
  entryPath(key) {
11334
- return path11.join(this.dir, `${key}.json`);
12684
+ return path12.join(this.dir, `${key}.json`);
11335
12685
  }
11336
12686
  };
11337
12687
  function runJudgeWithCache(options) {
@@ -11506,7 +12856,7 @@ function isBenchJudgeResult(value) {
11506
12856
  // src/benchmarks/published/ama-bench/runner.ts
11507
12857
  import { randomUUID as randomUUID2 } from "crypto";
11508
12858
  import { readFile as readFile11 } from "fs/promises";
11509
- import path12 from "path";
12859
+ import path13 from "path";
11510
12860
 
11511
12861
  // src/benchmarks/published/ama-bench/fixture.ts
11512
12862
  var AMA_BENCH_SMOKE_FIXTURE = [
@@ -12081,7 +13431,7 @@ async function loadDataset(mode, datasetDir, limit) {
12081
13431
  return episodes;
12082
13432
  };
12083
13433
  if (datasetDir) {
12084
- const filePath = path12.join(datasetDir, "open_end_qa_set.jsonl");
13434
+ const filePath = path13.join(datasetDir, "open_end_qa_set.jsonl");
12085
13435
  let raw;
12086
13436
  try {
12087
13437
  raw = await readFile11(filePath, "utf8");
@@ -12377,7 +13727,7 @@ function isValidQaPairs(value) {
12377
13727
  // src/benchmarks/published/amemgym/runner.ts
12378
13728
  import { randomUUID as randomUUID3 } from "crypto";
12379
13729
  import { readFile as readFile12 } from "fs/promises";
12380
- import path13 from "path";
13730
+ import path14 from "path";
12381
13731
 
12382
13732
  // src/benchmarks/published/amemgym/fixture.ts
12383
13733
  var AMEMGYM_SMOKE_FIXTURE = [
@@ -12906,7 +14256,7 @@ async function loadDataset2(mode, datasetDir, limit) {
12906
14256
  const datasetErrors = [];
12907
14257
  for (const filename of DATASET_FILENAMES) {
12908
14258
  try {
12909
- const raw = await readFile12(path13.join(datasetDir, filename), "utf8");
14259
+ const raw = await readFile12(path14.join(datasetDir, filename), "utf8");
12910
14260
  const parsed = parseDataset(raw, filename, normalizedLimit);
12911
14261
  return ensureDatasetProfiles(parsed);
12912
14262
  } catch (error) {
@@ -13081,7 +14431,7 @@ function normalizeRole(role) {
13081
14431
  // src/benchmarks/published/memory-arena/runner.ts
13082
14432
  import { randomUUID as randomUUID4 } from "crypto";
13083
14433
  import { readFile as readFile13, readdir as readdir5, stat as stat3 } from "fs/promises";
13084
- import path14 from "path";
14434
+ import path15 from "path";
13085
14435
  import { expandTildePath as expandTildePath2 } from "@remnic/core";
13086
14436
 
13087
14437
  // src/benchmarks/published/memory-arena/fixture.ts
@@ -13408,7 +14758,7 @@ async function loadDataset3(mode, datasetDir, limit) {
13408
14758
  if (remainingLimit2 === 0) {
13409
14759
  break;
13410
14760
  }
13411
- const raw = await readFile13(path14.join(datasetDir, filename), "utf8");
14761
+ const raw = await readFile13(path15.join(datasetDir, filename), "utf8");
13412
14762
  const parsedTasks = [];
13413
14763
  raw.split("\n").forEach((line, lineIndex) => {
13414
14764
  if (line.trim().length === 0) {
@@ -13764,14 +15114,14 @@ async function loadMemoryArenaWebshopProductCatalog(datasetDir) {
13764
15114
  async function resolveMemoryArenaWebshopProductCatalogPath(datasetDir) {
13765
15115
  const configuredPath = process.env[MEMORY_ARENA_WEBSHOP_PRODUCTS_ENV]?.trim();
13766
15116
  if (configuredPath && configuredPath.length > 0) {
13767
- return path14.resolve(expandTildePath2(configuredPath));
15117
+ return path15.resolve(expandTildePath2(configuredPath));
13768
15118
  }
13769
15119
  if (datasetDir === void 0) {
13770
15120
  return void 0;
13771
15121
  }
13772
15122
  const candidatePaths = [
13773
15123
  ...MEMORY_ARENA_WEBSHOP_PRODUCT_SIDECAR_FILENAMES
13774
- ].map((filename) => path14.join(datasetDir, filename));
15124
+ ].map((filename) => path15.join(datasetDir, filename));
13775
15125
  for (const candidatePath of candidatePaths) {
13776
15126
  try {
13777
15127
  const candidateStat = await stat3(candidatePath);
@@ -15194,7 +16544,7 @@ import { collectTemporalLexicalCues } from "@remnic/core";
15194
16544
 
15195
16545
  // src/benchmarks/published/dataset-loader.ts
15196
16546
  import { readFile as readFile14 } from "fs/promises";
15197
- import path15 from "path";
16547
+ import path16 from "path";
15198
16548
 
15199
16549
  // src/benchmarks/published/longmemeval/fixture.ts
15200
16550
  var LONG_MEM_EVAL_SMOKE_FIXTURE = [
@@ -15297,7 +16647,7 @@ async function loadDataset4(options) {
15297
16647
  const errors = [];
15298
16648
  if (options.datasetDir) {
15299
16649
  for (const filename of options.filenames) {
15300
- const abs = path15.join(options.datasetDir, filename);
16650
+ const abs = path16.join(options.datasetDir, filename);
15301
16651
  let raw;
15302
16652
  try {
15303
16653
  raw = await readFile14(abs, "utf8");
@@ -17037,7 +18387,7 @@ function normalizeQaArray(value, location) {
17037
18387
  import { randomUUID as randomUUID6 } from "crypto";
17038
18388
  import { createReadStream as createReadStream2 } from "fs";
17039
18389
  import { readdir as readdir6 } from "fs/promises";
17040
- import path16 from "path";
18390
+ import path17 from "path";
17041
18391
  import { createInterface } from "readline/promises";
17042
18392
  import {
17043
18393
  asyncBufferFromFile,
@@ -17508,8 +18858,8 @@ async function listBeamDatasetFiles(datasetDir) {
17508
18858
  return directFiles;
17509
18859
  }
17510
18860
  try {
17511
- const nestedFilenames = await readdir6(path16.join(datasetDir, "data"));
17512
- return nestedFilenames.filter((filename) => isBeamDatasetFilename(filename)).map((filename) => path16.join("data", filename));
18861
+ const nestedFilenames = await readdir6(path17.join(datasetDir, "data"));
18862
+ return nestedFilenames.filter((filename) => isBeamDatasetFilename(filename)).map((filename) => path17.join("data", filename));
17513
18863
  } catch {
17514
18864
  return [];
17515
18865
  }
@@ -17536,7 +18886,7 @@ async function* iterateDatasetFiles(datasetDir, datasetFiles, limit) {
17536
18886
  let remainingLimit = limit;
17537
18887
  for (const filename of datasetFiles) {
17538
18888
  const scale = inferScaleFromFilename(filename);
17539
- const filePath = path16.join(datasetDir, filename);
18889
+ const filePath = path17.join(datasetDir, filename);
17540
18890
  const conversations = filename.endsWith(".jsonl") ? streamJsonlDataset(filePath, filename, remainingLimit) : filename.endsWith(".parquet") ? streamParquetDataset(filePath, filename, remainingLimit) : streamJsonDataset(filePath, filename, remainingLimit);
17541
18891
  for await (const conversation of conversations) {
17542
18892
  yield {
@@ -18549,7 +19899,7 @@ var StructuredLiteralParser = class {
18549
19899
  // src/benchmarks/published/personamem/runner.ts
18550
19900
  import { createHash as createHash7, randomUUID as randomUUID7 } from "crypto";
18551
19901
  import { readFile as readFile15, realpath as realpath4 } from "fs/promises";
18552
- import path17 from "path";
19902
+ import path18 from "path";
18553
19903
 
18554
19904
  // src/benchmarks/published/personamem/fixture.ts
18555
19905
  var PERSONAMEM_SMOKE_FIXTURE = [
@@ -18825,7 +20175,7 @@ async function loadDataset8(mode, datasetDir, limit) {
18825
20175
  if (datasetDir) {
18826
20176
  const datasetErrors = [];
18827
20177
  for (const relativePath of DATASET_FILE_CANDIDATES) {
18828
- const datasetPath = path17.join(datasetDir, relativePath);
20178
+ const datasetPath = path18.join(datasetDir, relativePath);
18829
20179
  let raw;
18830
20180
  try {
18831
20181
  raw = await readFile15(datasetPath, "utf8");
@@ -19019,12 +20369,12 @@ function parseCsv(raw, limit) {
19019
20369
  return rows;
19020
20370
  }
19021
20371
  async function resolveDatasetFilePath(datasetRoot, relativePath) {
19022
- const rootPath = path17.resolve(datasetRoot);
20372
+ const rootPath = path18.resolve(datasetRoot);
19023
20373
  const rootRealPath = await realpath4(rootPath);
19024
- const candidatePath = path17.resolve(rootPath, relativePath);
20374
+ const candidatePath = path18.resolve(rootPath, relativePath);
19025
20375
  const candidateRealPath = await realpath4(candidatePath);
19026
- const relativeToRoot = path17.relative(rootRealPath, candidateRealPath);
19027
- if (relativeToRoot.startsWith("..") || path17.isAbsolute(relativeToRoot)) {
20376
+ const relativeToRoot = path18.relative(rootRealPath, candidateRealPath);
20377
+ if (relativeToRoot.startsWith("..") || path18.isAbsolute(relativeToRoot)) {
19028
20378
  throw new Error(
19029
20379
  `PersonaMem-v2 dataset file reference "${relativePath}" must stay within datasetDir.`
19030
20380
  );
@@ -19353,7 +20703,7 @@ function applyLimit6(items, limit) {
19353
20703
  // src/benchmarks/published/membench/runner.ts
19354
20704
  import { randomUUID as randomUUID8 } from "crypto";
19355
20705
  import { readFile as readFile16, readdir as readdir7 } from "fs/promises";
19356
- import path18 from "path";
20706
+ import path19 from "path";
19357
20707
 
19358
20708
  // src/benchmarks/published/membench/fixture.ts
19359
20709
  var MEMBENCH_SMOKE_FIXTURE = [
@@ -19614,7 +20964,7 @@ async function loadDataset9(mode, datasetDir, limit) {
19614
20964
  let remainingLimit = normalizedLimit;
19615
20965
  for (const filename of filenames) {
19616
20966
  try {
19617
- const raw = await readFile16(path18.join(datasetDir, filename), "utf8");
20967
+ const raw = await readFile16(path19.join(datasetDir, filename), "utf8");
19618
20968
  const parsed = filename.endsWith(".jsonl") ? parseJsonlDataset(raw, filename) : parseJsonDataset(raw, filename);
19619
20969
  const limitedCases = remainingLimit === 0 ? [] : applyLimit7(parsed, remainingLimit);
19620
20970
  if (limitedCases.length > 0) {
@@ -20482,7 +21832,7 @@ function isPlainObject4(value) {
20482
21832
  // src/benchmarks/published/memoryagentbench/runner.ts
20483
21833
  import { randomUUID as randomUUID9 } from "crypto";
20484
21834
  import { access, readFile as readFile17 } from "fs/promises";
20485
- import path19 from "path";
21835
+ import path20 from "path";
20486
21836
 
20487
21837
  // src/benchmarks/published/memoryagentbench/fixture.ts
20488
21838
  var MEMORY_AGENT_BENCH_SMOKE_FIXTURE = [
@@ -21561,21 +22911,21 @@ function recsysEntityMappingCandidates(datasetDir) {
21561
22911
  if (!datasetDir) {
21562
22912
  return [];
21563
22913
  }
21564
- const absoluteDatasetDir = path19.resolve(datasetDir);
22914
+ const absoluteDatasetDir = path20.resolve(datasetDir);
21565
22915
  const roots = [
21566
22916
  absoluteDatasetDir,
21567
- path19.dirname(absoluteDatasetDir)
22917
+ path20.dirname(absoluteDatasetDir)
21568
22918
  ];
21569
22919
  const canonicalSuffixes = [
21570
- path19.join("processed_data", "Recsys_Redial", "entity2id.json"),
21571
- path19.join("Recsys_Redial", "entity2id.json")
22920
+ path20.join("processed_data", "Recsys_Redial", "entity2id.json"),
22921
+ path20.join("Recsys_Redial", "entity2id.json")
21572
22922
  ];
21573
22923
  const looseSuffixes = ["entity2id.json"];
21574
22924
  return [
21575
22925
  ...roots.flatMap(
21576
- (root) => canonicalSuffixes.map((suffix) => path19.join(root, suffix))
22926
+ (root) => canonicalSuffixes.map((suffix) => path20.join(root, suffix))
21577
22927
  ),
21578
- ...looseSuffixes.map((suffix) => path19.join(absoluteDatasetDir, suffix))
22928
+ ...looseSuffixes.map((suffix) => path20.join(absoluteDatasetDir, suffix))
21579
22929
  ];
21580
22930
  }
21581
22931
  async function fileExists(filePath) {
@@ -21612,7 +22962,7 @@ async function loadDataset10(mode, datasetDir, limit) {
21612
22962
  const datasetErrors = [];
21613
22963
  for (const filename of DATASET_BUNDLE_CANDIDATES) {
21614
22964
  const parsed = await tryReadDatasetFile(
21615
- path19.join(datasetDir, filename),
22965
+ path20.join(datasetDir, filename),
21616
22966
  filename,
21617
22967
  datasetErrors
21618
22968
  );
@@ -21629,7 +22979,7 @@ async function loadDataset10(mode, datasetDir, limit) {
21629
22979
  let splitData;
21630
22980
  for (const filename of splitConfig.candidates) {
21631
22981
  try {
21632
- splitData = await readDatasetFile(path19.join(datasetDir, filename), filename);
22982
+ splitData = await readDatasetFile(path20.join(datasetDir, filename), filename);
21633
22983
  break;
21634
22984
  } catch (error) {
21635
22985
  if (!isFileNotFoundError2(error)) {
@@ -22277,8 +23627,8 @@ function loadCases(mode, limit) {
22277
23627
 
22278
23628
  // src/benchmarks/remnic/extraction-judge-calibration/runner.ts
22279
23629
  import { randomUUID as randomUUID11 } from "crypto";
22280
- import os4 from "os";
22281
- import path20 from "path";
23630
+ import os5 from "os";
23631
+ import path21 from "path";
22282
23632
  import {
22283
23633
  createVerdictCache,
22284
23634
  judgeFactDurability,
@@ -22388,8 +23738,8 @@ var extractionJudgeCalibrationDefinition = {
22388
23738
  async function runExtractionJudgeCalibrationBenchmark(options) {
22389
23739
  const cases = loadCases2(options.mode, options.limit);
22390
23740
  const config = parseConfig2({
22391
- memoryDir: path20.join(os4.tmpdir(), "remnic-bench-extraction-judge"),
22392
- workspaceDir: path20.join(os4.tmpdir(), "remnic-bench-extraction-judge-workspace"),
23741
+ memoryDir: path21.join(os5.tmpdir(), "remnic-bench-extraction-judge"),
23742
+ workspaceDir: path21.join(os5.tmpdir(), "remnic-bench-extraction-judge-workspace"),
22393
23743
  openaiApiKey: "bench-test-key",
22394
23744
  extractionJudgeEnabled: true,
22395
23745
  extractionJudgeBatchSize: 4,
@@ -22937,10 +24287,10 @@ function constantAggregate2(value) {
22937
24287
  }
22938
24288
 
22939
24289
  // src/benchmarks/remnic/entity-consolidation/runner.ts
22940
- import os5 from "os";
22941
- import path21 from "path";
24290
+ import os6 from "os";
24291
+ import path22 from "path";
22942
24292
  import { randomUUID as randomUUID13 } from "crypto";
22943
- import { mkdtemp as mkdtemp3, rm as rm4 } from "fs/promises";
24293
+ import { mkdtemp as mkdtemp4, rm as rm5 } from "fs/promises";
22944
24294
  import { StorageManager } from "@remnic/core";
22945
24295
 
22946
24296
  // src/benchmarks/remnic/entity-consolidation/fixture.ts
@@ -23101,7 +24451,7 @@ function loadCases4(mode, limit) {
23101
24451
  return limited;
23102
24452
  }
23103
24453
  async function executeCase(sample) {
23104
- const tmpDir = await mkdtemp3(path21.join(os5.tmpdir(), "remnic-bench-entity-consolidation-"));
24454
+ const tmpDir = await mkdtemp4(path22.join(os6.tmpdir(), "remnic-bench-entity-consolidation-"));
23105
24455
  try {
23106
24456
  const storage = new StorageManager(tmpDir);
23107
24457
  await storage.ensureDirectories();
@@ -23109,7 +24459,7 @@ async function executeCase(sample) {
23109
24459
  const rawEntity = await storage.readEntity(canonicalName);
23110
24460
  return summarizeEntity(rawEntity, canonicalName);
23111
24461
  } finally {
23112
- await rm4(tmpDir, { recursive: true, force: true });
24462
+ await rm5(tmpDir, { recursive: true, force: true });
23113
24463
  }
23114
24464
  }
23115
24465
  async function applyScenario(storage, sample) {
@@ -23280,9 +24630,9 @@ function parseNonNegativeInt(rawValue) {
23280
24630
 
23281
24631
  // src/benchmarks/remnic/page-versioning/runner.ts
23282
24632
  import { randomUUID as randomUUID14 } from "crypto";
23283
- import { mkdir as mkdir9, mkdtemp as mkdtemp4, readFile as readFile18, rm as rm5, writeFile as writeFile9 } from "fs/promises";
23284
- import os6 from "os";
23285
- import path22 from "path";
24633
+ import { mkdir as mkdir9, mkdtemp as mkdtemp5, readFile as readFile18, rm as rm6, writeFile as writeFile9 } from "fs/promises";
24634
+ import os7 from "os";
24635
+ import path23 from "path";
23286
24636
  import {
23287
24637
  createVersion,
23288
24638
  diffVersions,
@@ -23446,10 +24796,10 @@ function loadCases5(mode, limit) {
23446
24796
  return limited;
23447
24797
  }
23448
24798
  async function executeCase2(sample, dependencies) {
23449
- const tmpDir = await mkdtemp4(path22.join(os6.tmpdir(), "remnic-bench-page-versioning-"));
24799
+ const tmpDir = await mkdtemp5(path23.join(os7.tmpdir(), "remnic-bench-page-versioning-"));
23450
24800
  try {
23451
- const factsDir = path22.join(tmpDir, "facts");
23452
- const pagePath = path22.join(factsDir, `${sample.id}.md`);
24801
+ const factsDir = path23.join(tmpDir, "facts");
24802
+ const pagePath = path23.join(factsDir, `${sample.id}.md`);
23453
24803
  await mkdir9(factsDir, { recursive: true });
23454
24804
  const config = versioningConfig();
23455
24805
  switch (sample.scenario) {
@@ -23530,7 +24880,7 @@ async function executeCase2(sample, dependencies) {
23530
24880
  }
23531
24881
  }
23532
24882
  } finally {
23533
- await rm5(tmpDir, { recursive: true, force: true });
24883
+ await rm6(tmpDir, { recursive: true, force: true });
23534
24884
  }
23535
24885
  }
23536
24886
  function isMissingPageVersionError(error, pagePath, versionId) {
@@ -25803,9 +27153,9 @@ function loadCases9(mode, limit) {
25803
27153
 
25804
27154
  // src/benchmarks/remnic/procedural-recall/runner.ts
25805
27155
  import { randomUUID as randomUUID21 } from "crypto";
25806
- import { mkdtemp as mkdtemp5, rm as rm6 } from "fs/promises";
25807
- import os7 from "os";
25808
- import path23 from "path";
27156
+ import { mkdtemp as mkdtemp6, rm as rm7 } from "fs/promises";
27157
+ import os8 from "os";
27158
+ import path24 from "path";
25809
27159
  import {
25810
27160
  StorageManager as StorageManager2,
25811
27161
  parseConfig as parseConfig3,
@@ -25935,7 +27285,7 @@ async function runProceduralRecallBenchmark(options) {
25935
27285
  }
25936
27286
  for (const sample of e2eCases) {
25937
27287
  const startedAt = performance.now();
25938
- const dir = await mkdtemp5(path23.join(os7.tmpdir(), "remnic-bench-procedural-recall-"));
27288
+ const dir = await mkdtemp6(path24.join(os8.tmpdir(), "remnic-bench-procedural-recall-"));
25939
27289
  let section = null;
25940
27290
  try {
25941
27291
  const storage = new StorageManager2(dir);
@@ -25950,7 +27300,7 @@ ${body}`,
25950
27300
  );
25951
27301
  const config = parseConfig3({
25952
27302
  memoryDir: dir,
25953
- workspaceDir: path23.join(dir, "ws"),
27303
+ workspaceDir: path24.join(dir, "ws"),
25954
27304
  openaiApiKey: "bench-key",
25955
27305
  procedural: {
25956
27306
  enabled: sample.proceduralEnabled !== false,
@@ -25959,7 +27309,7 @@ ${body}`,
25959
27309
  });
25960
27310
  section = await buildProcedureRecallSection(storage, sample.prompt, config);
25961
27311
  } finally {
25962
- await rm6(dir, { recursive: true, force: true });
27312
+ await rm7(dir, { recursive: true, force: true });
25963
27313
  }
25964
27314
  const latencyMs = Math.round(performance.now() - startedAt);
25965
27315
  const nonNull = section !== null && section.length > 0;
@@ -26020,9 +27370,9 @@ ${body}`,
26020
27370
 
26021
27371
  // src/benchmarks/remnic/ingestion-entity-recall/runner.ts
26022
27372
  import { randomUUID as randomUUID22 } from "crypto";
26023
- import { mkdtemp as mkdtemp6, writeFile as writeFile10, rm as rm7, mkdir as mkdir10, realpath as realpath5 } from "fs/promises";
27373
+ import { mkdtemp as mkdtemp7, writeFile as writeFile10, rm as rm8, mkdir as mkdir10, realpath as realpath5 } from "fs/promises";
26024
27374
  import { tmpdir as tmpdir2 } from "os";
26025
- import path24 from "path";
27375
+ import path25 from "path";
26026
27376
 
26027
27377
  // src/ingestion-scorer.ts
26028
27378
  function normalize(value) {
@@ -26524,12 +27874,12 @@ async function runIngestionEntityRecallBenchmark(options) {
26524
27874
  throw new Error("ingestionAdapter is required for ingestion benchmarks");
26525
27875
  }
26526
27876
  const fixture = emailFixture.generate();
26527
- const fixtureDir = await mkdtemp6(path24.join(tmpdir2(), "bench-email-"));
27877
+ const fixtureDir = await mkdtemp7(path25.join(tmpdir2(), "bench-email-"));
26528
27878
  try {
26529
27879
  await options.ingestionAdapter.reset();
26530
27880
  for (const file of fixture.files) {
26531
- const filePath = path24.join(fixtureDir, file.relativePath);
26532
- await mkdir10(path24.dirname(filePath), { recursive: true });
27881
+ const filePath = path25.join(fixtureDir, file.relativePath);
27882
+ await mkdir10(path25.dirname(filePath), { recursive: true });
26533
27883
  await writeFile10(filePath, file.content, "utf8");
26534
27884
  }
26535
27885
  const { result: ingestionLog, durationMs } = await timed(
@@ -26611,7 +27961,7 @@ async function runIngestionEntityRecallBenchmark(options) {
26611
27961
  ];
26612
27962
  return buildResult(options, tasks, durationMs);
26613
27963
  } finally {
26614
- await rm7(fixtureDir, { recursive: true, force: true });
27964
+ await rm8(fixtureDir, { recursive: true, force: true });
26615
27965
  }
26616
27966
  }
26617
27967
  async function buildResult(options, tasks, totalLatencyMs) {
@@ -26657,9 +28007,9 @@ async function buildResult(options, tasks, totalLatencyMs) {
26657
28007
 
26658
28008
  // src/benchmarks/remnic/ingestion-schema-completeness/runner.ts
26659
28009
  import { randomUUID as randomUUID23 } from "crypto";
26660
- import { mkdtemp as mkdtemp7, writeFile as writeFile11, rm as rm8, mkdir as mkdir11, realpath as realpath6 } from "fs/promises";
28010
+ import { mkdtemp as mkdtemp8, writeFile as writeFile11, rm as rm9, mkdir as mkdir11, realpath as realpath6 } from "fs/promises";
26661
28011
  import { tmpdir as tmpdir3 } from "os";
26662
- import path25 from "path";
28012
+ import path26 from "path";
26663
28013
  var ingestionSchemaCompletenessDefinition = {
26664
28014
  id: "ingestion-schema-completeness",
26665
28015
  title: "Ingestion: Schema Completeness",
@@ -26678,12 +28028,12 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
26678
28028
  throw new Error("ingestionAdapter is required for ingestion benchmarks");
26679
28029
  }
26680
28030
  const fixture = emailFixture.generate();
26681
- const fixtureDir = await mkdtemp7(path25.join(tmpdir3(), "bench-email-"));
28031
+ const fixtureDir = await mkdtemp8(path26.join(tmpdir3(), "bench-email-"));
26682
28032
  try {
26683
28033
  await options.ingestionAdapter.reset();
26684
28034
  for (const file of fixture.files) {
26685
- const filePath = path25.join(fixtureDir, file.relativePath);
26686
- await mkdir11(path25.dirname(filePath), { recursive: true });
28035
+ const filePath = path26.join(fixtureDir, file.relativePath);
28036
+ await mkdir11(path26.dirname(filePath), { recursive: true });
26687
28037
  await writeFile11(filePath, file.content, "utf8");
26688
28038
  }
26689
28039
  const { result: ingestionLog, durationMs } = await timed(
@@ -26824,15 +28174,15 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
26824
28174
  }
26825
28175
  };
26826
28176
  } finally {
26827
- await rm8(fixtureDir, { recursive: true, force: true });
28177
+ await rm9(fixtureDir, { recursive: true, force: true });
26828
28178
  }
26829
28179
  }
26830
28180
 
26831
28181
  // src/benchmarks/remnic/ingestion-backlink-f1/runner.ts
26832
28182
  import { randomUUID as randomUUID24 } from "crypto";
26833
- import { mkdtemp as mkdtemp8, writeFile as writeFile12, rm as rm9, mkdir as mkdir12, realpath as realpath7 } from "fs/promises";
28183
+ import { mkdtemp as mkdtemp9, writeFile as writeFile12, rm as rm10, mkdir as mkdir12, realpath as realpath7 } from "fs/promises";
26834
28184
  import { tmpdir as tmpdir4 } from "os";
26835
- import path26 from "path";
28185
+ import path27 from "path";
26836
28186
  var ingestionBacklinkF1Definition = {
26837
28187
  id: "ingestion-backlink-f1",
26838
28188
  title: "Ingestion: Backlink F1",
@@ -26851,12 +28201,12 @@ async function runIngestionBacklinkF1Benchmark(options) {
26851
28201
  throw new Error("ingestionAdapter is required for ingestion benchmarks");
26852
28202
  }
26853
28203
  const fixture = emailFixture.generate();
26854
- const fixtureDir = await mkdtemp8(path26.join(tmpdir4(), "bench-email-"));
28204
+ const fixtureDir = await mkdtemp9(path27.join(tmpdir4(), "bench-email-"));
26855
28205
  try {
26856
28206
  await options.ingestionAdapter.reset();
26857
28207
  for (const file of fixture.files) {
26858
- const filePath = path26.join(fixtureDir, file.relativePath);
26859
- await mkdir12(path26.dirname(filePath), { recursive: true });
28208
+ const filePath = path27.join(fixtureDir, file.relativePath);
28209
+ await mkdir12(path27.dirname(filePath), { recursive: true });
26860
28210
  await writeFile12(filePath, file.content, "utf8");
26861
28211
  }
26862
28212
  const { result: ingestionLog, durationMs } = await timed(
@@ -26925,15 +28275,15 @@ async function runIngestionBacklinkF1Benchmark(options) {
26925
28275
  }
26926
28276
  };
26927
28277
  } finally {
26928
- await rm9(fixtureDir, { recursive: true, force: true });
28278
+ await rm10(fixtureDir, { recursive: true, force: true });
26929
28279
  }
26930
28280
  }
26931
28281
 
26932
28282
  // src/benchmarks/remnic/ingestion-setup-friction/runner.ts
26933
28283
  import { randomUUID as randomUUID25 } from "crypto";
26934
- import { mkdtemp as mkdtemp9, writeFile as writeFile13, rm as rm10, mkdir as mkdir13, realpath as realpath8 } from "fs/promises";
28284
+ import { mkdtemp as mkdtemp10, writeFile as writeFile13, rm as rm11, mkdir as mkdir13, realpath as realpath8 } from "fs/promises";
26935
28285
  import { tmpdir as tmpdir5 } from "os";
26936
- import path27 from "path";
28286
+ import path28 from "path";
26937
28287
  var INGESTION_SETUP_FRICTION_LOWER_IS_BETTER = /* @__PURE__ */ new Set(["setup_friction", "commands_count", "prompts_count", "errors_count"]);
26938
28288
  var ingestionSetupFrictionDefinition = {
26939
28289
  id: "ingestion-setup-friction",
@@ -26953,12 +28303,12 @@ async function runIngestionSetupFrictionBenchmark(options) {
26953
28303
  throw new Error("ingestionAdapter is required for ingestion benchmarks");
26954
28304
  }
26955
28305
  const fixture = emailFixture.generate();
26956
- const fixtureDir = await mkdtemp9(path27.join(tmpdir5(), "bench-friction-"));
28306
+ const fixtureDir = await mkdtemp10(path28.join(tmpdir5(), "bench-friction-"));
26957
28307
  try {
26958
28308
  await options.ingestionAdapter.reset();
26959
28309
  for (const file of fixture.files) {
26960
- const filePath = path27.join(fixtureDir, file.relativePath);
26961
- await mkdir13(path27.dirname(filePath), { recursive: true });
28310
+ const filePath = path28.join(fixtureDir, file.relativePath);
28311
+ await mkdir13(path28.dirname(filePath), { recursive: true });
26962
28312
  await writeFile13(filePath, file.content, "utf8");
26963
28313
  }
26964
28314
  const { result: ingestionLog, durationMs } = await timed(
@@ -27031,15 +28381,15 @@ async function runIngestionSetupFrictionBenchmark(options) {
27031
28381
  }
27032
28382
  };
27033
28383
  } finally {
27034
- await rm10(fixtureDir, { recursive: true, force: true });
28384
+ await rm11(fixtureDir, { recursive: true, force: true });
27035
28385
  }
27036
28386
  }
27037
28387
 
27038
28388
  // src/benchmarks/remnic/ingestion-citation-accuracy/runner.ts
27039
28389
  import { randomUUID as randomUUID26 } from "crypto";
27040
- import { mkdtemp as mkdtemp10, writeFile as writeFile14, rm as rm11, mkdir as mkdir14, realpath as realpath9 } from "fs/promises";
28390
+ import { mkdtemp as mkdtemp11, writeFile as writeFile14, rm as rm12, mkdir as mkdir14, realpath as realpath9 } from "fs/promises";
27041
28391
  import { tmpdir as tmpdir6 } from "os";
27042
- import path28 from "path";
28392
+ import path29 from "path";
27043
28393
  var CITATION_SUPPORT_THRESHOLD = 0.72;
27044
28394
  var ingestionCitationAccuracyDefinition = {
27045
28395
  id: "ingestion-citation-accuracy",
@@ -27098,10 +28448,10 @@ function resolveCitedSources(sourceRefs, seeAlso, pageRef, sourceContentMap) {
27098
28448
  return "";
27099
28449
  }
27100
28450
  for (const ref of normalizedRefs) {
27101
- const refBase = path28.basename(ref).toLowerCase();
28451
+ const refBase = path29.basename(ref).toLowerCase();
27102
28452
  let matched = false;
27103
28453
  for (const [relativePath, content] of sourceContentMap) {
27104
- if (relativePath === ref || relativePath.endsWith(ref) || path28.basename(relativePath).toLowerCase() === refBase) {
28454
+ if (relativePath === ref || relativePath.endsWith(ref) || path29.basename(relativePath).toLowerCase() === refBase) {
27105
28455
  resolved.push(content);
27106
28456
  matched = true;
27107
28457
  break;
@@ -27117,9 +28467,9 @@ function resolveCitedSources(sourceRefs, seeAlso, pageRef, sourceContentMap) {
27117
28467
  if (normalizedRefs.length > 0) {
27118
28468
  return "";
27119
28469
  }
27120
- const pageBase = path28.basename(pageRef).toLowerCase();
28470
+ const pageBase = path29.basename(pageRef).toLowerCase();
27121
28471
  for (const [relativePath, content] of sourceContentMap) {
27122
- if (path28.basename(relativePath).toLowerCase() === pageBase) {
28472
+ if (path29.basename(relativePath).toLowerCase() === pageBase) {
27123
28473
  return content;
27124
28474
  }
27125
28475
  }
@@ -27130,12 +28480,12 @@ async function runIngestionCitationAccuracyBenchmark(options) {
27130
28480
  throw new Error("ingestionAdapter is required for ingestion benchmarks");
27131
28481
  }
27132
28482
  const fixture = emailFixture.generate();
27133
- const fixtureDir = await mkdtemp10(path28.join(tmpdir6(), "bench-citation-"));
28483
+ const fixtureDir = await mkdtemp11(path29.join(tmpdir6(), "bench-citation-"));
27134
28484
  try {
27135
28485
  await options.ingestionAdapter.reset();
27136
28486
  for (const file of fixture.files) {
27137
- const filePath = path28.join(fixtureDir, file.relativePath);
27138
- await mkdir14(path28.dirname(filePath), { recursive: true });
28487
+ const filePath = path29.join(fixtureDir, file.relativePath);
28488
+ await mkdir14(path29.dirname(filePath), { recursive: true });
27139
28489
  await writeFile14(filePath, file.content, "utf8");
27140
28490
  }
27141
28491
  const benchmarkStart = performance.now();
@@ -27333,7 +28683,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
27333
28683
  }
27334
28684
  };
27335
28685
  } finally {
27336
- await rm11(fixtureDir, { recursive: true, force: true });
28686
+ await rm12(fixtureDir, { recursive: true, force: true });
27337
28687
  }
27338
28688
  }
27339
28689
  function citationSupportScore(claim, citedSources) {
@@ -27521,7 +28871,7 @@ var ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS = ASSISTANT_MORNING_BRIEF_SCENARIOS.
27521
28871
 
27522
28872
  // src/benchmarks/remnic/_assistant-common/runner.ts
27523
28873
  import { randomUUID as randomUUID27 } from "crypto";
27524
- import path30 from "path";
28874
+ import path31 from "path";
27525
28875
 
27526
28876
  // src/run-seeds.ts
27527
28877
  function buildBenchmarkRunSeeds(runCount, baseSeed) {
@@ -27613,7 +28963,7 @@ function pairedDeltaConfidenceInterval(candidateValues, baselineValues, options
27613
28963
  // src/judges/sealed-rubric.ts
27614
28964
  import { createHash as createHash8 } from "crypto";
27615
28965
  import { appendFileSync, mkdirSync } from "fs";
27616
- import path29 from "path";
28966
+ import path30 from "path";
27617
28967
 
27618
28968
  // src/judges/sealed-prompts/assistant-rubric-v1.ts
27619
28969
  var ASSISTANT_RUBRIC_V1 = `# Assistant rubric v1 (sealed)
@@ -27893,7 +29243,7 @@ function createSpotCheckFileLogger(options) {
27893
29243
  return { log() {
27894
29244
  } };
27895
29245
  }
27896
- const logPath = path29.join(directory, `${runId}.jsonl`);
29246
+ const logPath = path30.join(directory, `${runId}.jsonl`);
27897
29247
  let written = 0;
27898
29248
  let warnedOnWriteFailure = false;
27899
29249
  const cap = typeof sampleSize === "number" && sampleSize > 0 ? sampleSize : 5;
@@ -27982,7 +29332,7 @@ async function runAssistantBenchmark(definition, scenarios, resolved, runnerOpti
27982
29332
  const runId = buildRunId(definition.id);
27983
29333
  const spotCheckLogger = createSpotCheckFileLogger({
27984
29334
  runId,
27985
- directory: runnerOptions.spotCheckDir ?? path30.join(process.cwd(), "benchmarks", "results", "spot-checks"),
29335
+ directory: runnerOptions.spotCheckDir ?? path31.join(process.cwd(), "benchmarks", "results", "spot-checks"),
27986
29336
  sampleRate: 0.35,
27987
29337
  sampleSize: 5
27988
29338
  });
@@ -28631,9 +29981,9 @@ async function runAssistantSynthesisBenchmark(options) {
28631
29981
 
28632
29982
  // src/benchmarks/remnic/buffer-surprise-trigger/runner.ts
28633
29983
  import { randomUUID as randomUUID28 } from "crypto";
28634
- import path31 from "path";
28635
- import os8 from "os";
28636
- import { mkdir as mkdir15, rm as rm12 } from "fs/promises";
29984
+ import path32 from "path";
29985
+ import os9 from "os";
29986
+ import { mkdir as mkdir15, rm as rm13 } from "fs/promises";
28637
29987
  import {
28638
29988
  SmartBuffer,
28639
29989
  computeSurprise,
@@ -28862,8 +30212,8 @@ function hasExplicitTopicPivotCue(text) {
28862
30212
  }
28863
30213
  async function runBufferSurpriseTriggerBenchmark(options) {
28864
30214
  const cases = loadCases10(options.mode, options.limit);
28865
- const tmpRoot = path31.join(
28866
- os8.tmpdir(),
30215
+ const tmpRoot = path32.join(
30216
+ os9.tmpdir(),
28867
30217
  `remnic-bench-buffer-surprise-${randomUUID28()}`
28868
30218
  );
28869
30219
  await mkdir15(tmpRoot, { recursive: true });
@@ -28884,7 +30234,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
28884
30234
  tasks.push(buildTaskResult(caseDef, control, candidate));
28885
30235
  }
28886
30236
  } finally {
28887
- await rm12(tmpRoot, { recursive: true, force: true });
30237
+ await rm13(tmpRoot, { recursive: true, force: true });
28888
30238
  }
28889
30239
  const totalLatencyMs = Math.round(performance.now() - startedAt);
28890
30240
  const aggregates = buildAggregates2(tasks);
@@ -28931,11 +30281,11 @@ async function runBufferSurpriseTriggerBenchmark(options) {
28931
30281
  };
28932
30282
  }
28933
30283
  async function runSingleCase(caseDef, options) {
28934
- const memoryDir = path31.join(
30284
+ const memoryDir = path32.join(
28935
30285
  options.tmpRoot,
28936
30286
  `${caseDef.id}-${options.label}`
28937
30287
  );
28938
- const workspaceDir = path31.join(memoryDir, "workspace");
30288
+ const workspaceDir = path32.join(memoryDir, "workspace");
28939
30289
  await mkdir15(workspaceDir, { recursive: true });
28940
30290
  const config = parseConfig4({
28941
30291
  memoryDir,
@@ -30978,7 +32328,7 @@ async function runMemCorrectBenchmark(options) {
30978
32328
  // src/benchmarks/remnic/bounded-memory-contracts/runner.ts
30979
32329
  import { randomUUID as randomUUID32 } from "crypto";
30980
32330
  import { mkdir as mkdir16, writeFile as writeFile15 } from "fs/promises";
30981
- import path32 from "path";
32331
+ import path33 from "path";
30982
32332
 
30983
32333
  // src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
30984
32334
  import { createHash as createHash10 } from "crypto";
@@ -31463,17 +32813,7 @@ var BOUNDED_MEMORY_SMOKE_FIXTURE = [
31463
32813
  ];
31464
32814
  function fixtureHash(tasks) {
31465
32815
  const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
31466
- const payload = source.map(
31467
- (t) => [
31468
- t.id,
31469
- t.family,
31470
- t.expectedAnswer,
31471
- t.scope,
31472
- t.shouldAsk === void 0 ? "-" : String(t.shouldAsk),
31473
- t.memoryItems.map((m) => `${m.id}:${m.status}:${m.scope}`).join(","),
31474
- t.skills.map((s) => s.id).join(",")
31475
- ].join("|")
31476
- ).join("\n");
32816
+ const payload = JSON.stringify(source);
31477
32817
  return createHash10("sha256").update(payload, "utf8").digest("hex");
31478
32818
  }
31479
32819
 
@@ -31869,7 +33209,7 @@ function aggregateCondition(condition, scored, skillLog) {
31869
33209
  const tp = injected.filter((e) => e.outcome === "helped").length;
31870
33210
  const fp = injected.filter((e) => e.outcome === "harmed").length;
31871
33211
  const notInjected = considered.filter((e) => !e.injected);
31872
- const fn = notInjected.filter((e) => e.outcome === "harmed").length;
33212
+ const fn = skillLog.filter((e) => !e.injected && e.outcome === "harmed").length;
31873
33213
  const tn = notInjected.filter((e) => e.outcome === "irrelevant").length;
31874
33214
  return {
31875
33215
  condition,
@@ -32220,23 +33560,23 @@ async function runBoundedMemoryContractsBenchmark(options) {
32220
33560
  };
32221
33561
  }
32222
33562
  async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks) {
32223
- const root = path32.resolve(outputDir);
32224
- await mkdir16(path32.join(root, "conditions"), { recursive: true });
32225
- await mkdir16(path32.join(root, "prompts"), { recursive: true });
32226
- await mkdir16(path32.join(root, "retrieval"), { recursive: true });
32227
- await mkdir16(path32.join(root, "scores"), { recursive: true });
33563
+ const root = path33.resolve(outputDir);
33564
+ await mkdir16(path33.join(root, "conditions"), { recursive: true });
33565
+ await mkdir16(path33.join(root, "prompts"), { recursive: true });
33566
+ await mkdir16(path33.join(root, "retrieval"), { recursive: true });
33567
+ await mkdir16(path33.join(root, "scores"), { recursive: true });
32228
33568
  const csvRows = [
32229
33569
  "task_id,condition,family,scope,task_success,should_ask_accuracy,relevant_memory_recall,stale_memory_harm_rate,wrong_scope_retrieval_rate,supersession_respected_rate,citation_coverage,memory_tokens_injected,retrieved_item_count,compression_ratio_vs_raw_transcript"
32230
33570
  ];
32231
33571
  for (const condition of BOUNDED_MEMORY_CONDITIONS) {
32232
33572
  const results = byCondition.get(condition);
32233
- const condDir = path32.join(root, "conditions", condition);
33573
+ const condDir = path33.join(root, "conditions", condition);
32234
33574
  await mkdir16(condDir, { recursive: true });
32235
33575
  for (const { task, pack, decision } of results) {
32236
33576
  const scores = scoreTaskPair(task, pack, decision);
32237
33577
  const promptMd = renderPromptPack(task, condition, pack);
32238
- const promptPath = path32.join(root, "prompts", `${task.id}.${condition}.md`);
32239
- await mkdir16(path32.dirname(promptPath), { recursive: true });
33578
+ const promptPath = path33.join(root, "prompts", `${task.id}.${condition}.md`);
33579
+ await mkdir16(path33.dirname(promptPath), { recursive: true });
32240
33580
  await writeFile15(promptPath, promptMd, "utf8");
32241
33581
  const retrievalJson = `${JSON.stringify(
32242
33582
  {
@@ -32260,7 +33600,7 @@ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks
32260
33600
  2
32261
33601
  )}
32262
33602
  `;
32263
- const retrievalPath = path32.join(root, "retrieval", `${task.id}.${condition}.json`);
33603
+ const retrievalPath = path33.join(root, "retrieval", `${task.id}.${condition}.json`);
32264
33604
  await writeFile15(retrievalPath, retrievalJson, "utf8");
32265
33605
  csvRows.push(
32266
33606
  [
@@ -32282,23 +33622,23 @@ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks
32282
33622
  );
32283
33623
  }
32284
33624
  await writeFile15(
32285
- path32.join(condDir, "summary.json"),
33625
+ path33.join(condDir, "summary.json"),
32286
33626
  `${JSON.stringify(conditionAggregates[condition], null, 2)}
32287
33627
  `,
32288
33628
  "utf8"
32289
33629
  );
32290
33630
  }
32291
- await writeFile15(path32.join(root, "scores", "per-task.csv"), `${csvRows.join("\n")}
33631
+ await writeFile15(path33.join(root, "scores", "per-task.csv"), `${csvRows.join("\n")}
32292
33632
  `, "utf8");
32293
33633
  await writeFile15(
32294
- path32.join(root, "scores", "aggregate.json"),
33634
+ path33.join(root, "scores", "aggregate.json"),
32295
33635
  `${JSON.stringify(conditionAggregates, null, 2)}
32296
33636
  `,
32297
33637
  "utf8"
32298
33638
  );
32299
33639
  const report = renderReportMarkdown(tasks, conditionAggregates);
32300
- await writeFile15(path32.join(root, "report.md"), report, "utf8");
32301
- return path32.join(root, "report.md");
33640
+ await writeFile15(path33.join(root, "report.md"), report, "utf8");
33641
+ return path33.join(root, "report.md");
32302
33642
  }
32303
33643
  function renderPromptPack(task, condition, pack) {
32304
33644
  const lines = [];
@@ -32523,8 +33863,8 @@ function finalizeBenchmarkResultConfig(result, options) {
32523
33863
  }
32524
33864
 
32525
33865
  // src/benchmark.ts
32526
- var DEFAULT_BASELINE_PATH = path33.join(process.cwd(), "benchmarks", "baseline.json");
32527
- var DEFAULT_REPORT_PATH = path33.join(process.cwd(), "benchmarks", "report.json");
33866
+ var DEFAULT_BASELINE_PATH = path34.join(process.cwd(), "benchmarks", "baseline.json");
33867
+ var DEFAULT_REPORT_PATH = path34.join(process.cwd(), "benchmarks", "report.json");
32528
33868
  var BASELINE_VERSION = 1;
32529
33869
  var DEFAULT_TOLERANCE = 10;
32530
33870
  var DEFAULT_FULL_RUN_COUNT = 5;
@@ -32602,7 +33942,7 @@ async function runBenchmark(benchmarkId, options) {
32602
33942
  if (!willWrapPrimary && !willWrapCross) {
32603
33943
  return void 0;
32604
33944
  }
32605
- const cacheDir = options.judgeCacheDir ? path33.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path33.join(path33.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
33945
+ const cacheDir = options.judgeCacheDir ? path34.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path34.join(path34.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
32606
33946
  if (cacheDir === void 0) {
32607
33947
  return void 0;
32608
33948
  }
@@ -32791,7 +34131,7 @@ function loadBaseline(baselinePath) {
32791
34131
  return raw;
32792
34132
  }
32793
34133
  function saveBaseline(baselinePath, baseline) {
32794
- fs2.mkdirSync(path33.dirname(baselinePath), { recursive: true });
34134
+ fs2.mkdirSync(path34.dirname(baselinePath), { recursive: true });
32795
34135
  fs2.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}
32796
34136
  `);
32797
34137
  }
@@ -33021,7 +34361,7 @@ function generateReport(results, reportPath) {
33021
34361
  totalDurationMs: results.reduce((sum, result) => sum + result.totalDurationMs, 0)
33022
34362
  };
33023
34363
  if (reportPath) {
33024
- fs2.mkdirSync(path33.dirname(reportPath), { recursive: true });
34364
+ fs2.mkdirSync(path34.dirname(reportPath), { recursive: true });
33025
34365
  fs2.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}
33026
34366
  `);
33027
34367
  }
@@ -33487,7 +34827,7 @@ function formatError(error) {
33487
34827
 
33488
34828
  // src/benchmarks/custom/runner.ts
33489
34829
  import { randomUUID as randomUUID33 } from "crypto";
33490
- import path34 from "path";
34830
+ import path35 from "path";
33491
34831
  import { expandTildePath as expandTildePath4 } from "@remnic/core";
33492
34832
  async function runCustomBenchmarkFile(filePath, options) {
33493
34833
  const spec = await loadCustomBenchmarkFile(filePath);
@@ -33500,7 +34840,7 @@ async function runCustomBenchmarkFile(filePath, options) {
33500
34840
  let cacheRestore;
33501
34841
  let cacheCounters;
33502
34842
  if (spec.scoring === "llm_judge" && runOptions.system.judge !== void 0 && !runOptions.noJudgeCache && (runOptions.judgeProvider ?? null) !== null) {
33503
- const cacheDir = runOptions.judgeCacheDir ? path34.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path34.join(path34.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
34843
+ const cacheDir = runOptions.judgeCacheDir ? path35.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path35.join(path35.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
33504
34844
  if (cacheDir !== void 0) {
33505
34845
  const originalJudge = runOptions.system.judge;
33506
34846
  const wrapped = wrapJudgeWithCache({
@@ -33701,7 +35041,7 @@ async function scoreTask(scoring, options, question, actual, expected) {
33701
35041
  }
33702
35042
  }
33703
35043
  function createCustomBenchmarkDefinition(benchmark, filePath) {
33704
- const id = `custom:${slugify(path34.basename(filePath, path34.extname(filePath)) || benchmark.name)}`;
35044
+ const id = `custom:${slugify(path35.basename(filePath, path35.extname(filePath)) || benchmark.name)}`;
33705
35045
  return {
33706
35046
  id,
33707
35047
  title: benchmark.name,
@@ -34573,7 +35913,7 @@ var chatFixture = {
34573
35913
  // src/judges/calibration-slice.ts
34574
35914
  import { createHash as createHash12, randomBytes as randomBytes3 } from "crypto";
34575
35915
  import { mkdir as mkdir17, readFile as readFile21, rename as rename3, unlink as unlink3, writeFile as writeFile16 } from "fs/promises";
34576
- import path35 from "path";
35916
+ import path36 from "path";
34577
35917
 
34578
35918
  // src/judges/cohen-kappa.ts
34579
35919
  function computeCohensKappa(raterA, raterB) {
@@ -34710,7 +36050,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
34710
36050
  warning: result.warning,
34711
36051
  ...identities ? identities : {}
34712
36052
  };
34713
- const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
36053
+ const filePath = path36.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
34714
36054
  const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
34715
36055
  await writeFile16(tempPath, `${JSON.stringify(state, null, 2)}
34716
36056
  `, "utf8");
@@ -34723,7 +36063,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
34723
36063
  return filePath;
34724
36064
  }
34725
36065
  async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
34726
- const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
36066
+ const filePath = path36.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
34727
36067
  let raw;
34728
36068
  try {
34729
36069
  raw = await readFile21(filePath, "utf8");
@@ -34797,9 +36137,9 @@ function sanitizeCalibrationSegment(value) {
34797
36137
  }
34798
36138
 
34799
36139
  // src/benchmarks/remnic/procedural-recall/ablation.ts
34800
- import { mkdir as mkdir18, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile17, readFile as readFile22 } from "fs/promises";
34801
- import os9 from "os";
34802
- import path36 from "path";
36140
+ import { mkdir as mkdir18, mkdtemp as mkdtemp12, rm as rm14, writeFile as writeFile17, readFile as readFile22 } from "fs/promises";
36141
+ import os10 from "os";
36142
+ import path37 from "path";
34803
36143
  import {
34804
36144
  StorageManager as StorageManager3,
34805
36145
  parseConfig as parseConfig5,
@@ -34829,8 +36169,8 @@ function fixtureToAblationScenarios(fixture) {
34829
36169
  async function runSide(scenarios, proceduralEnabled) {
34830
36170
  const observed = [];
34831
36171
  for (const scenario of scenarios) {
34832
- const dir = await mkdtemp11(
34833
- path36.join(os9.tmpdir(), "remnic-bench-proc-ablation-")
36172
+ const dir = await mkdtemp12(
36173
+ path37.join(os10.tmpdir(), "remnic-bench-proc-ablation-")
34834
36174
  );
34835
36175
  try {
34836
36176
  const storage = new StorageManager3(dir);
@@ -34845,7 +36185,7 @@ ${body}`,
34845
36185
  );
34846
36186
  const config = parseConfig5({
34847
36187
  memoryDir: dir,
34848
- workspaceDir: path36.join(dir, "ws"),
36188
+ workspaceDir: path37.join(dir, "ws"),
34849
36189
  openaiApiKey: "bench-key",
34850
36190
  procedural: {
34851
36191
  enabled: proceduralEnabled,
@@ -34859,7 +36199,7 @@ ${body}`,
34859
36199
  );
34860
36200
  observed.push(section !== null && section.length > 0);
34861
36201
  } finally {
34862
- await rm13(dir, { recursive: true, force: true });
36202
+ await rm14(dir, { recursive: true, force: true });
34863
36203
  }
34864
36204
  }
34865
36205
  return observed;
@@ -35016,12 +36356,61 @@ async function runProceduralAblationCli(args) {
35016
36356
  random: args.random,
35017
36357
  seed: args.seed
35018
36358
  });
35019
- const outDir = path36.dirname(path36.resolve(args.outPath));
36359
+ const outDir = path37.dirname(path37.resolve(args.outPath));
35020
36360
  await mkdir18(outDir, { recursive: true });
35021
36361
  await writeFile17(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
35022
36362
  return artifact;
35023
36363
  }
35024
36364
 
36365
+ // src/ablations/single-flag-matrix.ts
36366
+ var SINGLE_FLAG_ABLATION_MATRIX = [
36367
+ {
36368
+ id: "memory-worth-off",
36369
+ label: "Memory Worth multiplier OFF",
36370
+ axis: "memory-worth",
36371
+ description: "Disables the Memory Worth recall multiplier (recallMemoryWorthFilterEnabled=false). Baseline (#1574) ran with it ON via the config.ts default; this cell measures the cost of removing it.",
36372
+ baselineState: "recallMemoryWorthFilterEnabled defaults true in config.ts; baseline ran ON.",
36373
+ configOverrides: {
36374
+ recallMemoryWorthFilterEnabled: false
36375
+ },
36376
+ primaryFlag: "recallMemoryWorthFilterEnabled"
36377
+ },
36378
+ {
36379
+ id: "contradiction-scan-on",
36380
+ label: "Contradiction scan ON",
36381
+ axis: "contradiction-scan",
36382
+ description: "Enables inline contradiction detection on the write path (contradictionDetectionEnabled=true) so supersessions land at ingest time, before answering. The batch contradictionScan cron is a no-op during a bench replay (it only registers a scheduled job), so the measurable axis is the inline write-path gate (orchestrator \xA715667); contradictionAutoResolve defaults true so detected contradictions are applied. Baseline (#1574) ran with it OFF.",
36383
+ baselineState: "contradictionDetectionEnabled defaults false (config.ts \xA72078); baseline ran OFF.",
36384
+ configOverrides: {
36385
+ contradictionDetectionEnabled: true
36386
+ },
36387
+ primaryFlag: "contradictionDetectionEnabled"
36388
+ },
36389
+ {
36390
+ id: "graph-recall-on",
36391
+ label: "Graph / temporal recall ON",
36392
+ axis: "graph-recall",
36393
+ description: "Enables graph recall + full-mode graph assist (graphRecallEnabled=true, graphAssistInFullModeEnabled=true, multiGraphMemoryEnabled=true). orchestrator.ts \xA71379 skips graph_mode unless BOTH graphRecallEnabled AND multiGraphMemoryEnabled are set, so the cell carries both gates (plus the full-mode assist) or it measures nothing. Baseline (#1574) ran with graph recall OFF; this cell measures the benefit of causal/timeline expansion.",
36394
+ baselineState: "graphRecallEnabled, graphAssistInFullModeEnabled, multiGraphMemoryEnabled all default false; baseline ran OFF.",
36395
+ configOverrides: {
36396
+ graphRecallEnabled: true,
36397
+ graphAssistInFullModeEnabled: true,
36398
+ multiGraphMemoryEnabled: true
36399
+ },
36400
+ primaryFlag: "graphRecallEnabled"
36401
+ }
36402
+ ];
36403
+ var DEFAULT_ABLATION_BENCHMARK = "locomo";
36404
+ function getAblationCell(id) {
36405
+ const cell = SINGLE_FLAG_ABLATION_MATRIX.find((c) => c.id === id);
36406
+ if (!cell) {
36407
+ throw new Error(
36408
+ `Unknown ablation cell id "${id}". Known ids: ${SINGLE_FLAG_ABLATION_MATRIX.map((c) => c.id).join(", ")}.`
36409
+ );
36410
+ }
36411
+ return cell;
36412
+ }
36413
+
35025
36414
  // src/benchmarks/remnic/procedural-recall/real-scenarios.ts
35026
36415
  var PROCEDURAL_REAL_SCENARIOS = [
35027
36416
  // --- exact-re-run (5) ---------------------------------------------------
@@ -36207,11 +37596,11 @@ function pickStableQualifiedName(repo, index) {
36207
37596
 
36208
37597
  // src/coding-graph/harness.ts
36209
37598
  import { performance as performance2 } from "perf_hooks";
36210
- import { mkdtemp as mkdtemp12, rm as rm14 } from "fs/promises";
37599
+ import { mkdtemp as mkdtemp13, rm as rm15 } from "fs/promises";
36211
37600
  import { statSync } from "fs";
36212
37601
  import { tmpdir as tmpdir7 } from "os";
36213
- import path37 from "path";
36214
- import os10 from "os";
37602
+ import path38 from "path";
37603
+ import os11 from "os";
36215
37604
  import {
36216
37605
  GraphStore
36217
37606
  } from "@remnic/coding-graph";
@@ -36237,14 +37626,14 @@ var CODING_GRAPH_BENCH_SCHEMA_VERSION = 2;
36237
37626
 
36238
37627
  // src/coding-graph/harness.ts
36239
37628
  function captureMachineFingerprint() {
36240
- const cpus = os10.cpus();
37629
+ const cpus = os11.cpus();
36241
37630
  return {
36242
37631
  arch: process.arch,
36243
37632
  platform: process.platform,
36244
37633
  nodeVersion: process.version,
36245
37634
  cpuModel: cpus.length > 0 ? cpus[0].model : null,
36246
37635
  cpuCores: cpus.length,
36247
- totalMemoryMb: Math.round(os10.totalmem() / (1024 * 1024))
37636
+ totalMemoryMb: Math.round(os11.totalmem() / (1024 * 1024))
36248
37637
  };
36249
37638
  }
36250
37639
  function percentile2(sorted, p) {
@@ -36307,8 +37696,8 @@ async function runCodingGraphBenchmark(config = {}) {
36307
37696
  const sampleRss = () => {
36308
37697
  peakRss = Math.max(peakRss, process.memoryUsage().rss);
36309
37698
  };
36310
- const dir = await mkdtemp12(path37.join(tmpdir7(), "coding-graph-bench-"));
36311
- const dbPath = path37.join(dir, "bench.sqlite");
37699
+ const dir = await mkdtemp13(path38.join(tmpdir7(), "coding-graph-bench-"));
37700
+ const dbPath = path38.join(dir, "bench.sqlite");
36312
37701
  try {
36313
37702
  const store = await GraphStore.open({ dbPath });
36314
37703
  try {
@@ -36444,7 +37833,7 @@ async function runCodingGraphBenchmark(config = {}) {
36444
37833
  await store.close();
36445
37834
  }
36446
37835
  } finally {
36447
- await rm14(dir, { recursive: true, force: true });
37836
+ await rm15(dir, { recursive: true, force: true });
36448
37837
  }
36449
37838
  }
36450
37839
 
@@ -36652,6 +38041,7 @@ export {
36652
38041
  DEFAULT_TOLERANCE_PERCENT as CODING_GRAPH_DEFAULT_TOLERANCE,
36653
38042
  MIN_ITERATIONS as CODING_GRAPH_MIN_ITERATIONS,
36654
38043
  DEFAULT_SMOKE_FIXTURE as CODING_GRAPH_SMOKE_FIXTURE,
38044
+ DEFAULT_ABLATION_BENCHMARK,
36655
38045
  DEFAULT_ABLATION_BOOTSTRAP_SEED,
36656
38046
  DEFAULT_ASSISTANT_RUBRIC_ID,
36657
38047
  DEFAULT_BASELINE_SCENARIOS,
@@ -36664,11 +38054,14 @@ export {
36664
38054
  LOCAL_LAB_PROVIDER_KINDS,
36665
38055
  LOCOMO_DATASET_FILENAMES,
36666
38056
  LONG_MEM_EVAL_DATASET_FILENAMES,
38057
+ LettaMemCorrectAdapter,
36667
38058
  LocalLabPreflightError,
36668
38059
  MEMORY_EVAL_DIMENSIONS,
36669
38060
  MEMORY_EVAL_PUBLIC_LINE,
36670
38061
  MIN_CALIBRATION_SOURCE_TASKS,
36671
38062
  MITIGATED_BASELINE_SCENARIOS,
38063
+ Mem0MemCorrectAdapter,
38064
+ MissingCredentialError,
36672
38065
  OTHER_NAMESPACE_MEMORIES,
36673
38066
  PROCEDURAL_REAL_SCENARIOS,
36674
38067
  PROCEDURAL_REAL_SCENARIOS_SMOKE,
@@ -36677,7 +38070,9 @@ export {
36677
38070
  SCHEMA_TIER_FIXTURE,
36678
38071
  SCHEMA_TIER_SMOKE_FIXTURE,
36679
38072
  SEALED_PROMPT_REGISTRY,
38073
+ SINGLE_FLAG_ABLATION_MATRIX,
36680
38074
  SYNTHETIC_MEMORIES,
38075
+ ZepMemCorrectAdapter,
36681
38076
  addContaminationEntry,
36682
38077
  aggregateTaskScores,
36683
38078
  answerBenchmarkQuestion,
@@ -36722,6 +38117,7 @@ export {
36722
38117
  createAmaBenchDiagnosticAdapter,
36723
38118
  createAnthropicProvider,
36724
38119
  createCanaryAdapter,
38120
+ createClaudeCliProvider,
36725
38121
  createCodexCliProvider,
36726
38122
  createSeededRng3 as createCodingGraphSeededRng,
36727
38123
  createDeterministicSpotCheckLogger,
@@ -36762,6 +38158,7 @@ export {
36762
38158
  formatMissingDatasetError,
36763
38159
  generateReport,
36764
38160
  generateSyntheticRepo,
38161
+ getAblationCell,
36765
38162
  getBenchmark,
36766
38163
  getBenchmarkLowerIsBetter,
36767
38164
  getMemoryEvalDimension,