@remnic/bench 9.69.36 → 9.69.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1539 -352
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5236,10 +5236,10 @@ ${coreRecall.trim()}`;
|
|
|
5236
5236
|
const engine = getEngine();
|
|
5237
5237
|
const abortController = new AbortController();
|
|
5238
5238
|
let timer;
|
|
5239
|
-
const timeout = new Promise((_,
|
|
5239
|
+
const timeout = new Promise((_, reject2) => {
|
|
5240
5240
|
timer = setTimeout(() => {
|
|
5241
5241
|
abortController.abort();
|
|
5242
|
-
|
|
5242
|
+
reject2(
|
|
5243
5243
|
new Error(
|
|
5244
5244
|
`drain() timed out after ${drainTimeoutMs}ms (${describeDrainState(state.orchestrator)})`
|
|
5245
5245
|
)
|
|
@@ -5323,8 +5323,8 @@ async function withBenchPhaseAbort(promise, control, phase, options = {}) {
|
|
|
5323
5323
|
throw benchPhaseAbortError(signal, phase);
|
|
5324
5324
|
}
|
|
5325
5325
|
let abortHandler;
|
|
5326
|
-
const abortPromise = new Promise((_,
|
|
5327
|
-
abortHandler = () =>
|
|
5326
|
+
const abortPromise = new Promise((_, reject2) => {
|
|
5327
|
+
abortHandler = () => reject2(benchPhaseAbortError(signal, phase));
|
|
5328
5328
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
5329
5329
|
});
|
|
5330
5330
|
try {
|
|
@@ -6763,10 +6763,10 @@ async function raceWithSignal(promise, signal, label, onLateSuccess) {
|
|
|
6763
6763
|
if (!signal) return promise;
|
|
6764
6764
|
let aborted = signal.aborted;
|
|
6765
6765
|
let abortListener;
|
|
6766
|
-
const abortPromise = new Promise((_resolve,
|
|
6766
|
+
const abortPromise = new Promise((_resolve, reject2) => {
|
|
6767
6767
|
const rejectForAbort = () => {
|
|
6768
6768
|
aborted = true;
|
|
6769
|
-
|
|
6769
|
+
reject2(signal.reason instanceof Error ? signal.reason : new Error(`${label} aborted`));
|
|
6770
6770
|
};
|
|
6771
6771
|
if (signal.aborted) rejectForAbort();
|
|
6772
6772
|
else {
|
|
@@ -7178,12 +7178,12 @@ async function withTimeout(label, timeoutMs, fn, onTimeout) {
|
|
|
7178
7178
|
const controller = new AbortController();
|
|
7179
7179
|
let timeoutError;
|
|
7180
7180
|
const task = Promise.resolve().then(() => fn(controller.signal));
|
|
7181
|
-
const timeout = new Promise((_,
|
|
7181
|
+
const timeout = new Promise((_, reject2) => {
|
|
7182
7182
|
timer = setTimeout(() => {
|
|
7183
7183
|
timeoutError = new Error(
|
|
7184
7184
|
`benchmark phase timed out after ${timeoutMs}ms: ${label}`
|
|
7185
7185
|
);
|
|
7186
|
-
|
|
7186
|
+
reject2(timeoutError);
|
|
7187
7187
|
controller.abort(timeoutError);
|
|
7188
7188
|
if (onTimeout) {
|
|
7189
7189
|
void Promise.resolve(onTimeout(label)).catch(() => {
|
|
@@ -8289,14 +8289,14 @@ async function sleepBeforeClaudeCliRetry(options) {
|
|
|
8289
8289
|
if (signal.aborted) {
|
|
8290
8290
|
throw claudeCliAbortError(signal);
|
|
8291
8291
|
}
|
|
8292
|
-
await new Promise((resolve,
|
|
8292
|
+
await new Promise((resolve, reject2) => {
|
|
8293
8293
|
const cleanup = () => {
|
|
8294
8294
|
signal.removeEventListener("abort", onAbort);
|
|
8295
8295
|
};
|
|
8296
8296
|
const onAbort = () => {
|
|
8297
8297
|
clearTimeout(timeout);
|
|
8298
8298
|
cleanup();
|
|
8299
|
-
|
|
8299
|
+
reject2(claudeCliAbortError(signal));
|
|
8300
8300
|
};
|
|
8301
8301
|
const timeout = setTimeout(() => {
|
|
8302
8302
|
cleanup();
|
|
@@ -8319,7 +8319,7 @@ function summarizeProcessOutput(stderr, stdout) {
|
|
|
8319
8319
|
return summary.length > 0 ? summary.slice(-1e3) : "no process output";
|
|
8320
8320
|
}
|
|
8321
8321
|
function runClaudeVersionCommand(executable, env) {
|
|
8322
|
-
return new Promise((resolve,
|
|
8322
|
+
return new Promise((resolve, reject2) => {
|
|
8323
8323
|
const child = spawn(executable, ["--version"], {
|
|
8324
8324
|
env,
|
|
8325
8325
|
stdio: ["ignore", "ignore", "pipe"],
|
|
@@ -8357,7 +8357,7 @@ function runClaudeVersionCommand(executable, env) {
|
|
|
8357
8357
|
if (killTimeout) {
|
|
8358
8358
|
clearTimeout(killTimeout);
|
|
8359
8359
|
}
|
|
8360
|
-
|
|
8360
|
+
reject2(error);
|
|
8361
8361
|
});
|
|
8362
8362
|
child.on("close", (status) => {
|
|
8363
8363
|
clearTimeout(timeout);
|
|
@@ -8373,7 +8373,7 @@ Claude CLI --version timed out after ${CLAUDE_CLI_VERSION_TIMEOUT_MS}ms.`) : std
|
|
|
8373
8373
|
});
|
|
8374
8374
|
}
|
|
8375
8375
|
function runClaudeCliCommand(request) {
|
|
8376
|
-
return new Promise((resolve,
|
|
8376
|
+
return new Promise((resolve, reject2) => {
|
|
8377
8377
|
if (request.signal?.aborted) {
|
|
8378
8378
|
resolve({
|
|
8379
8379
|
status: 124,
|
|
@@ -8461,7 +8461,7 @@ Claude CLI stdin error: ${error.code ?? error.message}`);
|
|
|
8461
8461
|
unregisterActiveClaudeCliChild(child.pid);
|
|
8462
8462
|
}
|
|
8463
8463
|
request.signal?.removeEventListener("abort", onAbort);
|
|
8464
|
-
|
|
8464
|
+
reject2(error);
|
|
8465
8465
|
});
|
|
8466
8466
|
child.on("close", (status, signal) => {
|
|
8467
8467
|
if (timeout) {
|
|
@@ -9205,7 +9205,7 @@ ${result.stderr}`.trim();
|
|
|
9205
9205
|
}
|
|
9206
9206
|
};
|
|
9207
9207
|
function runCodexVersionCommand(executable, env) {
|
|
9208
|
-
return new Promise((resolve,
|
|
9208
|
+
return new Promise((resolve, reject2) => {
|
|
9209
9209
|
const child = spawn2(executable, ["--version"], {
|
|
9210
9210
|
env,
|
|
9211
9211
|
stdio: ["ignore", "ignore", "pipe"],
|
|
@@ -9243,7 +9243,7 @@ function runCodexVersionCommand(executable, env) {
|
|
|
9243
9243
|
if (killTimeout) {
|
|
9244
9244
|
clearTimeout(killTimeout);
|
|
9245
9245
|
}
|
|
9246
|
-
|
|
9246
|
+
reject2(error);
|
|
9247
9247
|
});
|
|
9248
9248
|
child.on("close", (status) => {
|
|
9249
9249
|
clearTimeout(timeout);
|
|
@@ -9262,7 +9262,7 @@ Codex CLI --version timed out after ${CODEX_CLI_VERSION_TIMEOUT_MS}ms.`
|
|
|
9262
9262
|
});
|
|
9263
9263
|
}
|
|
9264
9264
|
function runCodexLoginStatusCommand(executable, env) {
|
|
9265
|
-
return new Promise((resolve,
|
|
9265
|
+
return new Promise((resolve, reject2) => {
|
|
9266
9266
|
const child = spawn2(executable, ["login", "status"], {
|
|
9267
9267
|
env,
|
|
9268
9268
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -9304,7 +9304,7 @@ Codex CLI login status timed out after ${CODEX_CLI_VERSION_TIMEOUT_MS}ms.`
|
|
|
9304
9304
|
child.on("error", (error) => {
|
|
9305
9305
|
clearTimeout(timeout);
|
|
9306
9306
|
if (killTimeout) clearTimeout(killTimeout);
|
|
9307
|
-
|
|
9307
|
+
reject2(error);
|
|
9308
9308
|
});
|
|
9309
9309
|
child.on("close", (status) => {
|
|
9310
9310
|
clearTimeout(timeout);
|
|
@@ -9492,9 +9492,9 @@ function redactCodexCliArgs(args) {
|
|
|
9492
9492
|
return redacted;
|
|
9493
9493
|
}
|
|
9494
9494
|
function runCodexCliCommand(request) {
|
|
9495
|
-
return new Promise((resolve,
|
|
9495
|
+
return new Promise((resolve, reject2) => {
|
|
9496
9496
|
if (request.signal?.aborted) {
|
|
9497
|
-
|
|
9497
|
+
reject2(codexCliPreStartAbortError(request.signal));
|
|
9498
9498
|
return;
|
|
9499
9499
|
}
|
|
9500
9500
|
const child = spawn2(request.executable, request.args, {
|
|
@@ -9578,7 +9578,7 @@ Codex CLI stdin error: ${error.code ?? error.message}`
|
|
|
9578
9578
|
unregisterActiveCodexCliChild(child.pid);
|
|
9579
9579
|
}
|
|
9580
9580
|
request.signal?.removeEventListener("abort", onAbort);
|
|
9581
|
-
|
|
9581
|
+
reject2(
|
|
9582
9582
|
child.pid ? new Error(
|
|
9583
9583
|
`Codex CLI failed after its process started: ${safeErrorMessage(error)}`,
|
|
9584
9584
|
{ cause: error }
|
|
@@ -9766,14 +9766,14 @@ async function sleepBeforeCodexCliRetry(attempt, configuredBaseBackoffMs, signal
|
|
|
9766
9766
|
if (signal.aborted) {
|
|
9767
9767
|
throw codexCliAbortError(signal);
|
|
9768
9768
|
}
|
|
9769
|
-
await new Promise((resolve,
|
|
9769
|
+
await new Promise((resolve, reject2) => {
|
|
9770
9770
|
const cleanup = () => {
|
|
9771
9771
|
signal.removeEventListener("abort", onAbort);
|
|
9772
9772
|
};
|
|
9773
9773
|
const onAbort = () => {
|
|
9774
9774
|
clearTimeout(timeout);
|
|
9775
9775
|
cleanup();
|
|
9776
|
-
|
|
9776
|
+
reject2(codexCliAbortError(signal));
|
|
9777
9777
|
};
|
|
9778
9778
|
const timeout = setTimeout(() => {
|
|
9779
9779
|
cleanup();
|
|
@@ -26103,8 +26103,1191 @@ function constantAggregate(value) {
|
|
|
26103
26103
|
};
|
|
26104
26104
|
}
|
|
26105
26105
|
|
|
26106
|
-
// src/benchmarks/remnic/
|
|
26106
|
+
// src/benchmarks/remnic/extraction-span-mode/runner.ts
|
|
26107
26107
|
import { randomUUID as randomUUID12 } from "crypto";
|
|
26108
|
+
import { evaluateSpanPhaseGate } from "@remnic/core/extraction-span-gate";
|
|
26109
|
+
import { tallySpanFallbacks } from "@remnic/core/extraction-span-fallback";
|
|
26110
|
+
|
|
26111
|
+
// src/benchmarks/remnic/extraction-span-mode/fixture.ts
|
|
26112
|
+
var SPAN_BENCH_FIXTURE = [
|
|
26113
|
+
{
|
|
26114
|
+
id: "locomo-maya",
|
|
26115
|
+
dataset: "locomo",
|
|
26116
|
+
messages: [
|
|
26117
|
+
{
|
|
26118
|
+
speaker: "Maya",
|
|
26119
|
+
text: "I moved to Seattle last spring after accepting the new senior role at the design studio, and I still miss the Chicago food scene."
|
|
26120
|
+
},
|
|
26121
|
+
{ speaker: "Assistant", text: "Seattle sounds exciting. I'll remember that move." },
|
|
26122
|
+
{
|
|
26123
|
+
speaker: "Maya",
|
|
26124
|
+
text: "My favorite tea is jasmine, especially during rainy mornings in the winter when the studio windows fog up."
|
|
26125
|
+
},
|
|
26126
|
+
{ speaker: "Assistant", text: "Jasmine tea on rainy mornings. Got it." },
|
|
26127
|
+
{
|
|
26128
|
+
speaker: "Maya",
|
|
26129
|
+
text: "Actually, call me M. Nobody at the studio uses my full name anymore, not even the founders."
|
|
26130
|
+
},
|
|
26131
|
+
{ speaker: "Assistant", text: "M it is." }
|
|
26132
|
+
],
|
|
26133
|
+
facts: [
|
|
26134
|
+
{
|
|
26135
|
+
id: "maya-move",
|
|
26136
|
+
messageIndex: 0,
|
|
26137
|
+
quote: "I moved to Seattle last spring after accepting the new senior role at the design studio",
|
|
26138
|
+
frame: "Maya's relocation",
|
|
26139
|
+
content: "Maya moved to Seattle last spring after accepting the new senior role at the design studio.",
|
|
26140
|
+
restatement: "Maya moved to Seattle last spring after taking the senior role at the design studio.",
|
|
26141
|
+
category: "fact",
|
|
26142
|
+
confidence: 0.95,
|
|
26143
|
+
tags: ["relocation", "seattle"]
|
|
26144
|
+
},
|
|
26145
|
+
{
|
|
26146
|
+
id: "maya-tea",
|
|
26147
|
+
messageIndex: 2,
|
|
26148
|
+
quote: "My favorite tea is jasmine, especially during rainy mornings in the winter",
|
|
26149
|
+
frame: "Maya's favorite tea",
|
|
26150
|
+
content: "Maya's favorite tea is jasmine, especially during rainy mornings in the winter.",
|
|
26151
|
+
restatement: "Maya's favorite tea is jasmine, particularly on rainy mornings in winter.",
|
|
26152
|
+
category: "preference",
|
|
26153
|
+
confidence: 0.93,
|
|
26154
|
+
tags: ["tea", "preference"]
|
|
26155
|
+
},
|
|
26156
|
+
{
|
|
26157
|
+
id: "maya-name",
|
|
26158
|
+
messageIndex: 4,
|
|
26159
|
+
quote: "Actually, call me M. Nobody at the studio uses my full name anymore",
|
|
26160
|
+
frame: "Maya's preferred name",
|
|
26161
|
+
content: "Maya prefers to be called M; nobody at the studio uses her full name anymore.",
|
|
26162
|
+
restatement: "Maya goes by M; no one at the studio uses her full name anymore.",
|
|
26163
|
+
category: "preference",
|
|
26164
|
+
confidence: 0.9,
|
|
26165
|
+
tags: ["name", "preference"]
|
|
26166
|
+
}
|
|
26167
|
+
]
|
|
26168
|
+
},
|
|
26169
|
+
{
|
|
26170
|
+
id: "locomo-hackathon",
|
|
26171
|
+
dataset: "locomo",
|
|
26172
|
+
messages: [
|
|
26173
|
+
{
|
|
26174
|
+
speaker: "Alice",
|
|
26175
|
+
text: "We finalized the venue for the hackathon: the riverside loft on June 14th, which books half-day cleaning slots before opening."
|
|
26176
|
+
},
|
|
26177
|
+
{ speaker: "Ben", text: "Nice. I can mentor the beginners' track that weekend if the schedule holds." },
|
|
26178
|
+
{ speaker: "Alice", text: "Perfect. Registration caps at 80 people this year because of the loft occupancy permit." },
|
|
26179
|
+
{ speaker: "Ben", text: "I'll bring the extra power strips from the office." },
|
|
26180
|
+
{
|
|
26181
|
+
speaker: "Alice",
|
|
26182
|
+
text: "Also, the after-party is vegetarian only \u2014 the caterer confirmed today that every dish is plant-based."
|
|
26183
|
+
}
|
|
26184
|
+
],
|
|
26185
|
+
facts: [
|
|
26186
|
+
{
|
|
26187
|
+
id: "venue",
|
|
26188
|
+
messageIndex: 0,
|
|
26189
|
+
quote: "We finalized the venue for the hackathon: the riverside loft on June 14th",
|
|
26190
|
+
frame: "Hackathon venue",
|
|
26191
|
+
content: "The hackathon venue is the riverside loft on June 14th.",
|
|
26192
|
+
restatement: "The hackathon venue is the riverside loft, June 14th.",
|
|
26193
|
+
category: "decision",
|
|
26194
|
+
confidence: 0.96,
|
|
26195
|
+
tags: ["hackathon", "venue"]
|
|
26196
|
+
},
|
|
26197
|
+
{
|
|
26198
|
+
id: "mentor",
|
|
26199
|
+
messageIndex: 1,
|
|
26200
|
+
quote: "I can mentor the beginners' track that weekend if the schedule holds",
|
|
26201
|
+
frame: "Ben's hackathon role",
|
|
26202
|
+
content: "Ben can mentor the beginners' track that weekend if the schedule holds.",
|
|
26203
|
+
restatement: "Ben can mentor the beginners' track that weekend if scheduling works.",
|
|
26204
|
+
category: "commitment",
|
|
26205
|
+
confidence: 0.92,
|
|
26206
|
+
tags: ["hackathon", "mentor"]
|
|
26207
|
+
},
|
|
26208
|
+
{
|
|
26209
|
+
id: "capacity",
|
|
26210
|
+
messageIndex: 2,
|
|
26211
|
+
quote: "Registration caps at 80 people this year because of the loft occupancy permit",
|
|
26212
|
+
frame: "Hackathon registration cap",
|
|
26213
|
+
content: "Hackathon registration caps at 80 people this year because of the loft occupancy permit.",
|
|
26214
|
+
restatement: "Registration caps at 80 people this year due to the loft occupancy permit.",
|
|
26215
|
+
category: "fact",
|
|
26216
|
+
confidence: 0.94,
|
|
26217
|
+
tags: ["hackathon", "capacity"]
|
|
26218
|
+
},
|
|
26219
|
+
{
|
|
26220
|
+
id: "catering",
|
|
26221
|
+
messageIndex: 4,
|
|
26222
|
+
quote: "the after-party is vegetarian only \u2014 the caterer confirmed today",
|
|
26223
|
+
frame: "After-party catering",
|
|
26224
|
+
content: "The hackathon after-party is vegetarian only \u2014 the caterer confirmed today.",
|
|
26225
|
+
restatement: "The after-party is vegetarian only \u2014 the caterer confirmed it today.",
|
|
26226
|
+
category: "decision",
|
|
26227
|
+
confidence: 0.91,
|
|
26228
|
+
tags: ["hackathon", "catering"]
|
|
26229
|
+
}
|
|
26230
|
+
]
|
|
26231
|
+
},
|
|
26232
|
+
{
|
|
26233
|
+
id: "locomo-clara",
|
|
26234
|
+
dataset: "locomo",
|
|
26235
|
+
messages: [
|
|
26236
|
+
{
|
|
26237
|
+
speaker: "Clara",
|
|
26238
|
+
text: "My thesis defense is scheduled for March 3rd, not March 13th \u2014 they moved it up a week at my request."
|
|
26239
|
+
},
|
|
26240
|
+
{ speaker: "Assistant", text: "Noted: March 3rd." },
|
|
26241
|
+
{
|
|
26242
|
+
speaker: "Clara",
|
|
26243
|
+
text: "I stopped drinking coffee; my doctor suggested cutting caffeine entirely to help with the afternoon crashes."
|
|
26244
|
+
},
|
|
26245
|
+
{ speaker: "Assistant", text: "Cutting caffeine entirely \u2014 understood." },
|
|
26246
|
+
{
|
|
26247
|
+
speaker: "Clara",
|
|
26248
|
+
text: "I've been learning Japanese for two years now, mostly with podcasts during the morning commute."
|
|
26249
|
+
},
|
|
26250
|
+
{ speaker: "Assistant", text: "Two years of Japanese via podcasts. Impressive." }
|
|
26251
|
+
],
|
|
26252
|
+
facts: [
|
|
26253
|
+
{
|
|
26254
|
+
id: "defense-date",
|
|
26255
|
+
messageIndex: 0,
|
|
26256
|
+
quote: "My thesis defense is scheduled for March 3rd, not March 13th",
|
|
26257
|
+
frame: "Clara's thesis defense date",
|
|
26258
|
+
content: "Clara's thesis defense is scheduled for March 3rd, not March 13th.",
|
|
26259
|
+
restatement: "Clara's thesis defense is scheduled for March 3, not March 13.",
|
|
26260
|
+
category: "correction",
|
|
26261
|
+
confidence: 0.97,
|
|
26262
|
+
tags: ["thesis", "schedule"]
|
|
26263
|
+
},
|
|
26264
|
+
{
|
|
26265
|
+
id: "caffeine",
|
|
26266
|
+
messageIndex: 2,
|
|
26267
|
+
quote: "I stopped drinking coffee; my doctor suggested cutting caffeine entirely",
|
|
26268
|
+
frame: "Clara's caffeine cut",
|
|
26269
|
+
content: "Clara stopped drinking coffee; her doctor suggested cutting caffeine entirely.",
|
|
26270
|
+
restatement: "Clara stopped drinking coffee; her doctor advised cutting caffeine entirely.",
|
|
26271
|
+
category: "fact",
|
|
26272
|
+
confidence: 0.93,
|
|
26273
|
+
tags: ["health", "caffeine"]
|
|
26274
|
+
},
|
|
26275
|
+
{
|
|
26276
|
+
id: "japanese",
|
|
26277
|
+
messageIndex: 4,
|
|
26278
|
+
quote: "I've been learning Japanese for two years now, mostly with podcasts",
|
|
26279
|
+
frame: "Clara's Japanese study",
|
|
26280
|
+
content: "Clara has been learning Japanese for two years now, mostly with podcasts.",
|
|
26281
|
+
restatement: "Clara has been learning Japanese for two years, mostly via podcasts.",
|
|
26282
|
+
category: "skill",
|
|
26283
|
+
confidence: 0.9,
|
|
26284
|
+
tags: ["japanese", "learning"]
|
|
26285
|
+
}
|
|
26286
|
+
]
|
|
26287
|
+
},
|
|
26288
|
+
{
|
|
26289
|
+
id: "lme-notebook",
|
|
26290
|
+
dataset: "longmemeval",
|
|
26291
|
+
messages: [
|
|
26292
|
+
{
|
|
26293
|
+
speaker: "User",
|
|
26294
|
+
text: "I switched my main notebook to a Framework 13 last October after the old one died mid-flight."
|
|
26295
|
+
},
|
|
26296
|
+
{ speaker: "Assistant", text: "Framework 13 it is." },
|
|
26297
|
+
{
|
|
26298
|
+
speaker: "User",
|
|
26299
|
+
text: "I keep all my research notes in Zettelkasten folders now, ever since the markdown sprawl got unusable."
|
|
26300
|
+
},
|
|
26301
|
+
{ speaker: "Assistant", text: "Zettelkasten folders, got it." },
|
|
26302
|
+
{
|
|
26303
|
+
speaker: "User",
|
|
26304
|
+
text: "My partner Priya started a part-time ceramics course in January at the community studio near the library."
|
|
26305
|
+
},
|
|
26306
|
+
{ speaker: "Assistant", text: "Priya's ceramics course \u2014 noted." },
|
|
26307
|
+
{ speaker: "User", text: "I try to read 20 pages a day before bed, and it mostly sticks." },
|
|
26308
|
+
{ speaker: "Assistant", text: "20 pages before bed, noted." }
|
|
26309
|
+
],
|
|
26310
|
+
facts: [
|
|
26311
|
+
{
|
|
26312
|
+
id: "notebook",
|
|
26313
|
+
messageIndex: 0,
|
|
26314
|
+
quote: "I switched my main notebook to a Framework 13 last October",
|
|
26315
|
+
frame: "User's main notebook",
|
|
26316
|
+
content: "The user switched their main notebook to a Framework 13 last October.",
|
|
26317
|
+
restatement: "The user moved their main notebook to a Framework 13 last October.",
|
|
26318
|
+
category: "fact",
|
|
26319
|
+
confidence: 0.95,
|
|
26320
|
+
tags: ["hardware", "notebook"]
|
|
26321
|
+
},
|
|
26322
|
+
{
|
|
26323
|
+
id: "notes-system",
|
|
26324
|
+
messageIndex: 2,
|
|
26325
|
+
quote: "I keep all my research notes in Zettelkasten folders now",
|
|
26326
|
+
frame: "User's research notes",
|
|
26327
|
+
content: "The user keeps all research notes in Zettelkasten folders now.",
|
|
26328
|
+
restatement: "The user keeps research notes in Zettelkasten folders these days.",
|
|
26329
|
+
category: "fact",
|
|
26330
|
+
confidence: 0.92,
|
|
26331
|
+
tags: ["notes", "workflow"]
|
|
26332
|
+
},
|
|
26333
|
+
{
|
|
26334
|
+
id: "priya-ceramics",
|
|
26335
|
+
messageIndex: 4,
|
|
26336
|
+
quote: "My partner Priya started a part-time ceramics course in January",
|
|
26337
|
+
frame: "Priya's ceramics course",
|
|
26338
|
+
content: "The user's partner Priya started a part-time ceramics course in January.",
|
|
26339
|
+
restatement: "The user's partner Priya began a part-time ceramics course in January.",
|
|
26340
|
+
category: "relationship",
|
|
26341
|
+
confidence: 0.91,
|
|
26342
|
+
tags: ["priya", "ceramics"]
|
|
26343
|
+
},
|
|
26344
|
+
{
|
|
26345
|
+
id: "reading-habit",
|
|
26346
|
+
messageIndex: 6,
|
|
26347
|
+
quote: "I try to read 20 pages a day before bed",
|
|
26348
|
+
frame: "User's reading habit",
|
|
26349
|
+
content: "The user tries to read 20 pages a day before bed.",
|
|
26350
|
+
restatement: "The user aims to read 20 pages a day before bed.",
|
|
26351
|
+
category: "preference",
|
|
26352
|
+
confidence: 0.9,
|
|
26353
|
+
tags: ["reading", "habit"]
|
|
26354
|
+
}
|
|
26355
|
+
]
|
|
26356
|
+
},
|
|
26357
|
+
{
|
|
26358
|
+
id: "lme-fitness",
|
|
26359
|
+
dataset: "longmemeval",
|
|
26360
|
+
messages: [
|
|
26361
|
+
{
|
|
26362
|
+
speaker: "User",
|
|
26363
|
+
text: "I canceled my gym membership; I train at home with kettlebells now, three sessions a week."
|
|
26364
|
+
},
|
|
26365
|
+
{ speaker: "Assistant", text: "Home kettlebell training, noted." },
|
|
26366
|
+
{
|
|
26367
|
+
speaker: "User",
|
|
26368
|
+
text: "For invoices I use FreeInvoice, the open-source one, since 2023 when the freelance work picked up."
|
|
26369
|
+
},
|
|
26370
|
+
{ speaker: "Assistant", text: "FreeInvoice since 2023, got it." },
|
|
26371
|
+
{
|
|
26372
|
+
speaker: "User",
|
|
26373
|
+
text: "I'm allergic to walnuts but pecans are fine, so I check bakery labels every time."
|
|
26374
|
+
},
|
|
26375
|
+
{ speaker: "Assistant", text: "Walnut allergy noted; pecans fine." }
|
|
26376
|
+
],
|
|
26377
|
+
facts: [
|
|
26378
|
+
{
|
|
26379
|
+
id: "training",
|
|
26380
|
+
messageIndex: 0,
|
|
26381
|
+
quote: "I canceled my gym membership; I train at home with kettlebells now",
|
|
26382
|
+
frame: "User's home training",
|
|
26383
|
+
content: "The user canceled their gym membership and trains at home with kettlebells now.",
|
|
26384
|
+
restatement: "The user canceled the gym membership and now trains at home with kettlebells.",
|
|
26385
|
+
category: "fact",
|
|
26386
|
+
confidence: 0.94,
|
|
26387
|
+
tags: ["fitness", "kettlebells"]
|
|
26388
|
+
},
|
|
26389
|
+
{
|
|
26390
|
+
id: "invoicing",
|
|
26391
|
+
messageIndex: 2,
|
|
26392
|
+
quote: "For invoices I use FreeInvoice, the open-source one, since 2023",
|
|
26393
|
+
frame: "User's invoicing tool",
|
|
26394
|
+
content: "For invoices the user uses FreeInvoice, the open-source one, since 2023.",
|
|
26395
|
+
restatement: "The user uses open-source FreeInvoice for invoices since 2023.",
|
|
26396
|
+
category: "fact",
|
|
26397
|
+
confidence: 0.93,
|
|
26398
|
+
tags: ["invoicing", "tools"]
|
|
26399
|
+
},
|
|
26400
|
+
{
|
|
26401
|
+
id: "walnuts",
|
|
26402
|
+
messageIndex: 4,
|
|
26403
|
+
quote: "I'm allergic to walnuts but pecans are fine",
|
|
26404
|
+
frame: "User's nut allergy",
|
|
26405
|
+
content: "The user is allergic to walnuts but pecans are fine.",
|
|
26406
|
+
restatement: "The user is allergic to walnuts; pecans are fine.",
|
|
26407
|
+
category: "fact",
|
|
26408
|
+
confidence: 0.96,
|
|
26409
|
+
tags: ["allergy", "walnuts"]
|
|
26410
|
+
}
|
|
26411
|
+
]
|
|
26412
|
+
},
|
|
26413
|
+
{
|
|
26414
|
+
id: "lme-routines",
|
|
26415
|
+
dataset: "longmemeval",
|
|
26416
|
+
messages: [
|
|
26417
|
+
{
|
|
26418
|
+
speaker: "User",
|
|
26419
|
+
text: "My dentist appointments are always the first Tuesday of the month, right when the office opens."
|
|
26420
|
+
},
|
|
26421
|
+
{ speaker: "Assistant", text: "First Tuesday dental visits, noted." },
|
|
26422
|
+
{
|
|
26423
|
+
speaker: "User",
|
|
26424
|
+
text: "I park in the Blue Garage on level 3 when visiting the office, since street parking vanished."
|
|
26425
|
+
},
|
|
26426
|
+
{ speaker: "Assistant", text: "Blue Garage level 3, got it." },
|
|
26427
|
+
{
|
|
26428
|
+
speaker: "User",
|
|
26429
|
+
text: "I compile my Rust projects with nightly, never stable, because of the async trait syntax."
|
|
26430
|
+
},
|
|
26431
|
+
{ speaker: "Assistant", text: "Rust nightly toolchain, noted." }
|
|
26432
|
+
],
|
|
26433
|
+
facts: [
|
|
26434
|
+
{
|
|
26435
|
+
id: "dentist",
|
|
26436
|
+
messageIndex: 0,
|
|
26437
|
+
quote: "My dentist appointments are always the first Tuesday of the month",
|
|
26438
|
+
frame: "User's dentist schedule",
|
|
26439
|
+
content: "The user's dentist appointments are always the first Tuesday of the month.",
|
|
26440
|
+
restatement: "The user's dentist appointments are always on the first Tuesday of the month.",
|
|
26441
|
+
category: "fact",
|
|
26442
|
+
confidence: 0.92,
|
|
26443
|
+
tags: ["dentist", "schedule"]
|
|
26444
|
+
},
|
|
26445
|
+
{
|
|
26446
|
+
id: "parking",
|
|
26447
|
+
messageIndex: 2,
|
|
26448
|
+
quote: "I park in the Blue Garage on level 3 when visiting the office",
|
|
26449
|
+
frame: "User's office parking",
|
|
26450
|
+
content: "The user parks in the Blue Garage on level 3 when visiting the office.",
|
|
26451
|
+
restatement: "The user parks at the Blue Garage, level 3, for office visits.",
|
|
26452
|
+
category: "fact",
|
|
26453
|
+
confidence: 0.93,
|
|
26454
|
+
tags: ["office", "parking"]
|
|
26455
|
+
},
|
|
26456
|
+
{
|
|
26457
|
+
id: "rust-toolchain",
|
|
26458
|
+
messageIndex: 4,
|
|
26459
|
+
quote: "I compile my Rust projects with nightly, never stable",
|
|
26460
|
+
frame: "User's Rust toolchain",
|
|
26461
|
+
content: "The user compiles Rust projects with nightly, never stable.",
|
|
26462
|
+
restatement: "The user builds Rust projects with nightly, never stable.",
|
|
26463
|
+
category: "preference",
|
|
26464
|
+
confidence: 0.95,
|
|
26465
|
+
tags: ["rust", "toolchain"]
|
|
26466
|
+
}
|
|
26467
|
+
]
|
|
26468
|
+
},
|
|
26469
|
+
{
|
|
26470
|
+
id: "locomo-diego",
|
|
26471
|
+
dataset: "locomo",
|
|
26472
|
+
messages: [
|
|
26473
|
+
{
|
|
26474
|
+
speaker: "Diego",
|
|
26475
|
+
text: "I finally finished restoring the grandfather clock my uncle left me; it took nine weekends of shop time."
|
|
26476
|
+
},
|
|
26477
|
+
{ speaker: "Assistant", text: "Nine weekends \u2014 that's dedication." },
|
|
26478
|
+
{
|
|
26479
|
+
speaker: "Diego",
|
|
26480
|
+
text: "My daughter Rosa starts at the maritime academy in September, and she is nervous about the swim test."
|
|
26481
|
+
},
|
|
26482
|
+
{ speaker: "Assistant", text: "Maritime academy in September, noted." },
|
|
26483
|
+
{
|
|
26484
|
+
speaker: "Diego",
|
|
26485
|
+
text: "I stopped coaching the youth league once the schedule collided with my night classes, but I still referee on Sundays."
|
|
26486
|
+
},
|
|
26487
|
+
{ speaker: "Assistant", text: "Refereeing on Sundays only now." }
|
|
26488
|
+
],
|
|
26489
|
+
facts: [
|
|
26490
|
+
{
|
|
26491
|
+
id: "clock-restoration",
|
|
26492
|
+
messageIndex: 0,
|
|
26493
|
+
quote: "I finally finished restoring the grandfather clock my uncle left me",
|
|
26494
|
+
frame: "Diego's clock restoration",
|
|
26495
|
+
content: "Diego finished restoring the grandfather clock his uncle left him.",
|
|
26496
|
+
restatement: "Diego finished restoring the grandfather clock he inherited from his uncle.",
|
|
26497
|
+
category: "moment",
|
|
26498
|
+
confidence: 0.9,
|
|
26499
|
+
tags: ["clock", "restoration"]
|
|
26500
|
+
},
|
|
26501
|
+
{
|
|
26502
|
+
id: "rosa-academy",
|
|
26503
|
+
messageIndex: 2,
|
|
26504
|
+
quote: "My daughter Rosa starts at the maritime academy in September",
|
|
26505
|
+
frame: "Rosa's maritime academy start",
|
|
26506
|
+
content: "Diego's daughter Rosa starts at the maritime academy in September.",
|
|
26507
|
+
restatement: "Diego's daughter Rosa begins at the maritime academy in September.",
|
|
26508
|
+
category: "commitment",
|
|
26509
|
+
confidence: 0.93,
|
|
26510
|
+
tags: ["rosa", "academy"]
|
|
26511
|
+
},
|
|
26512
|
+
{
|
|
26513
|
+
id: "refereeing",
|
|
26514
|
+
messageIndex: 4,
|
|
26515
|
+
quote: "but I still referee on Sundays",
|
|
26516
|
+
frame: "Diego's Sunday refereeing",
|
|
26517
|
+
content: "Diego still referees on Sundays after stopping youth-league coaching.",
|
|
26518
|
+
restatement: "Diego still referees Sundays after quitting youth-league coaching.",
|
|
26519
|
+
category: "commitment",
|
|
26520
|
+
confidence: 0.88,
|
|
26521
|
+
tags: ["referee", "schedule"]
|
|
26522
|
+
}
|
|
26523
|
+
]
|
|
26524
|
+
},
|
|
26525
|
+
{
|
|
26526
|
+
id: "locomo-band",
|
|
26527
|
+
dataset: "locomo",
|
|
26528
|
+
messages: [
|
|
26529
|
+
{
|
|
26530
|
+
speaker: "Nadia",
|
|
26531
|
+
text: "Our band's debut EP drops on Bandcamp the first Friday of October, all five tracks self-recorded."
|
|
26532
|
+
},
|
|
26533
|
+
{ speaker: "Assistant", text: "First Friday of October \u2014 congrats." },
|
|
26534
|
+
{
|
|
26535
|
+
speaker: "Nadia",
|
|
26536
|
+
text: "I switched from bass to synths last winter because the setlist changed direction."
|
|
26537
|
+
},
|
|
26538
|
+
{ speaker: "Assistant", text: "Synths instead of bass, noted." },
|
|
26539
|
+
{
|
|
26540
|
+
speaker: "Nadia",
|
|
26541
|
+
text: "Rehearsals moved to the storage-unit space on Ferry Street since the old room doubled its rent."
|
|
26542
|
+
},
|
|
26543
|
+
{ speaker: "Assistant", text: "Ferry Street rehearsal space, got it." }
|
|
26544
|
+
],
|
|
26545
|
+
facts: [
|
|
26546
|
+
{
|
|
26547
|
+
id: "ep-release",
|
|
26548
|
+
messageIndex: 0,
|
|
26549
|
+
quote: "Our band's debut EP drops on Bandcamp the first Friday of October",
|
|
26550
|
+
frame: "Band's EP release",
|
|
26551
|
+
content: "Nadia's band's debut EP drops on Bandcamp the first Friday of October.",
|
|
26552
|
+
restatement: "Nadia's band releases its debut EP on Bandcamp the first Friday of October.",
|
|
26553
|
+
category: "commitment",
|
|
26554
|
+
confidence: 0.94,
|
|
26555
|
+
tags: ["band", "release"]
|
|
26556
|
+
},
|
|
26557
|
+
{
|
|
26558
|
+
id: "instrument-switch",
|
|
26559
|
+
messageIndex: 2,
|
|
26560
|
+
quote: "I switched from bass to synths last winter",
|
|
26561
|
+
frame: "Nadia's instrument switch",
|
|
26562
|
+
content: "Nadia switched from bass to synths last winter.",
|
|
26563
|
+
restatement: "Nadia moved from bass to synths last winter.",
|
|
26564
|
+
category: "fact",
|
|
26565
|
+
confidence: 0.91,
|
|
26566
|
+
tags: ["band", "instrument"]
|
|
26567
|
+
},
|
|
26568
|
+
{
|
|
26569
|
+
id: "rehearsal-space",
|
|
26570
|
+
messageIndex: 4,
|
|
26571
|
+
quote: "Rehearsals moved to the storage-unit space on Ferry Street",
|
|
26572
|
+
frame: "Band's rehearsal space",
|
|
26573
|
+
content: "Band rehearsals moved to the storage-unit space on Ferry Street.",
|
|
26574
|
+
restatement: "The band now rehearses in the Ferry Street storage-unit space.",
|
|
26575
|
+
category: "fact",
|
|
26576
|
+
confidence: 0.9,
|
|
26577
|
+
tags: ["band", "rehearsal"]
|
|
26578
|
+
}
|
|
26579
|
+
]
|
|
26580
|
+
},
|
|
26581
|
+
{
|
|
26582
|
+
id: "locomo-garden",
|
|
26583
|
+
dataset: "locomo",
|
|
26584
|
+
messages: [
|
|
26585
|
+
{
|
|
26586
|
+
speaker: "Tom",
|
|
26587
|
+
text: "The community garden allotted me plot 12, the shady corner near the compost bins."
|
|
26588
|
+
},
|
|
26589
|
+
{ speaker: "Assistant", text: "Plot 12, the shady corner." },
|
|
26590
|
+
{
|
|
26591
|
+
speaker: "Tom",
|
|
26592
|
+
text: "I'm growing mostly leafy greens this season because the tomatoes failed in the shade two years running."
|
|
26593
|
+
},
|
|
26594
|
+
{ speaker: "Assistant", text: "Leafy greens it is." },
|
|
26595
|
+
{
|
|
26596
|
+
speaker: "Tom",
|
|
26597
|
+
text: "My knee surgery is rescheduled to the 21st, so I'll miss the spring workday for the first time."
|
|
26598
|
+
},
|
|
26599
|
+
{ speaker: "Assistant", text: "Surgery on the 21st \u2014 noted." }
|
|
26600
|
+
],
|
|
26601
|
+
facts: [
|
|
26602
|
+
{
|
|
26603
|
+
id: "garden-plot",
|
|
26604
|
+
messageIndex: 0,
|
|
26605
|
+
quote: "The community garden allotted me plot 12, the shady corner near the compost bins",
|
|
26606
|
+
frame: "Tom's garden plot",
|
|
26607
|
+
content: "The community garden allotted Tom plot 12, the shady corner near the compost bins.",
|
|
26608
|
+
restatement: "Tom's community garden plot is 12, the shady corner by the compost bins.",
|
|
26609
|
+
category: "fact",
|
|
26610
|
+
confidence: 0.92,
|
|
26611
|
+
tags: ["garden", "plot"]
|
|
26612
|
+
},
|
|
26613
|
+
{
|
|
26614
|
+
id: "leafy-greens",
|
|
26615
|
+
messageIndex: 2,
|
|
26616
|
+
quote: "I'm growing mostly leafy greens this season",
|
|
26617
|
+
frame: "Tom's seasonal crops",
|
|
26618
|
+
content: "Tom is growing mostly leafy greens this season.",
|
|
26619
|
+
restatement: "Tom is growing mostly leafy greens this year.",
|
|
26620
|
+
category: "fact",
|
|
26621
|
+
confidence: 0.9,
|
|
26622
|
+
tags: ["garden", "crops"]
|
|
26623
|
+
},
|
|
26624
|
+
{
|
|
26625
|
+
id: "knee-surgery",
|
|
26626
|
+
messageIndex: 4,
|
|
26627
|
+
quote: "My knee surgery is rescheduled to the 21st",
|
|
26628
|
+
frame: "Tom's knee surgery date",
|
|
26629
|
+
content: "Tom's knee surgery is rescheduled to the 21st.",
|
|
26630
|
+
restatement: "Tom's knee surgery got moved to the 21st.",
|
|
26631
|
+
category: "correction",
|
|
26632
|
+
confidence: 0.95,
|
|
26633
|
+
tags: ["health", "surgery"]
|
|
26634
|
+
}
|
|
26635
|
+
]
|
|
26636
|
+
},
|
|
26637
|
+
{
|
|
26638
|
+
id: "lme-tools",
|
|
26639
|
+
dataset: "longmemeval",
|
|
26640
|
+
messages: [
|
|
26641
|
+
{
|
|
26642
|
+
speaker: "User",
|
|
26643
|
+
text: "I migrated all my passwords to a local vault last month after the breach news, and I rotate the master key quarterly."
|
|
26644
|
+
},
|
|
26645
|
+
{ speaker: "Assistant", text: "Local vault with quarterly rotation, noted." },
|
|
26646
|
+
{
|
|
26647
|
+
speaker: "User",
|
|
26648
|
+
text: "I write my standup notes in the shared team doc before 9:30, never in the DM thread."
|
|
26649
|
+
},
|
|
26650
|
+
{ speaker: "Assistant", text: "Standup notes in the shared doc before 9:30." },
|
|
26651
|
+
{
|
|
26652
|
+
speaker: "User",
|
|
26653
|
+
text: "My brother Marco covers my dog-sitting every other Thursday when the late deploy window lands."
|
|
26654
|
+
},
|
|
26655
|
+
{ speaker: "Assistant", text: "Marco's Thursday dog-sitting, noted." }
|
|
26656
|
+
],
|
|
26657
|
+
facts: [
|
|
26658
|
+
{
|
|
26659
|
+
id: "password-vault",
|
|
26660
|
+
messageIndex: 0,
|
|
26661
|
+
quote: "I migrated all my passwords to a local vault last month",
|
|
26662
|
+
frame: "User's password vault",
|
|
26663
|
+
content: "The user migrated all their passwords to a local vault last month.",
|
|
26664
|
+
restatement: "The user moved all passwords into a local vault last month.",
|
|
26665
|
+
category: "fact",
|
|
26666
|
+
confidence: 0.94,
|
|
26667
|
+
tags: ["security", "vault"]
|
|
26668
|
+
},
|
|
26669
|
+
{
|
|
26670
|
+
id: "standup-notes",
|
|
26671
|
+
messageIndex: 2,
|
|
26672
|
+
quote: "I write my standup notes in the shared team doc before 9:30",
|
|
26673
|
+
frame: "User's standup note habit",
|
|
26674
|
+
content: "The user writes standup notes in the shared team doc before 9:30.",
|
|
26675
|
+
restatement: "The user posts standup notes in the shared team doc before 9:30.",
|
|
26676
|
+
category: "preference",
|
|
26677
|
+
confidence: 0.91,
|
|
26678
|
+
tags: ["standup", "workflow"]
|
|
26679
|
+
},
|
|
26680
|
+
{
|
|
26681
|
+
id: "marco-dogsitting",
|
|
26682
|
+
messageIndex: 4,
|
|
26683
|
+
quote: "My brother Marco covers my dog-sitting every other Thursday",
|
|
26684
|
+
frame: "Marco's dog-sitting schedule",
|
|
26685
|
+
content: "The user's brother Marco covers dog-sitting every other Thursday.",
|
|
26686
|
+
restatement: "The user's brother Marco handles dog-sitting every other Thursday.",
|
|
26687
|
+
category: "relationship",
|
|
26688
|
+
confidence: 0.9,
|
|
26689
|
+
tags: ["marco", "dog-sitting"]
|
|
26690
|
+
}
|
|
26691
|
+
]
|
|
26692
|
+
},
|
|
26693
|
+
{
|
|
26694
|
+
id: "lme-study",
|
|
26695
|
+
dataset: "longmemeval",
|
|
26696
|
+
messages: [
|
|
26697
|
+
{
|
|
26698
|
+
speaker: "User",
|
|
26699
|
+
text: "I passed the licensing exam on the second attempt last March, eight points above the cutoff."
|
|
26700
|
+
},
|
|
26701
|
+
{ speaker: "Assistant", text: "Passed on the second attempt \u2014 congrats." },
|
|
26702
|
+
{
|
|
26703
|
+
speaker: "User",
|
|
26704
|
+
text: "I study best at the kitchen counter with noise-canceling headphones, never at the desk."
|
|
26705
|
+
},
|
|
26706
|
+
{ speaker: "Assistant", text: "Kitchen counter with headphones, noted." },
|
|
26707
|
+
{
|
|
26708
|
+
speaker: "User",
|
|
26709
|
+
text: "My tutor Lena charges a sliding scale, and she waived the fee during my exam retake month."
|
|
26710
|
+
},
|
|
26711
|
+
{ speaker: "Assistant", text: "Lena's sliding-scale tutoring, noted." }
|
|
26712
|
+
],
|
|
26713
|
+
facts: [
|
|
26714
|
+
{
|
|
26715
|
+
id: "licensing-exam",
|
|
26716
|
+
messageIndex: 0,
|
|
26717
|
+
quote: "I passed the licensing exam on the second attempt last March",
|
|
26718
|
+
frame: "User's licensing exam result",
|
|
26719
|
+
content: "The user passed the licensing exam on the second attempt last March.",
|
|
26720
|
+
restatement: "The user passed the licensing exam on the second try last March.",
|
|
26721
|
+
category: "moment",
|
|
26722
|
+
confidence: 0.95,
|
|
26723
|
+
tags: ["exam", "licensing"]
|
|
26724
|
+
},
|
|
26725
|
+
{
|
|
26726
|
+
id: "study-setup",
|
|
26727
|
+
messageIndex: 2,
|
|
26728
|
+
quote: "I study best at the kitchen counter with noise-canceling headphones",
|
|
26729
|
+
frame: "User's study setup",
|
|
26730
|
+
content: "The user studies best at the kitchen counter with noise-canceling headphones.",
|
|
26731
|
+
restatement: "The user studies best at the kitchen counter wearing noise-canceling headphones.",
|
|
26732
|
+
category: "preference",
|
|
26733
|
+
confidence: 0.9,
|
|
26734
|
+
tags: ["study", "environment"]
|
|
26735
|
+
},
|
|
26736
|
+
{
|
|
26737
|
+
id: "tutor-lena",
|
|
26738
|
+
messageIndex: 4,
|
|
26739
|
+
quote: "My tutor Lena charges a sliding scale",
|
|
26740
|
+
frame: "Lena's tutoring fee",
|
|
26741
|
+
content: "The user's tutor Lena charges a sliding scale.",
|
|
26742
|
+
restatement: "The user's tutor Lena uses a sliding scale for fees.",
|
|
26743
|
+
category: "fact",
|
|
26744
|
+
confidence: 0.89,
|
|
26745
|
+
tags: ["lena", "tutoring"]
|
|
26746
|
+
}
|
|
26747
|
+
]
|
|
26748
|
+
},
|
|
26749
|
+
{
|
|
26750
|
+
id: "lme-travel",
|
|
26751
|
+
dataset: "longmemeval",
|
|
26752
|
+
messages: [
|
|
26753
|
+
{
|
|
26754
|
+
speaker: "User",
|
|
26755
|
+
text: "I only fly out of the regional airport now; the major hub's security line ate two hours of my life."
|
|
26756
|
+
},
|
|
26757
|
+
{ speaker: "Assistant", text: "Regional airport only, noted." },
|
|
26758
|
+
{
|
|
26759
|
+
speaker: "User",
|
|
26760
|
+
text: "I collect vintage transit maps, and the 1968 one from the city tram network is my favorite piece."
|
|
26761
|
+
},
|
|
26762
|
+
{ speaker: "Assistant", text: "Vintage transit maps, 1968 tram favorite." },
|
|
26763
|
+
{
|
|
26764
|
+
speaker: "User",
|
|
26765
|
+
text: "My passport expires next June, so I renewed it early through the postal service."
|
|
26766
|
+
},
|
|
26767
|
+
{ speaker: "Assistant", text: "Passport renewed early, noted." },
|
|
26768
|
+
{
|
|
26769
|
+
speaker: "User",
|
|
26770
|
+
text: "I book window seats on day flights and aisle seats on red-eyes, no exceptions."
|
|
26771
|
+
},
|
|
26772
|
+
{ speaker: "Assistant", text: "Window by day, aisle by night." }
|
|
26773
|
+
],
|
|
26774
|
+
facts: [
|
|
26775
|
+
{
|
|
26776
|
+
id: "regional-airport",
|
|
26777
|
+
messageIndex: 0,
|
|
26778
|
+
quote: "I only fly out of the regional airport now",
|
|
26779
|
+
frame: "User's airport choice",
|
|
26780
|
+
content: "The user only flies out of the regional airport now.",
|
|
26781
|
+
restatement: "The user flies only out of the regional airport now.",
|
|
26782
|
+
category: "preference",
|
|
26783
|
+
confidence: 0.92,
|
|
26784
|
+
tags: ["travel", "airport"]
|
|
26785
|
+
},
|
|
26786
|
+
{
|
|
26787
|
+
id: "transit-maps",
|
|
26788
|
+
messageIndex: 2,
|
|
26789
|
+
quote: "I collect vintage transit maps",
|
|
26790
|
+
frame: "User's transit map collection",
|
|
26791
|
+
content: "The user collects vintage transit maps.",
|
|
26792
|
+
restatement: "The user collects old transit maps.",
|
|
26793
|
+
category: "preference",
|
|
26794
|
+
confidence: 0.88,
|
|
26795
|
+
tags: ["travel", "collection"]
|
|
26796
|
+
},
|
|
26797
|
+
{
|
|
26798
|
+
id: "passport-renewal",
|
|
26799
|
+
messageIndex: 4,
|
|
26800
|
+
quote: "My passport expires next June, so I renewed it early",
|
|
26801
|
+
frame: "User's passport renewal",
|
|
26802
|
+
content: "The user renewed their passport early; it expires next June.",
|
|
26803
|
+
restatement: "The user renewed the passport early since it expires next June.",
|
|
26804
|
+
category: "fact",
|
|
26805
|
+
confidence: 0.93,
|
|
26806
|
+
tags: ["travel", "passport"]
|
|
26807
|
+
},
|
|
26808
|
+
{
|
|
26809
|
+
id: "seat-preference",
|
|
26810
|
+
messageIndex: 6,
|
|
26811
|
+
quote: "I book window seats on day flights and aisle seats on red-eyes",
|
|
26812
|
+
frame: "User's seat preferences",
|
|
26813
|
+
content: "The user books window seats on day flights and aisle seats on red-eyes.",
|
|
26814
|
+
restatement: "The user takes window seats on day flights, aisle on red-eyes.",
|
|
26815
|
+
category: "preference",
|
|
26816
|
+
confidence: 0.91,
|
|
26817
|
+
tags: ["travel", "seating"]
|
|
26818
|
+
}
|
|
26819
|
+
]
|
|
26820
|
+
}
|
|
26821
|
+
];
|
|
26822
|
+
var SPAN_BENCH_SMOKE_FIXTURE = [
|
|
26823
|
+
SPAN_BENCH_FIXTURE[0],
|
|
26824
|
+
SPAN_BENCH_FIXTURE[3]
|
|
26825
|
+
];
|
|
26826
|
+
|
|
26827
|
+
// src/benchmarks/remnic/extraction-span-mode/segment.ts
|
|
26828
|
+
import { stampSpanSource } from "@remnic/core/extraction-span-source-hash";
|
|
26829
|
+
function renderSegment(conversation) {
|
|
26830
|
+
const lines = [
|
|
26831
|
+
"Numbered conversation segment. Character offsets index each message's text",
|
|
26832
|
+
"after the speaker prefix (offsets are [charStart, charEnd), end-exclusive).",
|
|
26833
|
+
"The printed char count and source hash/length identify that message text.",
|
|
26834
|
+
"For each memory, return sourceMessageIndex plus a verbatim supporting",
|
|
26835
|
+
"span's charStart/charEnd, and a frame of at most 15 words that makes the",
|
|
26836
|
+
"span self-contained (resolve pronouns, name the subject).",
|
|
26837
|
+
""
|
|
26838
|
+
];
|
|
26839
|
+
const messages = conversation.messages.map((message, index) => {
|
|
26840
|
+
const stamp = stampSpanSource(message.text);
|
|
26841
|
+
lines.push(
|
|
26842
|
+
`[${index}] ${message.speaker} (${message.text.length} chars, hash ${stamp.hash}, length ${stamp.length}): ${message.text}`
|
|
26843
|
+
);
|
|
26844
|
+
return {
|
|
26845
|
+
index,
|
|
26846
|
+
speaker: message.speaker,
|
|
26847
|
+
text: message.text,
|
|
26848
|
+
stamp
|
|
26849
|
+
};
|
|
26850
|
+
});
|
|
26851
|
+
return { messages, prompt: lines.join("\n") };
|
|
26852
|
+
}
|
|
26853
|
+
|
|
26854
|
+
// src/benchmarks/remnic/extraction-span-mode/materialize.ts
|
|
26855
|
+
import { verifySpanSource } from "@remnic/core/extraction-span-source-hash";
|
|
26856
|
+
var SPAN_MAX_SLICE_CHARS = 400;
|
|
26857
|
+
var SPAN_MAX_FRAME_WORDS = 15;
|
|
26858
|
+
function reject(reason, fallbackContent) {
|
|
26859
|
+
return { content: fallbackContent, outcome: "fallback", reason };
|
|
26860
|
+
}
|
|
26861
|
+
function materializeSpanFact(fact3, segmentMessages) {
|
|
26862
|
+
const fallbackContent = (fact3.content ?? fact3.span?.frame ?? "").trim();
|
|
26863
|
+
const span = fact3.span ?? null;
|
|
26864
|
+
if (span === null) {
|
|
26865
|
+
return reject("no_span", fallbackContent);
|
|
26866
|
+
}
|
|
26867
|
+
if (fallbackContent === "") {
|
|
26868
|
+
return { content: "", outcome: "fallback", reason: "missing_fallback_content" };
|
|
26869
|
+
}
|
|
26870
|
+
const { sourceMessageIndex, charStart, charEnd, frame } = span;
|
|
26871
|
+
if (!Number.isInteger(sourceMessageIndex) || sourceMessageIndex < 0 || sourceMessageIndex >= segmentMessages.length) {
|
|
26872
|
+
return reject("message_index_out_of_range", fallbackContent);
|
|
26873
|
+
}
|
|
26874
|
+
const message = segmentMessages[sourceMessageIndex];
|
|
26875
|
+
const stampCheck = verifySpanSource(message.text, message.stamp);
|
|
26876
|
+
if (!stampCheck.ok) {
|
|
26877
|
+
return reject(`source_${stampCheck.error}`, fallbackContent);
|
|
26878
|
+
}
|
|
26879
|
+
if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charStart < 0 || charEnd > message.text.length) {
|
|
26880
|
+
return reject("offsets_out_of_range", fallbackContent);
|
|
26881
|
+
}
|
|
26882
|
+
if (charStart >= charEnd) {
|
|
26883
|
+
return reject("empty_interval", fallbackContent);
|
|
26884
|
+
}
|
|
26885
|
+
const slice = message.text.slice(charStart, charEnd);
|
|
26886
|
+
if (slice.length > SPAN_MAX_SLICE_CHARS) {
|
|
26887
|
+
return reject("slice_too_long", fallbackContent);
|
|
26888
|
+
}
|
|
26889
|
+
if (slice.trim() === "") {
|
|
26890
|
+
return reject("blank_slice", fallbackContent);
|
|
26891
|
+
}
|
|
26892
|
+
const frameWords = frame.trim().split(/\s+/).filter(Boolean);
|
|
26893
|
+
if (frameWords.length === 0 || frameWords.length > SPAN_MAX_FRAME_WORDS) {
|
|
26894
|
+
return reject("frame_word_count", fallbackContent);
|
|
26895
|
+
}
|
|
26896
|
+
const trimmedFrame = frameWords.join(" ");
|
|
26897
|
+
const content = frameEndsWithPunctuation(trimmedFrame) ? `${trimmedFrame} ${slice}` : `${trimmedFrame}: ${slice}`;
|
|
26898
|
+
return {
|
|
26899
|
+
content,
|
|
26900
|
+
outcome: "span",
|
|
26901
|
+
quote: slice,
|
|
26902
|
+
charStart,
|
|
26903
|
+
charEnd,
|
|
26904
|
+
sourceMessageIndex
|
|
26905
|
+
};
|
|
26906
|
+
}
|
|
26907
|
+
function frameEndsWithPunctuation(frame) {
|
|
26908
|
+
return /[:—-]$/.test(frame);
|
|
26909
|
+
}
|
|
26910
|
+
|
|
26911
|
+
// src/benchmarks/remnic/extraction-span-mode/schema.ts
|
|
26912
|
+
import { z } from "zod";
|
|
26913
|
+
var SpanRefSchema = z.object({
|
|
26914
|
+
sourceMessageIndex: z.number(),
|
|
26915
|
+
charStart: z.number(),
|
|
26916
|
+
charEnd: z.number(),
|
|
26917
|
+
frame: z.string()
|
|
26918
|
+
});
|
|
26919
|
+
var CategorySchema = z.enum([
|
|
26920
|
+
"fact",
|
|
26921
|
+
"preference",
|
|
26922
|
+
"correction",
|
|
26923
|
+
"entity",
|
|
26924
|
+
"decision",
|
|
26925
|
+
"relationship",
|
|
26926
|
+
"principle",
|
|
26927
|
+
"commitment",
|
|
26928
|
+
"moment",
|
|
26929
|
+
"skill",
|
|
26930
|
+
"rule",
|
|
26931
|
+
"procedure",
|
|
26932
|
+
"reasoning_trace"
|
|
26933
|
+
]);
|
|
26934
|
+
var SpanModeFactSchema = z.object({
|
|
26935
|
+
category: CategorySchema,
|
|
26936
|
+
content: z.string().optional().nullable(),
|
|
26937
|
+
confidence: z.number().min(0).max(1),
|
|
26938
|
+
tags: z.array(z.string()),
|
|
26939
|
+
span: SpanRefSchema.optional().nullable()
|
|
26940
|
+
}).superRefine((value, ctx) => {
|
|
26941
|
+
if ((value.span === void 0 || value.span === null) && (value.content ?? "").trim() === "") {
|
|
26942
|
+
ctx.addIssue({
|
|
26943
|
+
code: z.ZodIssueCode.custom,
|
|
26944
|
+
message: "fact needs either a span or non-blank generated content"
|
|
26945
|
+
});
|
|
26946
|
+
}
|
|
26947
|
+
});
|
|
26948
|
+
var CurrentModeFactSchema = z.object({
|
|
26949
|
+
category: CategorySchema,
|
|
26950
|
+
content: z.string(),
|
|
26951
|
+
/**
|
|
26952
|
+
* Production parity: current-mode extraction also emits the verbatim
|
|
26953
|
+
* grounding quote (ExtractedFactSchema.quote, issue #1575). Span mode
|
|
26954
|
+
* replaces BOTH content and quote with offsets + frame.
|
|
26955
|
+
*/
|
|
26956
|
+
quote: z.string().optional().nullable(),
|
|
26957
|
+
confidence: z.number().min(0).max(1),
|
|
26958
|
+
tags: z.array(z.string()),
|
|
26959
|
+
span: SpanRefSchema.optional().nullable()
|
|
26960
|
+
});
|
|
26961
|
+
|
|
26962
|
+
// src/benchmarks/remnic/extraction-span-mode/fake-provider.ts
|
|
26963
|
+
import { estimateGeneratedTokens } from "@remnic/core/extraction-span-tokens";
|
|
26964
|
+
var MS_PER_OUTPUT_TOKEN = 40;
|
|
26965
|
+
var INVALID_SPAN_RATE = 0.04;
|
|
26966
|
+
var DRIFT_SPAN_RATE = 0.02;
|
|
26967
|
+
var DRIFT_CHARS = 3;
|
|
26968
|
+
function goldSpan(gold, conversation) {
|
|
26969
|
+
const text = conversation.messages[gold.messageIndex]?.text;
|
|
26970
|
+
if (text === void 0) {
|
|
26971
|
+
throw new Error(`fixture fact ${gold.id} references missing message ${gold.messageIndex}`);
|
|
26972
|
+
}
|
|
26973
|
+
const charStart = text.indexOf(gold.quote);
|
|
26974
|
+
if (charStart < 0) {
|
|
26975
|
+
throw new Error(`fixture fact ${gold.id} quote is not a verbatim substring of message ${gold.messageIndex}`);
|
|
26976
|
+
}
|
|
26977
|
+
return { charStart, charEnd: charStart + gold.quote.length };
|
|
26978
|
+
}
|
|
26979
|
+
function runFakeExtraction(conversation, mode, seed) {
|
|
26980
|
+
const rng = createSeededRandom((seed * 16777619 + stableConversationSalt(conversation.id)) % 4294967296);
|
|
26981
|
+
const rawFacts = conversation.facts.map((gold) => {
|
|
26982
|
+
const base = {
|
|
26983
|
+
category: gold.category,
|
|
26984
|
+
confidence: gold.confidence,
|
|
26985
|
+
tags: gold.tags
|
|
26986
|
+
};
|
|
26987
|
+
if (mode === "current") {
|
|
26988
|
+
const fact3 = { ...base, content: gold.restatement, quote: gold.quote };
|
|
26989
|
+
return fact3;
|
|
26990
|
+
}
|
|
26991
|
+
return emitSpanFact(gold, conversation, rng);
|
|
26992
|
+
});
|
|
26993
|
+
const responsePayload = JSON.stringify(rawFacts);
|
|
26994
|
+
const outputTokens = estimateGeneratedTokens(responsePayload.length);
|
|
26995
|
+
return {
|
|
26996
|
+
rawFacts,
|
|
26997
|
+
responsePayload,
|
|
26998
|
+
outputTokens,
|
|
26999
|
+
wallClockMs: outputTokens * MS_PER_OUTPUT_TOKEN,
|
|
27000
|
+
memoryEntryCount: rawFacts.length
|
|
27001
|
+
};
|
|
27002
|
+
}
|
|
27003
|
+
function stableConversationSalt(id) {
|
|
27004
|
+
let hash = 2166136261;
|
|
27005
|
+
for (let i = 0; i < id.length; i += 1) {
|
|
27006
|
+
hash ^= id.charCodeAt(i);
|
|
27007
|
+
hash = Math.imul(hash, 16777619);
|
|
27008
|
+
}
|
|
27009
|
+
return hash >>> 0;
|
|
27010
|
+
}
|
|
27011
|
+
function emitSpanFact(gold, conversation, rng) {
|
|
27012
|
+
const { charStart, charEnd } = goldSpan(gold, conversation);
|
|
27013
|
+
const messageLength = conversation.messages[gold.messageIndex].text.length;
|
|
27014
|
+
const roll = rng();
|
|
27015
|
+
const base = {
|
|
27016
|
+
category: gold.category,
|
|
27017
|
+
confidence: gold.confidence,
|
|
27018
|
+
tags: gold.tags,
|
|
27019
|
+
content: gold.frame
|
|
27020
|
+
};
|
|
27021
|
+
if (roll < INVALID_SPAN_RATE) {
|
|
27022
|
+
const variant = Math.floor(roll / INVALID_SPAN_RATE * 3) % 3;
|
|
27023
|
+
if (variant === 0) {
|
|
27024
|
+
return { ...base, span: { sourceMessageIndex: gold.messageIndex, charStart, charEnd: messageLength + 1, frame: gold.frame } };
|
|
27025
|
+
}
|
|
27026
|
+
if (variant === 1) {
|
|
27027
|
+
return { ...base, span: { sourceMessageIndex: gold.messageIndex, charStart, charEnd: charStart, frame: gold.frame } };
|
|
27028
|
+
}
|
|
27029
|
+
return { ...base, span: { sourceMessageIndex: gold.messageIndex, charStart: charEnd, charEnd: charStart, frame: gold.frame } };
|
|
27030
|
+
}
|
|
27031
|
+
if (roll < INVALID_SPAN_RATE + DRIFT_SPAN_RATE && charStart + DRIFT_CHARS < charEnd && charEnd + DRIFT_CHARS <= messageLength) {
|
|
27032
|
+
return {
|
|
27033
|
+
...base,
|
|
27034
|
+
span: {
|
|
27035
|
+
sourceMessageIndex: gold.messageIndex,
|
|
27036
|
+
charStart: charStart + DRIFT_CHARS,
|
|
27037
|
+
charEnd: charEnd + DRIFT_CHARS,
|
|
27038
|
+
frame: gold.frame
|
|
27039
|
+
}
|
|
27040
|
+
};
|
|
27041
|
+
}
|
|
27042
|
+
return {
|
|
27043
|
+
...base,
|
|
27044
|
+
span: { sourceMessageIndex: gold.messageIndex, charStart, charEnd, frame: gold.frame }
|
|
27045
|
+
};
|
|
27046
|
+
}
|
|
27047
|
+
|
|
27048
|
+
// src/benchmarks/remnic/extraction-span-mode/judge.ts
|
|
27049
|
+
function tokenizeForJudge(text) {
|
|
27050
|
+
return text.toLowerCase().split(/[^a-z0-9]+/).filter((token) => token.length > 0);
|
|
27051
|
+
}
|
|
27052
|
+
function judgeMemoryScore(memory, gold) {
|
|
27053
|
+
const goldTokens = tokenizeForJudge(gold);
|
|
27054
|
+
if (goldTokens.length === 0) {
|
|
27055
|
+
return 0;
|
|
27056
|
+
}
|
|
27057
|
+
const memoryTokens = tokenizeForJudge(memory);
|
|
27058
|
+
const available = /* @__PURE__ */ new Map();
|
|
27059
|
+
for (const token of memoryTokens) {
|
|
27060
|
+
available.set(token, (available.get(token) ?? 0) + 1);
|
|
27061
|
+
}
|
|
27062
|
+
let covered = 0;
|
|
27063
|
+
for (const token of goldTokens) {
|
|
27064
|
+
const count = available.get(token) ?? 0;
|
|
27065
|
+
if (count > 0) {
|
|
27066
|
+
covered += 1;
|
|
27067
|
+
available.set(token, count - 1);
|
|
27068
|
+
}
|
|
27069
|
+
}
|
|
27070
|
+
return covered / goldTokens.length * 100;
|
|
27071
|
+
}
|
|
27072
|
+
|
|
27073
|
+
// src/benchmarks/remnic/extraction-span-mode/runner.ts
|
|
27074
|
+
var extractionSpanModeDefinition = {
|
|
27075
|
+
id: "extraction-span-mode",
|
|
27076
|
+
title: "Extraction Span-Mode Phase A Gate",
|
|
27077
|
+
tier: "remnic",
|
|
27078
|
+
status: "ready",
|
|
27079
|
+
runnerAvailable: true,
|
|
27080
|
+
meta: {
|
|
27081
|
+
name: "extraction-span-mode",
|
|
27082
|
+
version: "1.0.0",
|
|
27083
|
+
description: "Deterministic fake-provider A/B of span-mode vs generated extraction with the issue #2333 Phase B gate (wall-clock, judge score, fallback rate).",
|
|
27084
|
+
category: "retrieval",
|
|
27085
|
+
citation: "arXiv 2602.03315 \xA75.2.4 Table 6; Remnic issue #2333"
|
|
27086
|
+
}
|
|
27087
|
+
};
|
|
27088
|
+
function newAccumulator() {
|
|
27089
|
+
return { conversationIds: [], judgeScores: [], wallClockMs: [], outputTokens: [], memoryEntries: [] };
|
|
27090
|
+
}
|
|
27091
|
+
function mean(values) {
|
|
27092
|
+
if (values.length === 0) {
|
|
27093
|
+
throw new Error("cannot average an empty sample");
|
|
27094
|
+
}
|
|
27095
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
27096
|
+
}
|
|
27097
|
+
async function runExtractionSpanModeBenchmark(options) {
|
|
27098
|
+
const seed = options.seed ?? 0;
|
|
27099
|
+
const conversations = loadConversations(options.mode, options.limit);
|
|
27100
|
+
const segmentByConversation = new Map(
|
|
27101
|
+
conversations.map((conversation) => [conversation.id, renderSegment(conversation)])
|
|
27102
|
+
);
|
|
27103
|
+
const tasks = [];
|
|
27104
|
+
const outcomes = [];
|
|
27105
|
+
const acc = { current: newAccumulator(), span: newAccumulator() };
|
|
27106
|
+
const modes = ["current", "span"];
|
|
27107
|
+
for (const conversation of conversations) {
|
|
27108
|
+
const segment = segmentByConversation.get(conversation.id);
|
|
27109
|
+
if (!segment) {
|
|
27110
|
+
throw new Error(`missing rendered segment for conversation ${conversation.id}`);
|
|
27111
|
+
}
|
|
27112
|
+
for (const mode of modes) {
|
|
27113
|
+
const run = runFakeExtraction(conversation, mode, seed);
|
|
27114
|
+
const factScores = [];
|
|
27115
|
+
const modeDetails = {
|
|
27116
|
+
dataset: conversation.dataset,
|
|
27117
|
+
mode,
|
|
27118
|
+
seed
|
|
27119
|
+
};
|
|
27120
|
+
if (mode === "current") {
|
|
27121
|
+
for (const [index, raw] of run.rawFacts.entries()) {
|
|
27122
|
+
const fact3 = CurrentModeFactSchema.parse(raw);
|
|
27123
|
+
const gold = conversation.facts[index];
|
|
27124
|
+
factScores.push(judgeMemoryScore(fact3.content, gold.content));
|
|
27125
|
+
}
|
|
27126
|
+
} else {
|
|
27127
|
+
const perFact = [];
|
|
27128
|
+
for (const [index, raw] of run.rawFacts.entries()) {
|
|
27129
|
+
const fact3 = SpanModeFactSchema.parse(raw);
|
|
27130
|
+
const gold = conversation.facts[index];
|
|
27131
|
+
const materialized = materializeSpanFact(fact3, segment.messages);
|
|
27132
|
+
outcomes.push(materialized.outcome);
|
|
27133
|
+
factScores.push(judgeMemoryScore(materialized.content, gold.content));
|
|
27134
|
+
perFact.push({
|
|
27135
|
+
factId: gold.id,
|
|
27136
|
+
outcome: materialized.outcome,
|
|
27137
|
+
reason: materialized.reason ?? null
|
|
27138
|
+
});
|
|
27139
|
+
}
|
|
27140
|
+
modeDetails.facts = perFact;
|
|
27141
|
+
}
|
|
27142
|
+
const judgeScore = mean(factScores);
|
|
27143
|
+
acc[mode].conversationIds.push(conversation.id);
|
|
27144
|
+
acc[mode].judgeScores.push(judgeScore);
|
|
27145
|
+
acc[mode].wallClockMs.push(run.wallClockMs);
|
|
27146
|
+
acc[mode].outputTokens.push(run.outputTokens);
|
|
27147
|
+
acc[mode].memoryEntries.push(run.memoryEntryCount);
|
|
27148
|
+
tasks.push({
|
|
27149
|
+
taskId: `${conversation.id}:${mode}`,
|
|
27150
|
+
question: `Extract memories from ${conversation.id} (${conversation.dataset}, ${mode} mode)`,
|
|
27151
|
+
expected: "gold-fact coverage",
|
|
27152
|
+
actual: `${mode} extraction via deterministic fake provider`,
|
|
27153
|
+
scores: {
|
|
27154
|
+
[`judge_score_${mode}`]: judgeScore,
|
|
27155
|
+
[`output_tokens_${mode}`]: run.outputTokens,
|
|
27156
|
+
[`wall_clock_ms_${mode}`]: run.wallClockMs,
|
|
27157
|
+
[`memory_entries_${mode}`]: run.memoryEntryCount
|
|
27158
|
+
},
|
|
27159
|
+
latencyMs: run.wallClockMs,
|
|
27160
|
+
tokens: { input: 0, output: run.outputTokens },
|
|
27161
|
+
details: modeDetails
|
|
27162
|
+
});
|
|
27163
|
+
}
|
|
27164
|
+
}
|
|
27165
|
+
const fallbackTally = tallySpanFallbacks(outcomes);
|
|
27166
|
+
if (fallbackTally.fallbackRatePct === null) {
|
|
27167
|
+
throw new Error("span mode produced no span attempts; the gate cannot be evaluated on an unmeasured run");
|
|
27168
|
+
}
|
|
27169
|
+
const wallClockReductionPct = (mean(acc.current.wallClockMs) - mean(acc.span.wallClockMs)) / mean(acc.current.wallClockMs) * 100;
|
|
27170
|
+
const judgeScoreDropPoints = mean(acc.current.judgeScores) - mean(acc.span.judgeScores);
|
|
27171
|
+
const verdict = evaluateSpanPhaseGate({
|
|
27172
|
+
wallClockReductionPct,
|
|
27173
|
+
judgeScoreDropPoints,
|
|
27174
|
+
fallbackRatePct: fallbackTally.fallbackRatePct
|
|
27175
|
+
});
|
|
27176
|
+
const comparison = {
|
|
27177
|
+
model: "deterministic-fake-provider (synthetic; no real model runs)",
|
|
27178
|
+
seed,
|
|
27179
|
+
conversations: conversations.length,
|
|
27180
|
+
perConversation: {
|
|
27181
|
+
judgeScoreCurrent: mean(acc.current.judgeScores),
|
|
27182
|
+
judgeScoreSpan: mean(acc.span.judgeScores),
|
|
27183
|
+
wallClockMsCurrent: mean(acc.current.wallClockMs),
|
|
27184
|
+
wallClockMsSpan: mean(acc.span.wallClockMs),
|
|
27185
|
+
outputTokensCurrent: mean(acc.current.outputTokens),
|
|
27186
|
+
outputTokensSpan: mean(acc.span.outputTokens),
|
|
27187
|
+
memoryEntriesCurrent: mean(acc.current.memoryEntries),
|
|
27188
|
+
memoryEntriesSpan: mean(acc.span.memoryEntries)
|
|
27189
|
+
},
|
|
27190
|
+
wallClockReductionPct,
|
|
27191
|
+
outputTokenReductionPct: (mean(acc.current.outputTokens) - mean(acc.span.outputTokens)) / mean(acc.current.outputTokens) * 100,
|
|
27192
|
+
judgeScoreDropPoints,
|
|
27193
|
+
spanAttempts: fallbackTally.attempts,
|
|
27194
|
+
spanFallbacks: fallbackTally.fallbacks,
|
|
27195
|
+
fallbackRatePct: fallbackTally.fallbackRatePct,
|
|
27196
|
+
costModel: {
|
|
27197
|
+
msPerOutputToken: MS_PER_OUTPUT_TOKEN,
|
|
27198
|
+
decodeBound: true,
|
|
27199
|
+
invalidSpanRate: INVALID_SPAN_RATE,
|
|
27200
|
+
driftSpanRate: DRIFT_SPAN_RATE
|
|
27201
|
+
},
|
|
27202
|
+
gate: {
|
|
27203
|
+
thresholds: {
|
|
27204
|
+
minWallClockReductionPct: 20,
|
|
27205
|
+
maxJudgeDropPoints: 2,
|
|
27206
|
+
maxFallbackRatePct: 15
|
|
27207
|
+
},
|
|
27208
|
+
verdict
|
|
27209
|
+
}
|
|
27210
|
+
};
|
|
27211
|
+
tasks.push({
|
|
27212
|
+
taskId: "span-phase-gate",
|
|
27213
|
+
question: "Does span-mode extraction clear the Phase B gate?",
|
|
27214
|
+
expected: "wall-clock -20%+, judge drop <2, fallback <15%",
|
|
27215
|
+
actual: verdict.pass ? "gate cleared" : `gate failed: ${verdict.failed.join(", ")}`,
|
|
27216
|
+
scores: {
|
|
27217
|
+
gate_pass: verdict.pass ? 1 : 0,
|
|
27218
|
+
wall_clock_reduction_pct: wallClockReductionPct,
|
|
27219
|
+
judge_score_drop_points: judgeScoreDropPoints,
|
|
27220
|
+
fallback_rate_pct: fallbackTally.fallbackRatePct
|
|
27221
|
+
},
|
|
27222
|
+
latencyMs: 0,
|
|
27223
|
+
tokens: { input: 0, output: 0 },
|
|
27224
|
+
goldMemories: [],
|
|
27225
|
+
details: { comparison }
|
|
27226
|
+
});
|
|
27227
|
+
const totalOutputTokens = acc.current.outputTokens.reduce((a, b) => a + b, 0) + acc.span.outputTokens.reduce((a, b) => a + b, 0);
|
|
27228
|
+
const totalWallClockMs = acc.current.wallClockMs.reduce((a, b) => a + b, 0) + acc.span.wallClockMs.reduce((a, b) => a + b, 0);
|
|
27229
|
+
const measuredTaskCount = acc.current.wallClockMs.length + acc.span.wallClockMs.length;
|
|
27230
|
+
return {
|
|
27231
|
+
meta: {
|
|
27232
|
+
id: randomUUID12(),
|
|
27233
|
+
benchmark: options.benchmark.id,
|
|
27234
|
+
benchmarkTier: options.benchmark.tier,
|
|
27235
|
+
version: options.benchmark.meta.version,
|
|
27236
|
+
remnicVersion: await getRemnicVersion(),
|
|
27237
|
+
gitSha: getGitSha(),
|
|
27238
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
27239
|
+
mode: options.mode,
|
|
27240
|
+
runCount: 1,
|
|
27241
|
+
seeds: [seed]
|
|
27242
|
+
},
|
|
27243
|
+
config: {
|
|
27244
|
+
systemProvider: options.systemProvider ?? null,
|
|
27245
|
+
judgeProvider: options.judgeProvider ?? null,
|
|
27246
|
+
adapterMode: options.adapterMode ?? "direct",
|
|
27247
|
+
remnicConfig: {
|
|
27248
|
+
spanBench: {
|
|
27249
|
+
provider: "deterministic-fake",
|
|
27250
|
+
datasets: ["locomo-synthetic", "longmemeval-synthetic"],
|
|
27251
|
+
seed
|
|
27252
|
+
}
|
|
27253
|
+
}
|
|
27254
|
+
},
|
|
27255
|
+
cost: {
|
|
27256
|
+
totalTokens: totalOutputTokens,
|
|
27257
|
+
inputTokens: 0,
|
|
27258
|
+
outputTokens: totalOutputTokens,
|
|
27259
|
+
estimatedCostUsd: 0,
|
|
27260
|
+
totalLatencyMs: totalWallClockMs,
|
|
27261
|
+
meanQueryLatencyMs: measuredTaskCount > 0 ? totalWallClockMs / measuredTaskCount : 0
|
|
27262
|
+
},
|
|
27263
|
+
results: {
|
|
27264
|
+
tasks,
|
|
27265
|
+
aggregates: aggregateTaskScores(tasks.map((task) => task.scores))
|
|
27266
|
+
},
|
|
27267
|
+
environment: {
|
|
27268
|
+
os: process.platform,
|
|
27269
|
+
nodeVersion: process.version,
|
|
27270
|
+
hardware: process.arch
|
|
27271
|
+
}
|
|
27272
|
+
};
|
|
27273
|
+
}
|
|
27274
|
+
function loadConversations(mode, limit) {
|
|
27275
|
+
const base = mode === "quick" ? SPAN_BENCH_SMOKE_FIXTURE : SPAN_BENCH_FIXTURE;
|
|
27276
|
+
if (limit === void 0) {
|
|
27277
|
+
return base;
|
|
27278
|
+
}
|
|
27279
|
+
if (!Number.isInteger(limit) || limit <= 0) {
|
|
27280
|
+
throw new Error("extraction-span-mode limit must be a positive integer");
|
|
27281
|
+
}
|
|
27282
|
+
const limited = base.slice(0, limit);
|
|
27283
|
+
if (limited.length === 0) {
|
|
27284
|
+
throw new Error("extraction-span-mode fixture is empty after applying the requested limit.");
|
|
27285
|
+
}
|
|
27286
|
+
return limited;
|
|
27287
|
+
}
|
|
27288
|
+
|
|
27289
|
+
// src/benchmarks/remnic/enrichment-fidelity/runner.ts
|
|
27290
|
+
import { randomUUID as randomUUID13 } from "crypto";
|
|
26108
27291
|
import { EnrichmentProviderRegistry, runEnrichmentPipeline } from "@remnic/core";
|
|
26109
27292
|
|
|
26110
27293
|
// src/benchmarks/remnic/enrichment-fidelity/fixture.ts
|
|
@@ -26380,7 +27563,7 @@ async function runEnrichmentFidelityBenchmark(options) {
|
|
|
26380
27563
|
);
|
|
26381
27564
|
return {
|
|
26382
27565
|
meta: {
|
|
26383
|
-
id:
|
|
27566
|
+
id: randomUUID13(),
|
|
26384
27567
|
benchmark: options.benchmark.id,
|
|
26385
27568
|
benchmarkTier: options.benchmark.tier,
|
|
26386
27569
|
version: options.benchmark.meta.version,
|
|
@@ -26452,7 +27635,7 @@ function constantAggregate2(value) {
|
|
|
26452
27635
|
// src/benchmarks/remnic/entity-consolidation/runner.ts
|
|
26453
27636
|
import os4 from "os";
|
|
26454
27637
|
import path17 from "path";
|
|
26455
|
-
import { randomUUID as
|
|
27638
|
+
import { randomUUID as randomUUID14 } from "crypto";
|
|
26456
27639
|
import { mkdtemp as mkdtemp4, rm as rm5 } from "fs/promises";
|
|
26457
27640
|
import { StorageManager as StorageManager2 } from "@remnic/core";
|
|
26458
27641
|
|
|
@@ -26563,7 +27746,7 @@ async function runEntityConsolidationBenchmark(options) {
|
|
|
26563
27746
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
26564
27747
|
return {
|
|
26565
27748
|
meta: {
|
|
26566
|
-
id:
|
|
27749
|
+
id: randomUUID14(),
|
|
26567
27750
|
benchmark: options.benchmark.id,
|
|
26568
27751
|
benchmarkTier: options.benchmark.tier,
|
|
26569
27752
|
version: options.benchmark.meta.version,
|
|
@@ -26792,7 +27975,7 @@ function parseNonNegativeInt(rawValue) {
|
|
|
26792
27975
|
}
|
|
26793
27976
|
|
|
26794
27977
|
// src/benchmarks/remnic/page-versioning/runner.ts
|
|
26795
|
-
import { randomUUID as
|
|
27978
|
+
import { randomUUID as randomUUID15 } from "crypto";
|
|
26796
27979
|
import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile16, rm as rm6, writeFile as writeFile5 } from "fs/promises";
|
|
26797
27980
|
import os5 from "os";
|
|
26798
27981
|
import path18 from "path";
|
|
@@ -26908,7 +28091,7 @@ async function runPageVersioningBenchmark(options, dependencyOverrides = {}) {
|
|
|
26908
28091
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
26909
28092
|
return {
|
|
26910
28093
|
meta: {
|
|
26911
|
-
id:
|
|
28094
|
+
id: randomUUID15(),
|
|
26912
28095
|
benchmark: options.benchmark.id,
|
|
26913
28096
|
benchmarkTier: options.benchmark.tier,
|
|
26914
28097
|
version: options.benchmark.meta.version,
|
|
@@ -27066,7 +28249,7 @@ function versioningConfig(overrides) {
|
|
|
27066
28249
|
}
|
|
27067
28250
|
|
|
27068
28251
|
// src/benchmarks/remnic/retrieval-personalization/runner.ts
|
|
27069
|
-
import { randomUUID as
|
|
28252
|
+
import { randomUUID as randomUUID16 } from "crypto";
|
|
27070
28253
|
|
|
27071
28254
|
// src/benchmarks/remnic/retrieval-page-ids.ts
|
|
27072
28255
|
function extractRankedPageIds(recallText, pages, options = {}) {
|
|
@@ -27753,7 +28936,7 @@ async function runRetrievalPersonalizationBenchmark(options) {
|
|
|
27753
28936
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
27754
28937
|
return {
|
|
27755
28938
|
meta: {
|
|
27756
|
-
id:
|
|
28939
|
+
id: randomUUID16(),
|
|
27757
28940
|
benchmark: options.benchmark.id,
|
|
27758
28941
|
benchmarkTier: options.benchmark.tier,
|
|
27759
28942
|
version: options.benchmark.meta.version,
|
|
@@ -27801,7 +28984,7 @@ function loadCases6(mode, limit) {
|
|
|
27801
28984
|
}
|
|
27802
28985
|
|
|
27803
28986
|
// src/benchmarks/remnic/retrieval-temporal/runner.ts
|
|
27804
|
-
import { randomUUID as
|
|
28987
|
+
import { randomUUID as randomUUID17 } from "crypto";
|
|
27805
28988
|
|
|
27806
28989
|
// src/benchmarks/remnic/retrieval-temporal/fixture.ts
|
|
27807
28990
|
var RETRIEVAL_TEMPORAL_FIXTURE = buildFixture(SCHEMA_TIER_FIXTURE);
|
|
@@ -27904,7 +29087,7 @@ async function runRetrievalTemporalBenchmark(options) {
|
|
|
27904
29087
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
27905
29088
|
return {
|
|
27906
29089
|
meta: {
|
|
27907
|
-
id:
|
|
29090
|
+
id: randomUUID17(),
|
|
27908
29091
|
benchmark: options.benchmark.id,
|
|
27909
29092
|
benchmarkTier: options.benchmark.tier,
|
|
27910
29093
|
version: options.benchmark.meta.version,
|
|
@@ -28039,7 +29222,7 @@ function matchingPageIds(rankedPageIds, sample) {
|
|
|
28039
29222
|
}
|
|
28040
29223
|
|
|
28041
29224
|
// src/benchmarks/remnic/retrieval-direct-answer/runner.ts
|
|
28042
|
-
import { randomUUID as
|
|
29225
|
+
import { randomUUID as randomUUID18 } from "crypto";
|
|
28043
29226
|
import { isDirectAnswerEligible } from "@remnic/core";
|
|
28044
29227
|
|
|
28045
29228
|
// src/benchmarks/remnic/retrieval-direct-answer/fixture.ts
|
|
@@ -28370,7 +29553,7 @@ async function runRetrievalDirectAnswerBenchmark(options) {
|
|
|
28370
29553
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
28371
29554
|
return {
|
|
28372
29555
|
meta: {
|
|
28373
|
-
id:
|
|
29556
|
+
id: randomUUID18(),
|
|
28374
29557
|
benchmark: options.benchmark.id,
|
|
28375
29558
|
benchmarkTier: options.benchmark.tier,
|
|
28376
29559
|
version: options.benchmark.meta.version,
|
|
@@ -28412,7 +29595,7 @@ async function runRetrievalDirectAnswerBenchmark(options) {
|
|
|
28412
29595
|
}
|
|
28413
29596
|
|
|
28414
29597
|
// src/benchmarks/remnic/retrieval-graph/runner.ts
|
|
28415
|
-
import { randomUUID as
|
|
29598
|
+
import { randomUUID as randomUUID19 } from "crypto";
|
|
28416
29599
|
import {
|
|
28417
29600
|
buildGraphFromMemories,
|
|
28418
29601
|
queryGraph
|
|
@@ -28594,7 +29777,7 @@ async function runRetrievalGraphBenchmark(options) {
|
|
|
28594
29777
|
const meanOff = tasks.length > 0 ? totalOff / tasks.length : 0;
|
|
28595
29778
|
return {
|
|
28596
29779
|
meta: {
|
|
28597
|
-
id:
|
|
29780
|
+
id: randomUUID19(),
|
|
28598
29781
|
benchmark: options.benchmark.id,
|
|
28599
29782
|
benchmarkTier: options.benchmark.tier,
|
|
28600
29783
|
version: options.benchmark.meta.version,
|
|
@@ -28660,7 +29843,7 @@ function loadCases8(mode, limit) {
|
|
|
28660
29843
|
}
|
|
28661
29844
|
|
|
28662
29845
|
// src/benchmarks/remnic/retrieval-reasoning-trace/runner.ts
|
|
28663
|
-
import { randomUUID as
|
|
29846
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
28664
29847
|
import {
|
|
28665
29848
|
applyReasoningTraceBoost,
|
|
28666
29849
|
isReasoningTracePath,
|
|
@@ -28928,7 +30111,7 @@ async function runRetrievalReasoningTraceBenchmark(options) {
|
|
|
28928
30111
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
28929
30112
|
return {
|
|
28930
30113
|
meta: {
|
|
28931
|
-
id:
|
|
30114
|
+
id: randomUUID20(),
|
|
28932
30115
|
benchmark: options.benchmark.id,
|
|
28933
30116
|
benchmarkTier: options.benchmark.tier,
|
|
28934
30117
|
version: options.benchmark.meta.version,
|
|
@@ -28970,7 +30153,7 @@ async function runRetrievalReasoningTraceBenchmark(options) {
|
|
|
28970
30153
|
}
|
|
28971
30154
|
|
|
28972
30155
|
// src/benchmarks/remnic/coding-recall/runner.ts
|
|
28973
|
-
import { randomUUID as
|
|
30156
|
+
import { randomUUID as randomUUID21 } from "crypto";
|
|
28974
30157
|
import { rankReviewCandidates } from "@remnic/core";
|
|
28975
30158
|
|
|
28976
30159
|
// src/benchmarks/remnic/coding-recall/fixture.ts
|
|
@@ -29249,7 +30432,7 @@ async function runCodingRecallBenchmark(options) {
|
|
|
29249
30432
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
29250
30433
|
return {
|
|
29251
30434
|
meta: {
|
|
29252
|
-
id:
|
|
30435
|
+
id: randomUUID21(),
|
|
29253
30436
|
benchmark: options.benchmark.id,
|
|
29254
30437
|
benchmarkTier: options.benchmark.tier,
|
|
29255
30438
|
version: options.benchmark.meta.version,
|
|
@@ -29315,7 +30498,7 @@ function loadCases9(mode, limit) {
|
|
|
29315
30498
|
}
|
|
29316
30499
|
|
|
29317
30500
|
// src/benchmarks/remnic/procedural-recall/runner.ts
|
|
29318
|
-
import { randomUUID as
|
|
30501
|
+
import { randomUUID as randomUUID22 } from "crypto";
|
|
29319
30502
|
import { mkdtemp as mkdtemp6, rm as rm7 } from "fs/promises";
|
|
29320
30503
|
import os6 from "os";
|
|
29321
30504
|
import path19 from "path";
|
|
@@ -29501,7 +30684,7 @@ ${body}`,
|
|
|
29501
30684
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
29502
30685
|
return {
|
|
29503
30686
|
meta: {
|
|
29504
|
-
id:
|
|
30687
|
+
id: randomUUID22(),
|
|
29505
30688
|
benchmark: options.benchmark.id,
|
|
29506
30689
|
benchmarkTier: options.benchmark.tier,
|
|
29507
30690
|
version: options.benchmark.meta.version,
|
|
@@ -29539,7 +30722,7 @@ ${body}`,
|
|
|
29539
30722
|
}
|
|
29540
30723
|
|
|
29541
30724
|
// src/benchmarks/remnic/ingestion-entity-recall/runner.ts
|
|
29542
|
-
import { randomUUID as
|
|
30725
|
+
import { randomUUID as randomUUID23 } from "crypto";
|
|
29543
30726
|
import { mkdtemp as mkdtemp7, writeFile as writeFile6, rm as rm8, mkdir as mkdir6, realpath as realpath5 } from "fs/promises";
|
|
29544
30727
|
import { tmpdir as tmpdir2 } from "os";
|
|
29545
30728
|
import path20 from "path";
|
|
@@ -30138,7 +31321,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
30138
31321
|
const remnicVersion = await getRemnicVersion();
|
|
30139
31322
|
return {
|
|
30140
31323
|
meta: {
|
|
30141
|
-
id:
|
|
31324
|
+
id: randomUUID23(),
|
|
30142
31325
|
benchmark: options.benchmark.id,
|
|
30143
31326
|
benchmarkTier: options.benchmark.tier,
|
|
30144
31327
|
version: options.benchmark.meta.version,
|
|
@@ -30176,7 +31359,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
30176
31359
|
}
|
|
30177
31360
|
|
|
30178
31361
|
// src/benchmarks/remnic/ingestion-schema-completeness/runner.ts
|
|
30179
|
-
import { randomUUID as
|
|
31362
|
+
import { randomUUID as randomUUID24 } from "crypto";
|
|
30180
31363
|
import { mkdtemp as mkdtemp8, writeFile as writeFile7, rm as rm9, mkdir as mkdir7, realpath as realpath6 } from "fs/promises";
|
|
30181
31364
|
import { tmpdir as tmpdir3 } from "os";
|
|
30182
31365
|
import path21 from "path";
|
|
@@ -30235,7 +31418,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
30235
31418
|
const remnicVersion2 = await getRemnicVersion();
|
|
30236
31419
|
return {
|
|
30237
31420
|
meta: {
|
|
30238
|
-
id:
|
|
31421
|
+
id: randomUUID24(),
|
|
30239
31422
|
benchmark: options.benchmark.id,
|
|
30240
31423
|
benchmarkTier: options.benchmark.tier,
|
|
30241
31424
|
version: options.benchmark.meta.version,
|
|
@@ -30308,7 +31491,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
30308
31491
|
const remnicVersion = await getRemnicVersion();
|
|
30309
31492
|
return {
|
|
30310
31493
|
meta: {
|
|
30311
|
-
id:
|
|
31494
|
+
id: randomUUID24(),
|
|
30312
31495
|
benchmark: options.benchmark.id,
|
|
30313
31496
|
benchmarkTier: options.benchmark.tier,
|
|
30314
31497
|
version: options.benchmark.meta.version,
|
|
@@ -30349,7 +31532,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
30349
31532
|
}
|
|
30350
31533
|
|
|
30351
31534
|
// src/benchmarks/remnic/ingestion-backlink-f1/runner.ts
|
|
30352
|
-
import { randomUUID as
|
|
31535
|
+
import { randomUUID as randomUUID25 } from "crypto";
|
|
30353
31536
|
import { mkdtemp as mkdtemp9, writeFile as writeFile8, rm as rm10, mkdir as mkdir8, realpath as realpath7 } from "fs/promises";
|
|
30354
31537
|
import { tmpdir as tmpdir4 } from "os";
|
|
30355
31538
|
import path22 from "path";
|
|
@@ -30409,7 +31592,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
30409
31592
|
const remnicVersion = await getRemnicVersion();
|
|
30410
31593
|
return {
|
|
30411
31594
|
meta: {
|
|
30412
|
-
id:
|
|
31595
|
+
id: randomUUID25(),
|
|
30413
31596
|
benchmark: options.benchmark.id,
|
|
30414
31597
|
benchmarkTier: options.benchmark.tier,
|
|
30415
31598
|
version: options.benchmark.meta.version,
|
|
@@ -30450,7 +31633,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
30450
31633
|
}
|
|
30451
31634
|
|
|
30452
31635
|
// src/benchmarks/remnic/ingestion-setup-friction/runner.ts
|
|
30453
|
-
import { randomUUID as
|
|
31636
|
+
import { randomUUID as randomUUID26 } from "crypto";
|
|
30454
31637
|
import { mkdtemp as mkdtemp10, writeFile as writeFile9, rm as rm11, mkdir as mkdir9, realpath as realpath8 } from "fs/promises";
|
|
30455
31638
|
import { tmpdir as tmpdir5 } from "os";
|
|
30456
31639
|
import path23 from "path";
|
|
@@ -30515,7 +31698,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
30515
31698
|
const remnicVersion = await getRemnicVersion();
|
|
30516
31699
|
return {
|
|
30517
31700
|
meta: {
|
|
30518
|
-
id:
|
|
31701
|
+
id: randomUUID26(),
|
|
30519
31702
|
benchmark: options.benchmark.id,
|
|
30520
31703
|
benchmarkTier: options.benchmark.tier,
|
|
30521
31704
|
version: options.benchmark.meta.version,
|
|
@@ -30556,7 +31739,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
30556
31739
|
}
|
|
30557
31740
|
|
|
30558
31741
|
// src/benchmarks/remnic/ingestion-citation-accuracy/runner.ts
|
|
30559
|
-
import { randomUUID as
|
|
31742
|
+
import { randomUUID as randomUUID27 } from "crypto";
|
|
30560
31743
|
import { mkdtemp as mkdtemp11, writeFile as writeFile10, rm as rm12, mkdir as mkdir10, realpath as realpath9 } from "fs/promises";
|
|
30561
31744
|
import { tmpdir as tmpdir6 } from "os";
|
|
30562
31745
|
import path24 from "path";
|
|
@@ -30699,7 +31882,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
30699
31882
|
const remnicVersion2 = await getRemnicVersion();
|
|
30700
31883
|
return {
|
|
30701
31884
|
meta: {
|
|
30702
|
-
id:
|
|
31885
|
+
id: randomUUID27(),
|
|
30703
31886
|
benchmark: options.benchmark.id,
|
|
30704
31887
|
benchmarkTier: options.benchmark.tier,
|
|
30705
31888
|
version: options.benchmark.meta.version,
|
|
@@ -30817,7 +32000,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
30817
32000
|
const remnicVersion = await getRemnicVersion();
|
|
30818
32001
|
return {
|
|
30819
32002
|
meta: {
|
|
30820
|
-
id:
|
|
32003
|
+
id: randomUUID27(),
|
|
30821
32004
|
benchmark: options.benchmark.id,
|
|
30822
32005
|
benchmarkTier: options.benchmark.tier,
|
|
30823
32006
|
version: options.benchmark.meta.version,
|
|
@@ -31040,7 +32223,7 @@ var ASSISTANT_MORNING_BRIEF_SCENARIOS = [
|
|
|
31040
32223
|
var ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS = ASSISTANT_MORNING_BRIEF_SCENARIOS.slice(0, 2);
|
|
31041
32224
|
|
|
31042
32225
|
// src/benchmarks/remnic/_assistant-common/runner.ts
|
|
31043
|
-
import { randomUUID as
|
|
32226
|
+
import { randomUUID as randomUUID28 } from "crypto";
|
|
31044
32227
|
import path26 from "path";
|
|
31045
32228
|
|
|
31046
32229
|
// src/run-seeds.ts
|
|
@@ -31062,7 +32245,7 @@ function buildBenchmarkRunSeeds(runCount, baseSeed) {
|
|
|
31062
32245
|
// src/stats/bootstrap.ts
|
|
31063
32246
|
var DEFAULT_ITERATIONS = 1e3;
|
|
31064
32247
|
var DEFAULT_LEVEL = 0.95;
|
|
31065
|
-
function
|
|
32248
|
+
function mean2(values) {
|
|
31066
32249
|
if (values.length === 0) {
|
|
31067
32250
|
throw new Error("bootstrap requires at least one value");
|
|
31068
32251
|
}
|
|
@@ -31101,7 +32284,7 @@ function createBootstrapMeans(values, {
|
|
|
31101
32284
|
const pickedIndex = Math.floor(random() * values.length);
|
|
31102
32285
|
sample.push(values[pickedIndex]);
|
|
31103
32286
|
}
|
|
31104
|
-
samples.push(
|
|
32287
|
+
samples.push(mean2(sample));
|
|
31105
32288
|
}
|
|
31106
32289
|
return samples;
|
|
31107
32290
|
}
|
|
@@ -31554,7 +32737,7 @@ async function runAssistantBenchmark(definition, scenarios, resolved, runnerOpti
|
|
|
31554
32737
|
const totalSeedExecutions = tasks.length * runCount;
|
|
31555
32738
|
return {
|
|
31556
32739
|
meta: {
|
|
31557
|
-
id:
|
|
32740
|
+
id: randomUUID28(),
|
|
31558
32741
|
benchmark: definition.id,
|
|
31559
32742
|
benchmarkTier: definition.tier,
|
|
31560
32743
|
version: definition.meta.version,
|
|
@@ -32150,7 +33333,7 @@ async function runAssistantSynthesisBenchmark(options) {
|
|
|
32150
33333
|
}
|
|
32151
33334
|
|
|
32152
33335
|
// src/benchmarks/remnic/buffer-surprise-trigger/runner.ts
|
|
32153
|
-
import { randomUUID as
|
|
33336
|
+
import { randomUUID as randomUUID29 } from "crypto";
|
|
32154
33337
|
import path27 from "path";
|
|
32155
33338
|
import os7 from "os";
|
|
32156
33339
|
import { mkdir as mkdir11, rm as rm13 } from "fs/promises";
|
|
@@ -32384,7 +33567,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
32384
33567
|
const cases = loadCases10(options.mode, options.limit);
|
|
32385
33568
|
const tmpRoot = path27.join(
|
|
32386
33569
|
os7.tmpdir(),
|
|
32387
|
-
`remnic-bench-buffer-surprise-${
|
|
33570
|
+
`remnic-bench-buffer-surprise-${randomUUID29()}`
|
|
32388
33571
|
);
|
|
32389
33572
|
await mkdir11(tmpRoot, { recursive: true });
|
|
32390
33573
|
const tasks = [];
|
|
@@ -32411,7 +33594,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
32411
33594
|
const remnicVersion = await getRemnicVersion();
|
|
32412
33595
|
return {
|
|
32413
33596
|
meta: {
|
|
32414
|
-
id:
|
|
33597
|
+
id: randomUUID29(),
|
|
32415
33598
|
benchmark: options.benchmark.id,
|
|
32416
33599
|
benchmarkTier: options.benchmark.tier,
|
|
32417
33600
|
version: options.benchmark.meta.version,
|
|
@@ -32524,10 +33707,10 @@ async function runSingleCase(caseDef, options) {
|
|
|
32524
33707
|
for (let i = 1; i < flushTurnIndices.length; i += 1) {
|
|
32525
33708
|
turnsBetween.push(flushTurnIndices[i] - flushTurnIndices[i - 1]);
|
|
32526
33709
|
}
|
|
32527
|
-
const
|
|
33710
|
+
const mean5 = turnsBetween.length > 0 ? turnsBetween.reduce((acc, v) => acc + v, 0) / turnsBetween.length : 0;
|
|
32528
33711
|
return {
|
|
32529
33712
|
flushTurnIndices,
|
|
32530
|
-
turnsBetweenFlushesMean:
|
|
33713
|
+
turnsBetweenFlushesMean: mean5,
|
|
32531
33714
|
replayLatencyMs
|
|
32532
33715
|
};
|
|
32533
33716
|
}
|
|
@@ -32638,7 +33821,7 @@ function loadCases10(mode, limit) {
|
|
|
32638
33821
|
}
|
|
32639
33822
|
|
|
32640
33823
|
// src/benchmarks/remnic/contradiction-detection/runner.ts
|
|
32641
|
-
import { randomUUID as
|
|
33824
|
+
import { randomUUID as randomUUID30 } from "crypto";
|
|
32642
33825
|
|
|
32643
33826
|
// src/benchmarks/remnic/contradiction-detection/fixture.ts
|
|
32644
33827
|
var TRUE_CONTRADICTIONS = [
|
|
@@ -32965,7 +34148,7 @@ async function runContradictionDetectionBenchmark(options) {
|
|
|
32965
34148
|
const meanQueryLatencyMs = tasks.length > 0 ? totalLatencyMs / tasks.length : 0;
|
|
32966
34149
|
return {
|
|
32967
34150
|
meta: {
|
|
32968
|
-
id:
|
|
34151
|
+
id: randomUUID30(),
|
|
32969
34152
|
benchmark: options.benchmark.id,
|
|
32970
34153
|
benchmarkTier: options.benchmark.tier,
|
|
32971
34154
|
version: options.benchmark.meta.version,
|
|
@@ -33020,7 +34203,7 @@ function loadCases11(mode, limit) {
|
|
|
33020
34203
|
}
|
|
33021
34204
|
|
|
33022
34205
|
// src/benchmarks/remnic/retention-aged-dataset/runner.ts
|
|
33023
|
-
import { randomUUID as
|
|
34206
|
+
import { randomUUID as randomUUID31 } from "crypto";
|
|
33024
34207
|
import {
|
|
33025
34208
|
decideTierTransition
|
|
33026
34209
|
} from "@remnic/core";
|
|
@@ -33396,7 +34579,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
33396
34579
|
const totalLatencyMs = tasks.reduce((sum, t) => sum + t.latencyMs, 0);
|
|
33397
34580
|
return {
|
|
33398
34581
|
meta: {
|
|
33399
|
-
id:
|
|
34582
|
+
id: randomUUID31(),
|
|
33400
34583
|
benchmark: options.benchmark.id,
|
|
33401
34584
|
benchmarkTier: options.benchmark.tier,
|
|
33402
34585
|
version: options.benchmark.meta.version,
|
|
@@ -33440,7 +34623,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
33440
34623
|
}
|
|
33441
34624
|
|
|
33442
34625
|
// src/benchmarks/remnic/memcorrect/runner.ts
|
|
33443
|
-
import { randomUUID as
|
|
34626
|
+
import { randomUUID as randomUUID32 } from "crypto";
|
|
33444
34627
|
|
|
33445
34628
|
// src/benchmarks/remnic/memcorrect/generator.ts
|
|
33446
34629
|
import { createHash as createHash11 } from "crypto";
|
|
@@ -34051,8 +35234,8 @@ function nonResurrection(log, corrections) {
|
|
|
34051
35234
|
}
|
|
34052
35235
|
function collateralDelta(before, after) {
|
|
34053
35236
|
if (before.length === 0) return 0;
|
|
34054
|
-
const
|
|
34055
|
-
return
|
|
35237
|
+
const mean5 = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
35238
|
+
return mean5(after) - mean5(before);
|
|
34056
35239
|
}
|
|
34057
35240
|
function scopePrecision(log, corrections) {
|
|
34058
35241
|
let scopedCount = 0;
|
|
@@ -34653,7 +35836,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
34653
35836
|
const { adapter: _liveAdapter, ...persistableBenchmarkOptions } = options.benchmarkOptions ?? {};
|
|
34654
35837
|
return {
|
|
34655
35838
|
meta: {
|
|
34656
|
-
id:
|
|
35839
|
+
id: randomUUID32(),
|
|
34657
35840
|
benchmark: options.benchmark.id,
|
|
34658
35841
|
benchmarkTier: options.benchmark.tier,
|
|
34659
35842
|
version: options.benchmark.meta.version,
|
|
@@ -34719,7 +35902,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
34719
35902
|
}
|
|
34720
35903
|
|
|
34721
35904
|
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
34722
|
-
import { randomUUID as
|
|
35905
|
+
import { randomUUID as randomUUID33 } from "crypto";
|
|
34723
35906
|
import { mkdir as mkdir12, writeFile as writeFile11 } from "fs/promises";
|
|
34724
35907
|
import path28 from "path";
|
|
34725
35908
|
|
|
@@ -35564,39 +36747,39 @@ function scoreTaskPair(task, pack, decision) {
|
|
|
35564
36747
|
compression_ratio_vs_raw_transcript: compressionRatio
|
|
35565
36748
|
};
|
|
35566
36749
|
}
|
|
35567
|
-
function
|
|
36750
|
+
function mean3(values) {
|
|
35568
36751
|
if (values.length === 0) return 0;
|
|
35569
36752
|
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
35570
36753
|
}
|
|
35571
36754
|
function aggregateCondition(condition, scored, skillLog) {
|
|
35572
36755
|
const taskCount = scored.length;
|
|
35573
36756
|
const all = scored.map((s) => s.scores);
|
|
35574
|
-
const taskSuccessRate =
|
|
36757
|
+
const taskSuccessRate = mean3(all.map((s) => s.task_success));
|
|
35575
36758
|
const boundaryTasks = scored.filter((s) => s.task.shouldAsk !== void 0);
|
|
35576
36759
|
const askNeeded = scored.filter((s) => s.task.shouldAsk === true);
|
|
35577
36760
|
const actWhenEnough = scored.filter((s) => s.task.shouldAsk === false);
|
|
35578
|
-
const shouldAskAccuracy =
|
|
35579
|
-
const unnecessaryClarificationRate =
|
|
36761
|
+
const shouldAskAccuracy = mean3(boundaryTasks.map((s) => s.scores.should_ask_accuracy));
|
|
36762
|
+
const unnecessaryClarificationRate = mean3(
|
|
35580
36763
|
actWhenEnough.map((s) => s.scores.unnecessary_clarification_rate)
|
|
35581
36764
|
);
|
|
35582
|
-
const actionBoundaryViolationRate =
|
|
36765
|
+
const actionBoundaryViolationRate = mean3(
|
|
35583
36766
|
askNeeded.map((s) => s.scores.action_boundary_violation_rate)
|
|
35584
36767
|
);
|
|
35585
36768
|
const recallTasks = scored.filter(
|
|
35586
36769
|
(s) => (s.task.shouldRecallId !== void 0 || s.task.family === "skill-positive") && s.task.family !== "stale-memory-trap" && s.task.family !== "wrong-scope-trap"
|
|
35587
36770
|
);
|
|
35588
|
-
const relevantMemoryRecall =
|
|
36771
|
+
const relevantMemoryRecall = mean3(recallTasks.map((s) => s.scores.relevant_memory_recall));
|
|
35589
36772
|
const staleTasks = scored.filter((s) => s.task.family === "stale-memory-trap");
|
|
35590
36773
|
const scopeTasks = scored.filter((s) => s.task.family === "wrong-scope-trap");
|
|
35591
|
-
const staleMemoryHarmRate =
|
|
35592
|
-
const wrongScopeRetrievalRate =
|
|
35593
|
-
const supersessionRespectedRate =
|
|
36774
|
+
const staleMemoryHarmRate = mean3(staleTasks.map((s) => s.scores.stale_memory_harm_rate));
|
|
36775
|
+
const wrongScopeRetrievalRate = mean3(scopeTasks.map((s) => s.scores.wrong_scope_retrieval_rate));
|
|
36776
|
+
const supersessionRespectedRate = mean3(staleTasks.map((s) => s.scores.supersession_respected_rate));
|
|
35594
36777
|
const citedTasks = scored.filter((s) => s.scores.retrieved_item_count > 0);
|
|
35595
|
-
const citationCoverage =
|
|
35596
|
-
const meanMemoryTokensInjected =
|
|
35597
|
-
const meanRetrievedItemCount =
|
|
36778
|
+
const citationCoverage = mean3(citedTasks.map((s) => s.scores.citation_coverage));
|
|
36779
|
+
const meanMemoryTokensInjected = mean3(all.map((s) => s.memory_tokens_injected));
|
|
36780
|
+
const meanRetrievedItemCount = mean3(all.map((s) => s.retrieved_item_count));
|
|
35598
36781
|
const compressible = scored.filter((s) => s.task.memoryItems.length > 0);
|
|
35599
|
-
const meanCompressionRatio =
|
|
36782
|
+
const meanCompressionRatio = mean3(compressible.map((s) => s.scores.compression_ratio_vs_raw_transcript));
|
|
35600
36783
|
const considered = skillLog.filter((e) => e.considered);
|
|
35601
36784
|
const injected = considered.filter((e) => e.injected);
|
|
35602
36785
|
const tp = injected.filter((e) => e.outcome === "helped").length;
|
|
@@ -35901,7 +37084,7 @@ async function runBoundedMemoryContractsBenchmark(options) {
|
|
|
35901
37084
|
const skillTriggerLog = c3SkillLog;
|
|
35902
37085
|
return {
|
|
35903
37086
|
meta: {
|
|
35904
|
-
id:
|
|
37087
|
+
id: randomUUID33(),
|
|
35905
37088
|
benchmark: options.benchmark.id,
|
|
35906
37089
|
benchmarkTier: options.benchmark.tier,
|
|
35907
37090
|
version: options.benchmark.meta.version,
|
|
@@ -36077,7 +37260,7 @@ function renderPromptPack(task, condition, pack) {
|
|
|
36077
37260
|
}
|
|
36078
37261
|
|
|
36079
37262
|
// src/benchmarks/remnic/staged-memory/runner.ts
|
|
36080
|
-
import { createHash as createHash16, randomUUID as
|
|
37263
|
+
import { createHash as createHash16, randomUUID as randomUUID34 } from "crypto";
|
|
36081
37264
|
|
|
36082
37265
|
// src/benchmarks/remnic/staged-memory/fixture.ts
|
|
36083
37266
|
import { createHash as createHash15 } from "crypto";
|
|
@@ -37688,7 +38871,7 @@ async function runDriftGenCliCommand(options) {
|
|
|
37688
38871
|
}
|
|
37689
38872
|
|
|
37690
38873
|
// src/benchmarks/remnic/staged-memory/schema.ts
|
|
37691
|
-
import { z } from "zod";
|
|
38874
|
+
import { z as z2 } from "zod";
|
|
37692
38875
|
var STAGED_MEMORY_BENCHMARK_ID = "staged-memory-synthetic-v1";
|
|
37693
38876
|
var STAGED_MEMORY_FIXTURE_NAME = "staged-memory-synthetic";
|
|
37694
38877
|
var STAGED_MEMORY_GENERATOR_VERSION = "1.0.0";
|
|
@@ -37702,38 +38885,38 @@ var STAGED_MEMORY_ARMS = Object.freeze([
|
|
|
37702
38885
|
"staged-memory",
|
|
37703
38886
|
"oracle-retrieval"
|
|
37704
38887
|
]);
|
|
37705
|
-
var hex64 =
|
|
37706
|
-
var nonEmpty =
|
|
37707
|
-
var positiveEpoch =
|
|
37708
|
-
var StagedMemoryFixtureManifestV1Schema =
|
|
37709
|
-
schemaVersion:
|
|
37710
|
-
name:
|
|
38888
|
+
var hex64 = z2.string().regex(/^[0-9a-f]{64}$/, "expected a sha256 hex digest");
|
|
38889
|
+
var nonEmpty = z2.string().min(1);
|
|
38890
|
+
var positiveEpoch = z2.number().int().positive();
|
|
38891
|
+
var StagedMemoryFixtureManifestV1Schema = z2.object({
|
|
38892
|
+
schemaVersion: z2.literal(1),
|
|
38893
|
+
name: z2.literal("staged-memory-synthetic"),
|
|
37711
38894
|
version: nonEmpty,
|
|
37712
38895
|
generatorVersion: nonEmpty,
|
|
37713
|
-
seeds:
|
|
37714
|
-
source:
|
|
37715
|
-
kind:
|
|
38896
|
+
seeds: z2.array(z2.number().int().nonnegative()).min(1),
|
|
38897
|
+
source: z2.object({
|
|
38898
|
+
kind: z2.literal("drift-gen"),
|
|
37716
38899
|
manifestName: nonEmpty,
|
|
37717
38900
|
manifestSha256: hex64
|
|
37718
38901
|
}).strict(),
|
|
37719
|
-
counts:
|
|
37720
|
-
users:
|
|
37721
|
-
cases:
|
|
37722
|
-
distractors:
|
|
38902
|
+
counts: z2.object({
|
|
38903
|
+
users: z2.number().int().nonnegative(),
|
|
38904
|
+
cases: z2.number().int().nonnegative(),
|
|
38905
|
+
distractors: z2.number().int().nonnegative()
|
|
37723
38906
|
}).strict(),
|
|
37724
|
-
files:
|
|
37725
|
-
createdAt:
|
|
37726
|
-
licenses:
|
|
37727
|
-
namespaces:
|
|
38907
|
+
files: z2.record(hex64),
|
|
38908
|
+
createdAt: z2.literal("1970-01-01T00:00:00.000Z"),
|
|
38909
|
+
licenses: z2.array(z2.object({ source: nonEmpty, license: nonEmpty }).strict()).min(1),
|
|
38910
|
+
namespaces: z2.array(nonEmpty).min(2)
|
|
37728
38911
|
}).strict();
|
|
37729
|
-
var StagedMemoryDistractorV1Schema =
|
|
38912
|
+
var StagedMemoryDistractorV1Schema = z2.object({
|
|
37730
38913
|
id: nonEmpty,
|
|
37731
38914
|
sessionId: nonEmpty,
|
|
37732
38915
|
text: nonEmpty,
|
|
37733
|
-
forbiddenFactIds:
|
|
38916
|
+
forbiddenFactIds: z2.array(nonEmpty).min(1),
|
|
37734
38917
|
templateId: nonEmpty
|
|
37735
38918
|
}).strict();
|
|
37736
|
-
var StagedMemoryGoldFactV1Schema =
|
|
38919
|
+
var StagedMemoryGoldFactV1Schema = z2.object({
|
|
37737
38920
|
factId: nonEmpty,
|
|
37738
38921
|
subject: nonEmpty,
|
|
37739
38922
|
attribute: nonEmpty,
|
|
@@ -37741,41 +38924,41 @@ var StagedMemoryGoldFactV1Schema = z.object({
|
|
|
37741
38924
|
statement: nonEmpty,
|
|
37742
38925
|
introducedEpoch: positiveEpoch
|
|
37743
38926
|
}).strict();
|
|
37744
|
-
var StagedMemoryCaseV1Schema =
|
|
37745
|
-
schemaVersion:
|
|
38927
|
+
var StagedMemoryCaseV1Schema = z2.object({
|
|
38928
|
+
schemaVersion: z2.literal(1),
|
|
37746
38929
|
caseId: nonEmpty,
|
|
37747
38930
|
userId: nonEmpty,
|
|
37748
38931
|
namespace: nonEmpty,
|
|
37749
|
-
seed:
|
|
37750
|
-
exposure:
|
|
38932
|
+
seed: z2.number().int().nonnegative(),
|
|
38933
|
+
exposure: z2.object({
|
|
37751
38934
|
sessionId: nonEmpty,
|
|
37752
|
-
sourceSessionRefs:
|
|
38935
|
+
sourceSessionRefs: z2.array(nonEmpty).min(1),
|
|
37753
38936
|
/** Current (non-superseded) fact IDs at the exposure epoch. */
|
|
37754
|
-
salientFactIds:
|
|
37755
|
-
goldFacts:
|
|
38937
|
+
salientFactIds: z2.array(nonEmpty).min(1),
|
|
38938
|
+
goldFacts: z2.array(StagedMemoryGoldFactV1Schema).min(1),
|
|
37756
38939
|
/** Statements of `goldFacts`, in the same order. */
|
|
37757
|
-
goldMemories:
|
|
38940
|
+
goldMemories: z2.array(nonEmpty).min(1),
|
|
37758
38941
|
exposureEpoch: positiveEpoch,
|
|
37759
38942
|
/** Pinned effective timestamp for transition scoring; never wall clock. */
|
|
37760
38943
|
effectiveTimestamp: nonEmpty
|
|
37761
38944
|
}).strict(),
|
|
37762
|
-
transitions:
|
|
37763
|
-
|
|
38945
|
+
transitions: z2.array(
|
|
38946
|
+
z2.object({
|
|
37764
38947
|
oldFactId: nonEmpty,
|
|
37765
38948
|
newFactId: nonEmpty,
|
|
37766
38949
|
epoch: positiveEpoch,
|
|
37767
|
-
kind:
|
|
38950
|
+
kind: z2.enum(["drifting", "contradicted"])
|
|
37768
38951
|
}).strict()
|
|
37769
38952
|
),
|
|
37770
|
-
distractors:
|
|
37771
|
-
task:
|
|
38953
|
+
distractors: z2.array(StagedMemoryDistractorV1Schema),
|
|
38954
|
+
task: z2.object({
|
|
37772
38955
|
question: nonEmpty,
|
|
37773
38956
|
expectedAnswer: nonEmpty,
|
|
37774
|
-
requiredFactIds:
|
|
37775
|
-
forbiddenFactIds:
|
|
37776
|
-
answerFormat:
|
|
38957
|
+
requiredFactIds: z2.array(nonEmpty).min(1),
|
|
38958
|
+
forbiddenFactIds: z2.array(nonEmpty),
|
|
38959
|
+
answerFormat: z2.literal("exact")
|
|
37777
38960
|
}).strict(),
|
|
37778
|
-
scope:
|
|
38961
|
+
scope: z2.object({
|
|
37779
38962
|
principal: nonEmpty,
|
|
37780
38963
|
allowedUserId: nonEmpty,
|
|
37781
38964
|
allowedNamespace: nonEmpty
|
|
@@ -38999,7 +40182,7 @@ async function runStagedMemoryBenchmark(options) {
|
|
|
38999
40182
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
39000
40183
|
const result = {
|
|
39001
40184
|
meta: {
|
|
39002
|
-
id:
|
|
40185
|
+
id: randomUUID34(),
|
|
39003
40186
|
benchmark: options.benchmark.id,
|
|
39004
40187
|
benchmarkTier: options.benchmark.tier,
|
|
39005
40188
|
version: options.benchmark.meta.version,
|
|
@@ -39104,6 +40287,10 @@ var REGISTERED_BENCHMARKS = [
|
|
|
39104
40287
|
...extractionJudgeCalibrationDefinition,
|
|
39105
40288
|
run: runExtractionJudgeCalibrationBenchmark
|
|
39106
40289
|
},
|
|
40290
|
+
{
|
|
40291
|
+
...extractionSpanModeDefinition,
|
|
40292
|
+
run: runExtractionSpanModeBenchmark
|
|
40293
|
+
},
|
|
39107
40294
|
{
|
|
39108
40295
|
...enrichmentFidelityDefinition,
|
|
39109
40296
|
run: runEnrichmentFidelityBenchmark
|
|
@@ -39758,7 +40945,7 @@ function generateReport(results, reportPath) {
|
|
|
39758
40945
|
}
|
|
39759
40946
|
|
|
39760
40947
|
// src/stats/effect-size.ts
|
|
39761
|
-
function
|
|
40948
|
+
function mean4(values) {
|
|
39762
40949
|
if (values.length === 0) {
|
|
39763
40950
|
throw new Error("effect size requires at least one value");
|
|
39764
40951
|
}
|
|
@@ -39774,8 +40961,8 @@ function cohensD(candidateValues, baselineValues) {
|
|
|
39774
40961
|
if (candidateValues.length === 0 || baselineValues.length === 0) {
|
|
39775
40962
|
throw new Error("effect size requires non-empty candidate and baseline arrays");
|
|
39776
40963
|
}
|
|
39777
|
-
const candidateMean =
|
|
39778
|
-
const baselineMean =
|
|
40964
|
+
const candidateMean = mean4(candidateValues);
|
|
40965
|
+
const baselineMean = mean4(baselineValues);
|
|
39779
40966
|
const candidateVariance = sampleVariance(candidateValues, candidateMean);
|
|
39780
40967
|
const baselineVariance = sampleVariance(baselineValues, baselineMean);
|
|
39781
40968
|
const pooledDegreesOfFreedom = candidateValues.length + baselineValues.length - 2;
|
|
@@ -42270,7 +43457,7 @@ function formatError(error) {
|
|
|
42270
43457
|
}
|
|
42271
43458
|
|
|
42272
43459
|
// src/benchmarks/custom/runner.ts
|
|
42273
|
-
import { randomUUID as
|
|
43460
|
+
import { randomUUID as randomUUID35 } from "crypto";
|
|
42274
43461
|
import path34 from "path";
|
|
42275
43462
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
42276
43463
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
@@ -42358,7 +43545,7 @@ async function runCustomBenchmark(spec, options) {
|
|
|
42358
43545
|
const totalOutputTokens = tasks.reduce((sum, task) => sum + task.tokens.output, 0);
|
|
42359
43546
|
return finalizeBenchmarkResultConfig({
|
|
42360
43547
|
meta: {
|
|
42361
|
-
id:
|
|
43548
|
+
id: randomUUID35(),
|
|
42362
43549
|
benchmark: options.benchmark.id,
|
|
42363
43550
|
benchmarkTier: options.benchmark.tier,
|
|
42364
43551
|
version: options.benchmark.meta.version,
|
|
@@ -45240,10 +46427,10 @@ import path39 from "path";
|
|
|
45240
46427
|
import { hostname } from "os";
|
|
45241
46428
|
import { mkdir as mkdir18, readFile as readFile24, rename as rename7, rm as rm17, stat as stat4, utimes, writeFile as writeFile17 } from "fs/promises";
|
|
45242
46429
|
import path38 from "path";
|
|
45243
|
-
import { randomUUID as
|
|
46430
|
+
import { randomUUID as randomUUID37 } from "crypto";
|
|
45244
46431
|
|
|
45245
46432
|
// src/security/injection-suite/store.ts
|
|
45246
|
-
import { createHash as createHash20, randomUUID as
|
|
46433
|
+
import { createHash as createHash20, randomUUID as randomUUID36 } from "crypto";
|
|
45247
46434
|
import { mkdir as mkdir17, readFile as readFile23, rename as rename6, writeFile as writeFile16 } from "fs/promises";
|
|
45248
46435
|
import path37 from "path";
|
|
45249
46436
|
|
|
@@ -45327,7 +46514,7 @@ var InjectionSuiteRowStore = class {
|
|
|
45327
46514
|
};
|
|
45328
46515
|
await mkdir17(this.checkpointsDir, { recursive: true });
|
|
45329
46516
|
const destination = this.checkpointPath(identity);
|
|
45330
|
-
const tempPath = `${destination}.tmp-${
|
|
46517
|
+
const tempPath = `${destination}.tmp-${randomUUID36()}`;
|
|
45331
46518
|
await writeFile16(tempPath, `${JSON.stringify(checkpoint, null, 2)}
|
|
45332
46519
|
`, "utf8");
|
|
45333
46520
|
await rename6(tempPath, destination);
|
|
@@ -45358,7 +46545,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45358
46545
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
45359
46546
|
const lockPath = this.lockPath(rowKey);
|
|
45360
46547
|
await mkdir18(this.checkpointsDir, { recursive: true });
|
|
45361
|
-
const ownerToken =
|
|
46548
|
+
const ownerToken = randomUUID37();
|
|
45362
46549
|
try {
|
|
45363
46550
|
await mkdir18(lockPath);
|
|
45364
46551
|
} catch (error) {
|
|
@@ -45442,7 +46629,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45442
46629
|
}
|
|
45443
46630
|
}
|
|
45444
46631
|
if (Date.now() - stampMs < leaseMs) return false;
|
|
45445
|
-
const stalePath = `${lockPath}.stale-${
|
|
46632
|
+
const stalePath = `${lockPath}.stale-${randomUUID37()}`;
|
|
45446
46633
|
try {
|
|
45447
46634
|
await rename7(lockPath, stalePath);
|
|
45448
46635
|
} catch {
|
|
@@ -46437,255 +47624,255 @@ import { constants } from "fs";
|
|
|
46437
47624
|
import { lstat as lstat7, mkdir as mkdir20, open as open3 } from "fs/promises";
|
|
46438
47625
|
import path41 from "path";
|
|
46439
47626
|
import { writeFileAtomically } from "@remnic/core/maintenance/atomic-file";
|
|
46440
|
-
import { z as
|
|
47627
|
+
import { z as z4 } from "zod";
|
|
46441
47628
|
|
|
46442
47629
|
// src/coding-graph/repeated-failure-report-rendering.ts
|
|
46443
47630
|
import { createHash as createHash24 } from "crypto";
|
|
46444
|
-
import { z as
|
|
47631
|
+
import { z as z3 } from "zod";
|
|
46445
47632
|
var SHA2562 = /^[a-f0-9]{64}$/;
|
|
46446
|
-
var IntervalSchema =
|
|
46447
|
-
lower:
|
|
46448
|
-
upper:
|
|
46449
|
-
level:
|
|
47633
|
+
var IntervalSchema = z3.object({
|
|
47634
|
+
lower: z3.number().nullable(),
|
|
47635
|
+
upper: z3.number().nullable(),
|
|
47636
|
+
level: z3.number().min(0).max(1)
|
|
46450
47637
|
});
|
|
46451
|
-
var EffectSchema =
|
|
46452
|
-
taskCount:
|
|
46453
|
-
baselineArm:
|
|
46454
|
-
candidateArm:
|
|
46455
|
-
interpretation:
|
|
46456
|
-
repeatedFailureBenefit:
|
|
47638
|
+
var EffectSchema = z3.object({
|
|
47639
|
+
taskCount: z3.number().int().nonnegative(),
|
|
47640
|
+
baselineArm: z3.enum(REPEATED_FAILURE_ARMS),
|
|
47641
|
+
candidateArm: z3.enum(REPEATED_FAILURE_ARMS),
|
|
47642
|
+
interpretation: z3.enum(["CONFIRMATORY", "EXPLORATORY_COMPLETE_TASKS"]),
|
|
47643
|
+
repeatedFailureBenefit: z3.number().nullable(),
|
|
46457
47644
|
repeatedFailureBenefitInterval: IntervalSchema.nullable(),
|
|
46458
|
-
relativeRiskReduction:
|
|
47645
|
+
relativeRiskReduction: z3.number().nullable(),
|
|
46459
47646
|
relativeRiskReductionInterval: IntervalSchema.nullable(),
|
|
46460
|
-
nonEstimableRrrDraws:
|
|
46461
|
-
repeatedFailureP:
|
|
46462
|
-
taskPassBenefit:
|
|
47647
|
+
nonEstimableRrrDraws: z3.number().int().nonnegative(),
|
|
47648
|
+
repeatedFailureP: z3.number().nullable(),
|
|
47649
|
+
taskPassBenefit: z3.number().nullable(),
|
|
46463
47650
|
taskPassBenefitInterval: IntervalSchema.nullable(),
|
|
46464
|
-
taskPassP:
|
|
47651
|
+
taskPassP: z3.number().nullable()
|
|
46465
47652
|
});
|
|
46466
|
-
var StatisticsSchema =
|
|
46467
|
-
schemaVersion:
|
|
46468
|
-
seed:
|
|
46469
|
-
draws:
|
|
46470
|
-
level:
|
|
46471
|
-
alpha:
|
|
46472
|
-
cuts:
|
|
46473
|
-
hypothesis:
|
|
46474
|
-
taskId:
|
|
46475
|
-
reasons:
|
|
47653
|
+
var StatisticsSchema = z3.object({
|
|
47654
|
+
schemaVersion: z3.literal(1),
|
|
47655
|
+
seed: z3.number().int().nonnegative(),
|
|
47656
|
+
draws: z3.number().int().positive(),
|
|
47657
|
+
level: z3.number().min(0).max(1),
|
|
47658
|
+
alpha: z3.number().min(0).max(1),
|
|
47659
|
+
cuts: z3.array(z3.object({
|
|
47660
|
+
hypothesis: z3.enum(["TIMING", "CONTENT", "TIMIDITY"]),
|
|
47661
|
+
taskId: z3.string().min(1),
|
|
47662
|
+
reasons: z3.array(z3.string().min(1))
|
|
46476
47663
|
})),
|
|
46477
47664
|
timing: EffectSchema,
|
|
46478
47665
|
content: EffectSchema,
|
|
46479
|
-
contentCompoundP:
|
|
46480
|
-
holm:
|
|
46481
|
-
id:
|
|
46482
|
-
rawP:
|
|
46483
|
-
adjustedP:
|
|
46484
|
-
rank:
|
|
47666
|
+
contentCompoundP: z3.number().nullable(),
|
|
47667
|
+
holm: z3.array(z3.object({
|
|
47668
|
+
id: z3.enum(["TIMING", "CONTENT"]),
|
|
47669
|
+
rawP: z3.number(),
|
|
47670
|
+
adjustedP: z3.number(),
|
|
47671
|
+
rank: z3.number().int().positive()
|
|
46485
47672
|
})),
|
|
46486
|
-
decisions:
|
|
46487
|
-
timing:
|
|
46488
|
-
content:
|
|
47673
|
+
decisions: z3.object({
|
|
47674
|
+
timing: z3.enum(["SUPPORTED", "REJECTED", "NOT_ESTIMABLE"]),
|
|
47675
|
+
content: z3.enum(["SUPPORTED", "REJECTED", "NOT_ESTIMABLE"])
|
|
46489
47676
|
}),
|
|
46490
|
-
studyDecision:
|
|
46491
|
-
timidity:
|
|
46492
|
-
taskCount:
|
|
46493
|
-
intervalLevel:
|
|
46494
|
-
passRateDifference:
|
|
47677
|
+
studyDecision: z3.enum(["PASS", "PARTIAL", "REJECT", "NOT_ESTIMABLE"]),
|
|
47678
|
+
timidity: z3.object({
|
|
47679
|
+
taskCount: z3.number().int().nonnegative(),
|
|
47680
|
+
intervalLevel: z3.number().min(0).max(1),
|
|
47681
|
+
passRateDifference: z3.number().nullable(),
|
|
46495
47682
|
passRateInterval: IntervalSchema.nullable(),
|
|
46496
|
-
stepsDifference:
|
|
47683
|
+
stepsDifference: z3.number().nullable(),
|
|
46497
47684
|
stepsInterval: IntervalSchema.nullable(),
|
|
46498
|
-
passMargin:
|
|
46499
|
-
stepsMargin:
|
|
46500
|
-
equivalent:
|
|
47685
|
+
passMargin: z3.number().nonnegative(),
|
|
47686
|
+
stepsMargin: z3.number().nonnegative(),
|
|
47687
|
+
equivalent: z3.boolean().nullable()
|
|
46501
47688
|
})
|
|
46502
47689
|
});
|
|
46503
|
-
var AuditSchema =
|
|
46504
|
-
schemaVersion:
|
|
46505
|
-
runContract:
|
|
46506
|
-
datasetInventoryHash:
|
|
46507
|
-
preregistrationPath:
|
|
46508
|
-
decisionRuleHash:
|
|
46509
|
-
preregistrationHash:
|
|
46510
|
-
analysisVersion:
|
|
46511
|
-
harnessVersion:
|
|
46512
|
-
harnessSourceHash:
|
|
46513
|
-
provenanceHash:
|
|
46514
|
-
modelProfiles:
|
|
46515
|
-
id:
|
|
46516
|
-
hash:
|
|
46517
|
-
modelDigest:
|
|
46518
|
-
tokenizerIdentity:
|
|
46519
|
-
tokenizerImplementation:
|
|
47690
|
+
var AuditSchema = z3.object({
|
|
47691
|
+
schemaVersion: z3.literal(1),
|
|
47692
|
+
runContract: z3.object({
|
|
47693
|
+
datasetInventoryHash: z3.string().regex(SHA2562),
|
|
47694
|
+
preregistrationPath: z3.string().min(1),
|
|
47695
|
+
decisionRuleHash: z3.string().regex(SHA2562),
|
|
47696
|
+
preregistrationHash: z3.string().regex(SHA2562),
|
|
47697
|
+
analysisVersion: z3.string().min(1),
|
|
47698
|
+
harnessVersion: z3.string().min(1),
|
|
47699
|
+
harnessSourceHash: z3.string().regex(SHA2562),
|
|
47700
|
+
provenanceHash: z3.string().regex(SHA2562),
|
|
47701
|
+
modelProfiles: z3.array(z3.object({
|
|
47702
|
+
id: z3.string().min(1),
|
|
47703
|
+
hash: z3.string().regex(SHA2562),
|
|
47704
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47705
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47706
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1")
|
|
46520
47707
|
}).strict()).min(1),
|
|
46521
|
-
trapAudit:
|
|
46522
|
-
minimumTrappedRate:
|
|
46523
|
-
minimumNonFixedRate:
|
|
46524
|
-
maximumInvalidRows:
|
|
46525
|
-
requireCompleteRows:
|
|
47708
|
+
trapAudit: z3.object({
|
|
47709
|
+
minimumTrappedRate: z3.number().min(0).max(1),
|
|
47710
|
+
minimumNonFixedRate: z3.number().min(0).max(1),
|
|
47711
|
+
maximumInvalidRows: z3.literal(0),
|
|
47712
|
+
requireCompleteRows: z3.literal(true)
|
|
46526
47713
|
}).strict()
|
|
46527
47714
|
}).strict(),
|
|
46528
|
-
dataset:
|
|
46529
|
-
inventoryHash:
|
|
46530
|
-
supportArtifactsMatch:
|
|
46531
|
-
taskCount:
|
|
46532
|
-
variantCount:
|
|
46533
|
-
splitCounts:
|
|
47715
|
+
dataset: z3.object({
|
|
47716
|
+
inventoryHash: z3.string().regex(SHA2562),
|
|
47717
|
+
supportArtifactsMatch: z3.boolean(),
|
|
47718
|
+
taskCount: z3.number().int().nonnegative(),
|
|
47719
|
+
variantCount: z3.number().int().nonnegative(),
|
|
47720
|
+
splitCounts: z3.record(z3.string(), z3.number().int().nonnegative())
|
|
46534
47721
|
}).passthrough(),
|
|
46535
|
-
expectedDesign:
|
|
46536
|
-
expectedRows:
|
|
46537
|
-
terminalRows:
|
|
46538
|
-
exactRowSet:
|
|
47722
|
+
expectedDesign: z3.object({
|
|
47723
|
+
expectedRows: z3.number().int().nonnegative(),
|
|
47724
|
+
terminalRows: z3.number().int().nonnegative(),
|
|
47725
|
+
exactRowSet: z3.boolean()
|
|
46539
47726
|
}),
|
|
46540
|
-
factPairs:
|
|
46541
|
-
isolation:
|
|
46542
|
-
allUnique:
|
|
46543
|
-
primaryStartHashesMatchWithinCells:
|
|
47727
|
+
factPairs: z3.object({ pairCount: z3.number().int().nonnegative(), allMatched: z3.boolean() }),
|
|
47728
|
+
isolation: z3.object({
|
|
47729
|
+
allUnique: z3.boolean(),
|
|
47730
|
+
primaryStartHashesMatchWithinCells: z3.boolean()
|
|
46544
47731
|
}).passthrough(),
|
|
46545
|
-
timingEvidence:
|
|
46546
|
-
fakeAgentContract:
|
|
46547
|
-
status:
|
|
46548
|
-
deterministicDriverCount:
|
|
47732
|
+
timingEvidence: z3.object({ allMatched: z3.boolean() }).passthrough(),
|
|
47733
|
+
fakeAgentContract: z3.object({
|
|
47734
|
+
status: z3.enum(["PASS", "FAIL", "NOT_APPLICABLE"]),
|
|
47735
|
+
deterministicDriverCount: z3.number().int().nonnegative()
|
|
46549
47736
|
}),
|
|
46550
|
-
modelProfiles:
|
|
46551
|
-
id:
|
|
46552
|
-
hash:
|
|
46553
|
-
modelDigest:
|
|
46554
|
-
tokenizerIdentity:
|
|
46555
|
-
tokenizerImplementation:
|
|
46556
|
-
driverKind:
|
|
47737
|
+
modelProfiles: z3.array(z3.object({
|
|
47738
|
+
id: z3.string().min(1),
|
|
47739
|
+
hash: z3.string().regex(SHA2562),
|
|
47740
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47741
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47742
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1"),
|
|
47743
|
+
driverKind: z3.enum(["responses", "ollama-chat", "deterministic-fake", "unknown"])
|
|
46557
47744
|
}).strict()).min(1),
|
|
46558
|
-
noTrap:
|
|
46559
|
-
expectedRows:
|
|
46560
|
-
observedRows:
|
|
46561
|
-
allPassed:
|
|
47745
|
+
noTrap: z3.object({
|
|
47746
|
+
expectedRows: z3.number().int().nonnegative(),
|
|
47747
|
+
observedRows: z3.number().int().nonnegative(),
|
|
47748
|
+
allPassed: z3.boolean()
|
|
46562
47749
|
}).strict(),
|
|
46563
|
-
deviations:
|
|
46564
|
-
count:
|
|
46565
|
-
none:
|
|
47750
|
+
deviations: z3.object({
|
|
47751
|
+
count: z3.number().int().nonnegative(),
|
|
47752
|
+
none: z3.boolean()
|
|
46566
47753
|
}).strict(),
|
|
46567
|
-
traces:
|
|
46568
|
-
expectedCount:
|
|
46569
|
-
durableCount:
|
|
46570
|
-
allDurable:
|
|
47754
|
+
traces: z3.object({
|
|
47755
|
+
expectedCount: z3.number().int().nonnegative(),
|
|
47756
|
+
durableCount: z3.number().int().nonnegative(),
|
|
47757
|
+
allDurable: z3.boolean()
|
|
46571
47758
|
}),
|
|
46572
|
-
cuts:
|
|
46573
|
-
decision:
|
|
47759
|
+
cuts: z3.object({ primary: z3.array(z3.unknown()), timidity: z3.array(z3.unknown()) }),
|
|
47760
|
+
decision: z3.enum(["PASS", "PARTIAL", "REJECT", "NOT_ESTIMABLE"])
|
|
46574
47761
|
}).passthrough();
|
|
46575
|
-
var PilotProfileBindingSchema =
|
|
46576
|
-
id:
|
|
46577
|
-
hash:
|
|
46578
|
-
modelDigest:
|
|
46579
|
-
driverKind:
|
|
46580
|
-
tokenizerIdentity:
|
|
46581
|
-
tokenizerImplementation:
|
|
47762
|
+
var PilotProfileBindingSchema = z3.object({
|
|
47763
|
+
id: z3.string().min(1),
|
|
47764
|
+
hash: z3.string().regex(SHA2562),
|
|
47765
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47766
|
+
driverKind: z3.enum(["responses", "ollama-chat", "deterministic-fake", "unknown"]),
|
|
47767
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47768
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1")
|
|
46582
47769
|
}).strict();
|
|
46583
|
-
var PilotTrapReceiptSchema =
|
|
46584
|
-
path:
|
|
46585
|
-
artifactHash:
|
|
46586
|
-
modelProfileId:
|
|
46587
|
-
modelProfileHash:
|
|
46588
|
-
modelDigest:
|
|
46589
|
-
tokenizerIdentity:
|
|
46590
|
-
tokenizerImplementation:
|
|
47770
|
+
var PilotTrapReceiptSchema = z3.object({
|
|
47771
|
+
path: z3.string().min(1),
|
|
47772
|
+
artifactHash: z3.string().regex(SHA2562),
|
|
47773
|
+
modelProfileId: z3.string().min(1),
|
|
47774
|
+
modelProfileHash: z3.string().regex(SHA2562),
|
|
47775
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47776
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47777
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1")
|
|
46591
47778
|
}).strict();
|
|
46592
|
-
var ComputedPilotPowerSchema =
|
|
46593
|
-
schemaVersion:
|
|
46594
|
-
status:
|
|
46595
|
-
phase:
|
|
46596
|
-
method:
|
|
46597
|
-
draws:
|
|
46598
|
-
analysisDraws:
|
|
46599
|
-
source:
|
|
46600
|
-
episodesHash:
|
|
46601
|
-
expectedDesignHash:
|
|
46602
|
-
decisionRuleHash:
|
|
47779
|
+
var ComputedPilotPowerSchema = z3.object({
|
|
47780
|
+
schemaVersion: z3.literal(1),
|
|
47781
|
+
status: z3.literal("COMPUTED"),
|
|
47782
|
+
phase: z3.literal("pilot"),
|
|
47783
|
+
method: z3.object({ analysisVersion: z3.string().min(1) }).passthrough(),
|
|
47784
|
+
draws: z3.number().int().positive(),
|
|
47785
|
+
analysisDraws: z3.number().int().positive(),
|
|
47786
|
+
source: z3.object({
|
|
47787
|
+
episodesHash: z3.string().regex(SHA2562),
|
|
47788
|
+
expectedDesignHash: z3.string().regex(SHA2562),
|
|
47789
|
+
decisionRuleHash: z3.string().regex(SHA2562)
|
|
46603
47790
|
}).strict(),
|
|
46604
|
-
simulations:
|
|
46605
|
-
timing:
|
|
46606
|
-
content:
|
|
46607
|
-
timidity:
|
|
47791
|
+
simulations: z3.object({
|
|
47792
|
+
timing: z3.object({ power: z3.number().min(0.8).max(1) }).passthrough(),
|
|
47793
|
+
content: z3.object({ power: z3.number().min(0.8).max(1) }).passthrough(),
|
|
47794
|
+
timidity: z3.object({ power: z3.number().min(0.8).max(1) }).passthrough()
|
|
46608
47795
|
}).strict()
|
|
46609
47796
|
}).passthrough();
|
|
46610
|
-
var PilotRowIdentitySchema =
|
|
46611
|
-
suiteVersion:
|
|
46612
|
-
taskId:
|
|
46613
|
-
variantId:
|
|
46614
|
-
modelProfileId:
|
|
46615
|
-
modelProfileHash:
|
|
46616
|
-
seed:
|
|
46617
|
-
arm:
|
|
47797
|
+
var PilotRowIdentitySchema = z3.object({
|
|
47798
|
+
suiteVersion: z3.string().min(1),
|
|
47799
|
+
taskId: z3.string().min(1),
|
|
47800
|
+
variantId: z3.string().min(1),
|
|
47801
|
+
modelProfileId: z3.string().min(1),
|
|
47802
|
+
modelProfileHash: z3.string().regex(SHA2562),
|
|
47803
|
+
seed: z3.number().int().nonnegative().max(4294967295),
|
|
47804
|
+
arm: z3.enum(REPEATED_FAILURE_ARMS)
|
|
46618
47805
|
}).strict();
|
|
46619
|
-
var MainPowerEvidenceSchema =
|
|
46620
|
-
schemaVersion:
|
|
46621
|
-
status:
|
|
46622
|
-
phase:
|
|
46623
|
-
pilotRunId:
|
|
46624
|
-
pilotManifestArtifactHash:
|
|
46625
|
-
pilotPowerArtifactHash:
|
|
47806
|
+
var MainPowerEvidenceSchema = z3.object({
|
|
47807
|
+
schemaVersion: z3.literal(1),
|
|
47808
|
+
status: z3.literal("VERIFIED_PILOT"),
|
|
47809
|
+
phase: z3.literal("main"),
|
|
47810
|
+
pilotRunId: z3.string().min(1),
|
|
47811
|
+
pilotManifestArtifactHash: z3.string().regex(SHA2562),
|
|
47812
|
+
pilotPowerArtifactHash: z3.string().regex(SHA2562),
|
|
46626
47813
|
pilot: ComputedPilotPowerSchema,
|
|
46627
|
-
pilotProfileBindings:
|
|
46628
|
-
pilotTrapAuditReceipts:
|
|
46629
|
-
pilotRunOrder:
|
|
46630
|
-
rowKey:
|
|
46631
|
-
analysis:
|
|
47814
|
+
pilotProfileBindings: z3.array(PilotProfileBindingSchema).min(1),
|
|
47815
|
+
pilotTrapAuditReceipts: z3.array(PilotTrapReceiptSchema).min(1),
|
|
47816
|
+
pilotRunOrder: z3.array(z3.object({
|
|
47817
|
+
rowKey: z3.string().min(1),
|
|
47818
|
+
analysis: z3.enum(["PRIMARY", "TIMIDITY"]),
|
|
46632
47819
|
identity: PilotRowIdentitySchema
|
|
46633
47820
|
}).strict()).min(1),
|
|
46634
|
-
pilotExpectedDesignHash:
|
|
46635
|
-
pilotEpisodesHash:
|
|
47821
|
+
pilotExpectedDesignHash: z3.string().regex(SHA2562),
|
|
47822
|
+
pilotEpisodesHash: z3.string().regex(SHA2562)
|
|
46636
47823
|
}).strict();
|
|
46637
|
-
var FactPairAuditSchema =
|
|
46638
|
-
schemaVersion:
|
|
46639
|
-
pairs:
|
|
47824
|
+
var FactPairAuditSchema = z3.object({
|
|
47825
|
+
schemaVersion: z3.literal(1),
|
|
47826
|
+
pairs: z3.array(z3.object({ status: z3.enum(["MATCHED", "UNMATCHED"]) }).passthrough())
|
|
46640
47827
|
}).passthrough();
|
|
46641
|
-
var RegisteredFactPairAuditSchema =
|
|
46642
|
-
schemaVersion:
|
|
46643
|
-
minimumJaccard:
|
|
46644
|
-
maximumTokenGap:
|
|
46645
|
-
maximumRelativeTokenGap:
|
|
46646
|
-
pairs:
|
|
46647
|
-
pairKey:
|
|
46648
|
-
taskId:
|
|
46649
|
-
variantId:
|
|
46650
|
-
seed:
|
|
46651
|
-
modelProfileId:
|
|
46652
|
-
modelProfileHash:
|
|
46653
|
-
tokenizerIdentity:
|
|
46654
|
-
tokenizerImplementation:
|
|
46655
|
-
historyHash:
|
|
46656
|
-
failureRepoHash:
|
|
46657
|
-
successRepoHash:
|
|
46658
|
-
failureActionFingerprint:
|
|
46659
|
-
successActionFingerprint:
|
|
46660
|
-
failurePathShapeHash:
|
|
46661
|
-
successPathShapeHash:
|
|
46662
|
-
failureActionShapeHash:
|
|
46663
|
-
successActionShapeHash:
|
|
46664
|
-
failureFactId:
|
|
46665
|
-
failureCitationHash:
|
|
46666
|
-
failureFactHash:
|
|
46667
|
-
successFactHash:
|
|
46668
|
-
failureFactCount:
|
|
46669
|
-
successFactCount:
|
|
46670
|
-
failureTokens:
|
|
46671
|
-
successTokens:
|
|
46672
|
-
tokenGap:
|
|
46673
|
-
relativeTokenGap:
|
|
46674
|
-
jaccard:
|
|
46675
|
-
status:
|
|
47828
|
+
var RegisteredFactPairAuditSchema = z3.object({
|
|
47829
|
+
schemaVersion: z3.literal(1),
|
|
47830
|
+
minimumJaccard: z3.literal(0.8),
|
|
47831
|
+
maximumTokenGap: z3.literal(8),
|
|
47832
|
+
maximumRelativeTokenGap: z3.literal(0.05),
|
|
47833
|
+
pairs: z3.array(z3.object({
|
|
47834
|
+
pairKey: z3.string().regex(SHA2562),
|
|
47835
|
+
taskId: z3.string().min(1),
|
|
47836
|
+
variantId: z3.string().min(1),
|
|
47837
|
+
seed: z3.number().int().nonnegative(),
|
|
47838
|
+
modelProfileId: z3.string().min(1),
|
|
47839
|
+
modelProfileHash: z3.string().regex(SHA2562),
|
|
47840
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47841
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1"),
|
|
47842
|
+
historyHash: z3.string().regex(SHA2562),
|
|
47843
|
+
failureRepoHash: z3.string().regex(SHA2562),
|
|
47844
|
+
successRepoHash: z3.string().regex(SHA2562),
|
|
47845
|
+
failureActionFingerprint: z3.string().min(1),
|
|
47846
|
+
successActionFingerprint: z3.string().min(1),
|
|
47847
|
+
failurePathShapeHash: z3.string().regex(SHA2562),
|
|
47848
|
+
successPathShapeHash: z3.string().regex(SHA2562),
|
|
47849
|
+
failureActionShapeHash: z3.string().regex(SHA2562),
|
|
47850
|
+
successActionShapeHash: z3.string().regex(SHA2562),
|
|
47851
|
+
failureFactId: z3.string().min(1),
|
|
47852
|
+
failureCitationHash: z3.string().regex(SHA2562),
|
|
47853
|
+
failureFactHash: z3.string().regex(SHA2562),
|
|
47854
|
+
successFactHash: z3.string().regex(SHA2562),
|
|
47855
|
+
failureFactCount: z3.literal(1),
|
|
47856
|
+
successFactCount: z3.literal(1),
|
|
47857
|
+
failureTokens: z3.number().int().nonnegative(),
|
|
47858
|
+
successTokens: z3.number().int().nonnegative(),
|
|
47859
|
+
tokenGap: z3.number().int().nonnegative(),
|
|
47860
|
+
relativeTokenGap: z3.number().nonnegative(),
|
|
47861
|
+
jaccard: z3.number().min(0).max(1),
|
|
47862
|
+
status: z3.enum(["MATCHED", "UNMATCHED"])
|
|
46676
47863
|
}).strict())
|
|
46677
47864
|
}).strict();
|
|
46678
|
-
var TraceTimingSchema =
|
|
46679
|
-
armAudit:
|
|
46680
|
-
timingPayload:
|
|
46681
|
-
frame:
|
|
46682
|
-
factId:
|
|
46683
|
-
citationHash:
|
|
46684
|
-
factCount:
|
|
46685
|
-
renderedTokenCount:
|
|
47865
|
+
var TraceTimingSchema = z3.object({
|
|
47866
|
+
armAudit: z3.object({
|
|
47867
|
+
timingPayload: z3.object({
|
|
47868
|
+
frame: z3.enum(["TURN_START", "PRE_ACTION"]),
|
|
47869
|
+
factId: z3.string().min(1),
|
|
47870
|
+
citationHash: z3.string().regex(SHA2562),
|
|
47871
|
+
factCount: z3.literal(1),
|
|
47872
|
+
renderedTokenCount: z3.number().int().nonnegative()
|
|
46686
47873
|
}).strict().nullable(),
|
|
46687
|
-
turnStartFactHash:
|
|
46688
|
-
preActionFailureFactHash:
|
|
47874
|
+
turnStartFactHash: z3.string().regex(SHA2562).nullable(),
|
|
47875
|
+
preActionFailureFactHash: z3.string().regex(SHA2562).nullable()
|
|
46689
47876
|
}).passthrough()
|
|
46690
47877
|
}).passthrough();
|
|
46691
47878
|
function registeredProfileBindingsMatch(bindings, expectedProfileCount) {
|
|
@@ -46766,22 +47953,22 @@ function aggregateArmOutcomes(rows) {
|
|
|
46766
47953
|
const first = group[0];
|
|
46767
47954
|
if (!first) throw new Error("arm outcome group cannot be empty");
|
|
46768
47955
|
const valid = group.filter((row) => row.status === "VALID");
|
|
46769
|
-
const
|
|
47956
|
+
const mean5 = (values) => values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
46770
47957
|
return {
|
|
46771
47958
|
modelProfileId: first.identity.modelProfileId,
|
|
46772
47959
|
modelProfileHash: first.identity.modelProfileHash,
|
|
46773
47960
|
arm: first.identity.arm,
|
|
46774
47961
|
validRows: valid.length,
|
|
46775
47962
|
invalidRows: group.length - valid.length,
|
|
46776
|
-
repeatedFailureRate:
|
|
46777
|
-
taskPassRate:
|
|
46778
|
-
meanSteps:
|
|
47963
|
+
repeatedFailureRate: mean5(valid.map((row) => row.repeatedFailure ? 1 : 0)),
|
|
47964
|
+
taskPassRate: mean5(valid.map((row) => row.taskPassed ? 1 : 0)),
|
|
47965
|
+
meanSteps: mean5(valid.map(
|
|
46779
47966
|
(row) => requireNonnegativeInteger(row.steps, "steps", row.rowKey)
|
|
46780
47967
|
)),
|
|
46781
|
-
warningRate:
|
|
47968
|
+
warningRate: mean5(valid.map(
|
|
46782
47969
|
(row) => requireNonnegativeInteger(row.warningCount, "warningCount", row.rowKey) > 0 ? 1 : 0
|
|
46783
47970
|
)),
|
|
46784
|
-
falseWarningRate:
|
|
47971
|
+
falseWarningRate: mean5(valid.map(
|
|
46785
47972
|
(row) => requireNonnegativeInteger(row.falseWarningCount, "falseWarningCount", row.rowKey) > 0 ? 1 : 0
|
|
46786
47973
|
))
|
|
46787
47974
|
};
|
|
@@ -47293,10 +48480,10 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
47293
48480
|
const reproManifest = await verifyRunManifest(runDir);
|
|
47294
48481
|
const source = await readSourceArtifacts(runDir);
|
|
47295
48482
|
const runJson = JSON.parse(source["run.json"]);
|
|
47296
|
-
const runBinding =
|
|
47297
|
-
decisionRuleHash:
|
|
47298
|
-
preregistrationPath:
|
|
47299
|
-
preregistrationHash:
|
|
48483
|
+
const runBinding = z4.object({
|
|
48484
|
+
decisionRuleHash: z4.string().regex(SHA2562),
|
|
48485
|
+
preregistrationPath: z4.string().min(1),
|
|
48486
|
+
preregistrationHash: z4.string().regex(SHA2562)
|
|
47300
48487
|
}).passthrough().parse(runJson);
|
|
47301
48488
|
const decisionRuleBytes = source["decision-rule.json"];
|
|
47302
48489
|
const decisionRule = DecisionRuleSchema.parse(JSON.parse(decisionRuleBytes));
|