@remnic/bench 9.69.35 → 9.69.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js
CHANGED
|
@@ -7,6 +7,8 @@ import {
|
|
|
7
7
|
BaseTaskSchema,
|
|
8
8
|
BenchmarkRunBlockReason,
|
|
9
9
|
BenchmarkRunBlockedError,
|
|
10
|
+
CANARY_FIXED_RECALL,
|
|
11
|
+
CANARY_SCORE_FLOOR,
|
|
10
12
|
CodexCreditAccountingError,
|
|
11
13
|
CodexCreditDispatchError,
|
|
12
14
|
ControlledResponsesDriver,
|
|
@@ -54,6 +56,7 @@ import {
|
|
|
54
56
|
aggregateTaskScores,
|
|
55
57
|
analyzeRepeatedFailureRows,
|
|
56
58
|
applyPatchAndCommit,
|
|
59
|
+
assertCanaryUnderFloor,
|
|
57
60
|
assertIntegrityMetaPresent,
|
|
58
61
|
assertNoSymlinkComponents,
|
|
59
62
|
assertPublishableIntegrity,
|
|
@@ -82,6 +85,7 @@ import {
|
|
|
82
85
|
computeTrapAuditMetrics,
|
|
83
86
|
containsAnswer,
|
|
84
87
|
countFactTokens,
|
|
88
|
+
createCanaryAdapter,
|
|
85
89
|
createControlledResponsesAgentDriver,
|
|
86
90
|
createRepeatedFailureOllamaChatDriver,
|
|
87
91
|
createSeededRandom,
|
|
@@ -128,6 +132,7 @@ import {
|
|
|
128
132
|
materializeTaskRepo,
|
|
129
133
|
mergeContaminationManifests,
|
|
130
134
|
openSeal,
|
|
135
|
+
parseCanaryFloor,
|
|
131
136
|
parseCodexJsonlUsage,
|
|
132
137
|
parseDesign,
|
|
133
138
|
parseEpisodesJsonl,
|
|
@@ -145,9 +150,11 @@ import {
|
|
|
145
150
|
replayRepeatedFailureStatistics,
|
|
146
151
|
resolveBenchmarkResultReference,
|
|
147
152
|
resolveBenchmarkRunId,
|
|
153
|
+
resolveCanaryFloorFromEnv,
|
|
148
154
|
resolveCodexCreditBudgetConfig,
|
|
149
155
|
resolveCommittedH6FixtureDirectory,
|
|
150
156
|
resolveContainedPath,
|
|
157
|
+
resolveEffectiveCanaryFloor,
|
|
151
158
|
retryFetch,
|
|
152
159
|
rougeL,
|
|
153
160
|
runRepeatedFailureCliCommand,
|
|
@@ -180,7 +187,7 @@ import {
|
|
|
180
187
|
writeLeaderboardArtifactsForResult,
|
|
181
188
|
writeRepeatedFailureRunMetadata,
|
|
182
189
|
writeRepeatedFailureStatistics
|
|
183
|
-
} from "./chunk-
|
|
190
|
+
} from "./chunk-PWFYSAUK.js";
|
|
184
191
|
|
|
185
192
|
// src/build-week-evidence-receipt.ts
|
|
186
193
|
import { createHash as createHash2 } from "crypto";
|
|
@@ -5229,10 +5236,10 @@ ${coreRecall.trim()}`;
|
|
|
5229
5236
|
const engine = getEngine();
|
|
5230
5237
|
const abortController = new AbortController();
|
|
5231
5238
|
let timer;
|
|
5232
|
-
const timeout = new Promise((_,
|
|
5239
|
+
const timeout = new Promise((_, reject2) => {
|
|
5233
5240
|
timer = setTimeout(() => {
|
|
5234
5241
|
abortController.abort();
|
|
5235
|
-
|
|
5242
|
+
reject2(
|
|
5236
5243
|
new Error(
|
|
5237
5244
|
`drain() timed out after ${drainTimeoutMs}ms (${describeDrainState(state.orchestrator)})`
|
|
5238
5245
|
)
|
|
@@ -5316,8 +5323,8 @@ async function withBenchPhaseAbort(promise, control, phase, options = {}) {
|
|
|
5316
5323
|
throw benchPhaseAbortError(signal, phase);
|
|
5317
5324
|
}
|
|
5318
5325
|
let abortHandler;
|
|
5319
|
-
const abortPromise = new Promise((_,
|
|
5320
|
-
abortHandler = () =>
|
|
5326
|
+
const abortPromise = new Promise((_, reject2) => {
|
|
5327
|
+
abortHandler = () => reject2(benchPhaseAbortError(signal, phase));
|
|
5321
5328
|
signal.addEventListener("abort", abortHandler, { once: true });
|
|
5322
5329
|
});
|
|
5323
5330
|
try {
|
|
@@ -6756,10 +6763,10 @@ async function raceWithSignal(promise, signal, label, onLateSuccess) {
|
|
|
6756
6763
|
if (!signal) return promise;
|
|
6757
6764
|
let aborted = signal.aborted;
|
|
6758
6765
|
let abortListener;
|
|
6759
|
-
const abortPromise = new Promise((_resolve,
|
|
6766
|
+
const abortPromise = new Promise((_resolve, reject2) => {
|
|
6760
6767
|
const rejectForAbort = () => {
|
|
6761
6768
|
aborted = true;
|
|
6762
|
-
|
|
6769
|
+
reject2(signal.reason instanceof Error ? signal.reason : new Error(`${label} aborted`));
|
|
6763
6770
|
};
|
|
6764
6771
|
if (signal.aborted) rejectForAbort();
|
|
6765
6772
|
else {
|
|
@@ -7171,12 +7178,12 @@ async function withTimeout(label, timeoutMs, fn, onTimeout) {
|
|
|
7171
7178
|
const controller = new AbortController();
|
|
7172
7179
|
let timeoutError;
|
|
7173
7180
|
const task = Promise.resolve().then(() => fn(controller.signal));
|
|
7174
|
-
const timeout = new Promise((_,
|
|
7181
|
+
const timeout = new Promise((_, reject2) => {
|
|
7175
7182
|
timer = setTimeout(() => {
|
|
7176
7183
|
timeoutError = new Error(
|
|
7177
7184
|
`benchmark phase timed out after ${timeoutMs}ms: ${label}`
|
|
7178
7185
|
);
|
|
7179
|
-
|
|
7186
|
+
reject2(timeoutError);
|
|
7180
7187
|
controller.abort(timeoutError);
|
|
7181
7188
|
if (onTimeout) {
|
|
7182
7189
|
void Promise.resolve(onTimeout(label)).catch(() => {
|
|
@@ -7570,7 +7577,8 @@ var BENCHMARK_RESULT_SCHEMA = {
|
|
|
7570
7577
|
qrelsSealedHash: { type: "string", pattern: "^[0-9a-f]{64}$" },
|
|
7571
7578
|
judgePromptHash: { type: "string", pattern: "^[0-9a-f]{64}$" },
|
|
7572
7579
|
datasetHash: { type: "string", pattern: "^[0-9a-f]{64}$" },
|
|
7573
|
-
canaryScore: { type: "number" }
|
|
7580
|
+
canaryScore: { type: "number" },
|
|
7581
|
+
canaryFloor: { type: "number", minimum: 0 }
|
|
7574
7582
|
}
|
|
7575
7583
|
},
|
|
7576
7584
|
config: {
|
|
@@ -8281,14 +8289,14 @@ async function sleepBeforeClaudeCliRetry(options) {
|
|
|
8281
8289
|
if (signal.aborted) {
|
|
8282
8290
|
throw claudeCliAbortError(signal);
|
|
8283
8291
|
}
|
|
8284
|
-
await new Promise((resolve,
|
|
8292
|
+
await new Promise((resolve, reject2) => {
|
|
8285
8293
|
const cleanup = () => {
|
|
8286
8294
|
signal.removeEventListener("abort", onAbort);
|
|
8287
8295
|
};
|
|
8288
8296
|
const onAbort = () => {
|
|
8289
8297
|
clearTimeout(timeout);
|
|
8290
8298
|
cleanup();
|
|
8291
|
-
|
|
8299
|
+
reject2(claudeCliAbortError(signal));
|
|
8292
8300
|
};
|
|
8293
8301
|
const timeout = setTimeout(() => {
|
|
8294
8302
|
cleanup();
|
|
@@ -8311,7 +8319,7 @@ function summarizeProcessOutput(stderr, stdout) {
|
|
|
8311
8319
|
return summary.length > 0 ? summary.slice(-1e3) : "no process output";
|
|
8312
8320
|
}
|
|
8313
8321
|
function runClaudeVersionCommand(executable, env) {
|
|
8314
|
-
return new Promise((resolve,
|
|
8322
|
+
return new Promise((resolve, reject2) => {
|
|
8315
8323
|
const child = spawn(executable, ["--version"], {
|
|
8316
8324
|
env,
|
|
8317
8325
|
stdio: ["ignore", "ignore", "pipe"],
|
|
@@ -8349,7 +8357,7 @@ function runClaudeVersionCommand(executable, env) {
|
|
|
8349
8357
|
if (killTimeout) {
|
|
8350
8358
|
clearTimeout(killTimeout);
|
|
8351
8359
|
}
|
|
8352
|
-
|
|
8360
|
+
reject2(error);
|
|
8353
8361
|
});
|
|
8354
8362
|
child.on("close", (status) => {
|
|
8355
8363
|
clearTimeout(timeout);
|
|
@@ -8365,7 +8373,7 @@ Claude CLI --version timed out after ${CLAUDE_CLI_VERSION_TIMEOUT_MS}ms.`) : std
|
|
|
8365
8373
|
});
|
|
8366
8374
|
}
|
|
8367
8375
|
function runClaudeCliCommand(request) {
|
|
8368
|
-
return new Promise((resolve,
|
|
8376
|
+
return new Promise((resolve, reject2) => {
|
|
8369
8377
|
if (request.signal?.aborted) {
|
|
8370
8378
|
resolve({
|
|
8371
8379
|
status: 124,
|
|
@@ -8453,7 +8461,7 @@ Claude CLI stdin error: ${error.code ?? error.message}`);
|
|
|
8453
8461
|
unregisterActiveClaudeCliChild(child.pid);
|
|
8454
8462
|
}
|
|
8455
8463
|
request.signal?.removeEventListener("abort", onAbort);
|
|
8456
|
-
|
|
8464
|
+
reject2(error);
|
|
8457
8465
|
});
|
|
8458
8466
|
child.on("close", (status, signal) => {
|
|
8459
8467
|
if (timeout) {
|
|
@@ -9197,7 +9205,7 @@ ${result.stderr}`.trim();
|
|
|
9197
9205
|
}
|
|
9198
9206
|
};
|
|
9199
9207
|
function runCodexVersionCommand(executable, env) {
|
|
9200
|
-
return new Promise((resolve,
|
|
9208
|
+
return new Promise((resolve, reject2) => {
|
|
9201
9209
|
const child = spawn2(executable, ["--version"], {
|
|
9202
9210
|
env,
|
|
9203
9211
|
stdio: ["ignore", "ignore", "pipe"],
|
|
@@ -9235,7 +9243,7 @@ function runCodexVersionCommand(executable, env) {
|
|
|
9235
9243
|
if (killTimeout) {
|
|
9236
9244
|
clearTimeout(killTimeout);
|
|
9237
9245
|
}
|
|
9238
|
-
|
|
9246
|
+
reject2(error);
|
|
9239
9247
|
});
|
|
9240
9248
|
child.on("close", (status) => {
|
|
9241
9249
|
clearTimeout(timeout);
|
|
@@ -9254,7 +9262,7 @@ Codex CLI --version timed out after ${CODEX_CLI_VERSION_TIMEOUT_MS}ms.`
|
|
|
9254
9262
|
});
|
|
9255
9263
|
}
|
|
9256
9264
|
function runCodexLoginStatusCommand(executable, env) {
|
|
9257
|
-
return new Promise((resolve,
|
|
9265
|
+
return new Promise((resolve, reject2) => {
|
|
9258
9266
|
const child = spawn2(executable, ["login", "status"], {
|
|
9259
9267
|
env,
|
|
9260
9268
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -9296,7 +9304,7 @@ Codex CLI login status timed out after ${CODEX_CLI_VERSION_TIMEOUT_MS}ms.`
|
|
|
9296
9304
|
child.on("error", (error) => {
|
|
9297
9305
|
clearTimeout(timeout);
|
|
9298
9306
|
if (killTimeout) clearTimeout(killTimeout);
|
|
9299
|
-
|
|
9307
|
+
reject2(error);
|
|
9300
9308
|
});
|
|
9301
9309
|
child.on("close", (status) => {
|
|
9302
9310
|
clearTimeout(timeout);
|
|
@@ -9484,9 +9492,9 @@ function redactCodexCliArgs(args) {
|
|
|
9484
9492
|
return redacted;
|
|
9485
9493
|
}
|
|
9486
9494
|
function runCodexCliCommand(request) {
|
|
9487
|
-
return new Promise((resolve,
|
|
9495
|
+
return new Promise((resolve, reject2) => {
|
|
9488
9496
|
if (request.signal?.aborted) {
|
|
9489
|
-
|
|
9497
|
+
reject2(codexCliPreStartAbortError(request.signal));
|
|
9490
9498
|
return;
|
|
9491
9499
|
}
|
|
9492
9500
|
const child = spawn2(request.executable, request.args, {
|
|
@@ -9570,7 +9578,7 @@ Codex CLI stdin error: ${error.code ?? error.message}`
|
|
|
9570
9578
|
unregisterActiveCodexCliChild(child.pid);
|
|
9571
9579
|
}
|
|
9572
9580
|
request.signal?.removeEventListener("abort", onAbort);
|
|
9573
|
-
|
|
9581
|
+
reject2(
|
|
9574
9582
|
child.pid ? new Error(
|
|
9575
9583
|
`Codex CLI failed after its process started: ${safeErrorMessage(error)}`,
|
|
9576
9584
|
{ cause: error }
|
|
@@ -9758,14 +9766,14 @@ async function sleepBeforeCodexCliRetry(attempt, configuredBaseBackoffMs, signal
|
|
|
9758
9766
|
if (signal.aborted) {
|
|
9759
9767
|
throw codexCliAbortError(signal);
|
|
9760
9768
|
}
|
|
9761
|
-
await new Promise((resolve,
|
|
9769
|
+
await new Promise((resolve, reject2) => {
|
|
9762
9770
|
const cleanup = () => {
|
|
9763
9771
|
signal.removeEventListener("abort", onAbort);
|
|
9764
9772
|
};
|
|
9765
9773
|
const onAbort = () => {
|
|
9766
9774
|
clearTimeout(timeout);
|
|
9767
9775
|
cleanup();
|
|
9768
|
-
|
|
9776
|
+
reject2(codexCliAbortError(signal));
|
|
9769
9777
|
};
|
|
9770
9778
|
const timeout = setTimeout(() => {
|
|
9771
9779
|
cleanup();
|
|
@@ -26095,8 +26103,1191 @@ function constantAggregate(value) {
|
|
|
26095
26103
|
};
|
|
26096
26104
|
}
|
|
26097
26105
|
|
|
26098
|
-
// src/benchmarks/remnic/
|
|
26106
|
+
// src/benchmarks/remnic/extraction-span-mode/runner.ts
|
|
26099
26107
|
import { randomUUID as randomUUID12 } from "crypto";
|
|
26108
|
+
import { evaluateSpanPhaseGate } from "@remnic/core/extraction-span-gate";
|
|
26109
|
+
import { tallySpanFallbacks } from "@remnic/core/extraction-span-fallback";
|
|
26110
|
+
|
|
26111
|
+
// src/benchmarks/remnic/extraction-span-mode/fixture.ts
|
|
26112
|
+
var SPAN_BENCH_FIXTURE = [
|
|
26113
|
+
{
|
|
26114
|
+
id: "locomo-maya",
|
|
26115
|
+
dataset: "locomo",
|
|
26116
|
+
messages: [
|
|
26117
|
+
{
|
|
26118
|
+
speaker: "Maya",
|
|
26119
|
+
text: "I moved to Seattle last spring after accepting the new senior role at the design studio, and I still miss the Chicago food scene."
|
|
26120
|
+
},
|
|
26121
|
+
{ speaker: "Assistant", text: "Seattle sounds exciting. I'll remember that move." },
|
|
26122
|
+
{
|
|
26123
|
+
speaker: "Maya",
|
|
26124
|
+
text: "My favorite tea is jasmine, especially during rainy mornings in the winter when the studio windows fog up."
|
|
26125
|
+
},
|
|
26126
|
+
{ speaker: "Assistant", text: "Jasmine tea on rainy mornings. Got it." },
|
|
26127
|
+
{
|
|
26128
|
+
speaker: "Maya",
|
|
26129
|
+
text: "Actually, call me M. Nobody at the studio uses my full name anymore, not even the founders."
|
|
26130
|
+
},
|
|
26131
|
+
{ speaker: "Assistant", text: "M it is." }
|
|
26132
|
+
],
|
|
26133
|
+
facts: [
|
|
26134
|
+
{
|
|
26135
|
+
id: "maya-move",
|
|
26136
|
+
messageIndex: 0,
|
|
26137
|
+
quote: "I moved to Seattle last spring after accepting the new senior role at the design studio",
|
|
26138
|
+
frame: "Maya's relocation",
|
|
26139
|
+
content: "Maya moved to Seattle last spring after accepting the new senior role at the design studio.",
|
|
26140
|
+
restatement: "Maya moved to Seattle last spring after taking the senior role at the design studio.",
|
|
26141
|
+
category: "fact",
|
|
26142
|
+
confidence: 0.95,
|
|
26143
|
+
tags: ["relocation", "seattle"]
|
|
26144
|
+
},
|
|
26145
|
+
{
|
|
26146
|
+
id: "maya-tea",
|
|
26147
|
+
messageIndex: 2,
|
|
26148
|
+
quote: "My favorite tea is jasmine, especially during rainy mornings in the winter",
|
|
26149
|
+
frame: "Maya's favorite tea",
|
|
26150
|
+
content: "Maya's favorite tea is jasmine, especially during rainy mornings in the winter.",
|
|
26151
|
+
restatement: "Maya's favorite tea is jasmine, particularly on rainy mornings in winter.",
|
|
26152
|
+
category: "preference",
|
|
26153
|
+
confidence: 0.93,
|
|
26154
|
+
tags: ["tea", "preference"]
|
|
26155
|
+
},
|
|
26156
|
+
{
|
|
26157
|
+
id: "maya-name",
|
|
26158
|
+
messageIndex: 4,
|
|
26159
|
+
quote: "Actually, call me M. Nobody at the studio uses my full name anymore",
|
|
26160
|
+
frame: "Maya's preferred name",
|
|
26161
|
+
content: "Maya prefers to be called M; nobody at the studio uses her full name anymore.",
|
|
26162
|
+
restatement: "Maya goes by M; no one at the studio uses her full name anymore.",
|
|
26163
|
+
category: "preference",
|
|
26164
|
+
confidence: 0.9,
|
|
26165
|
+
tags: ["name", "preference"]
|
|
26166
|
+
}
|
|
26167
|
+
]
|
|
26168
|
+
},
|
|
26169
|
+
{
|
|
26170
|
+
id: "locomo-hackathon",
|
|
26171
|
+
dataset: "locomo",
|
|
26172
|
+
messages: [
|
|
26173
|
+
{
|
|
26174
|
+
speaker: "Alice",
|
|
26175
|
+
text: "We finalized the venue for the hackathon: the riverside loft on June 14th, which books half-day cleaning slots before opening."
|
|
26176
|
+
},
|
|
26177
|
+
{ speaker: "Ben", text: "Nice. I can mentor the beginners' track that weekend if the schedule holds." },
|
|
26178
|
+
{ speaker: "Alice", text: "Perfect. Registration caps at 80 people this year because of the loft occupancy permit." },
|
|
26179
|
+
{ speaker: "Ben", text: "I'll bring the extra power strips from the office." },
|
|
26180
|
+
{
|
|
26181
|
+
speaker: "Alice",
|
|
26182
|
+
text: "Also, the after-party is vegetarian only \u2014 the caterer confirmed today that every dish is plant-based."
|
|
26183
|
+
}
|
|
26184
|
+
],
|
|
26185
|
+
facts: [
|
|
26186
|
+
{
|
|
26187
|
+
id: "venue",
|
|
26188
|
+
messageIndex: 0,
|
|
26189
|
+
quote: "We finalized the venue for the hackathon: the riverside loft on June 14th",
|
|
26190
|
+
frame: "Hackathon venue",
|
|
26191
|
+
content: "The hackathon venue is the riverside loft on June 14th.",
|
|
26192
|
+
restatement: "The hackathon venue is the riverside loft, June 14th.",
|
|
26193
|
+
category: "decision",
|
|
26194
|
+
confidence: 0.96,
|
|
26195
|
+
tags: ["hackathon", "venue"]
|
|
26196
|
+
},
|
|
26197
|
+
{
|
|
26198
|
+
id: "mentor",
|
|
26199
|
+
messageIndex: 1,
|
|
26200
|
+
quote: "I can mentor the beginners' track that weekend if the schedule holds",
|
|
26201
|
+
frame: "Ben's hackathon role",
|
|
26202
|
+
content: "Ben can mentor the beginners' track that weekend if the schedule holds.",
|
|
26203
|
+
restatement: "Ben can mentor the beginners' track that weekend if scheduling works.",
|
|
26204
|
+
category: "commitment",
|
|
26205
|
+
confidence: 0.92,
|
|
26206
|
+
tags: ["hackathon", "mentor"]
|
|
26207
|
+
},
|
|
26208
|
+
{
|
|
26209
|
+
id: "capacity",
|
|
26210
|
+
messageIndex: 2,
|
|
26211
|
+
quote: "Registration caps at 80 people this year because of the loft occupancy permit",
|
|
26212
|
+
frame: "Hackathon registration cap",
|
|
26213
|
+
content: "Hackathon registration caps at 80 people this year because of the loft occupancy permit.",
|
|
26214
|
+
restatement: "Registration caps at 80 people this year due to the loft occupancy permit.",
|
|
26215
|
+
category: "fact",
|
|
26216
|
+
confidence: 0.94,
|
|
26217
|
+
tags: ["hackathon", "capacity"]
|
|
26218
|
+
},
|
|
26219
|
+
{
|
|
26220
|
+
id: "catering",
|
|
26221
|
+
messageIndex: 4,
|
|
26222
|
+
quote: "the after-party is vegetarian only \u2014 the caterer confirmed today",
|
|
26223
|
+
frame: "After-party catering",
|
|
26224
|
+
content: "The hackathon after-party is vegetarian only \u2014 the caterer confirmed today.",
|
|
26225
|
+
restatement: "The after-party is vegetarian only \u2014 the caterer confirmed it today.",
|
|
26226
|
+
category: "decision",
|
|
26227
|
+
confidence: 0.91,
|
|
26228
|
+
tags: ["hackathon", "catering"]
|
|
26229
|
+
}
|
|
26230
|
+
]
|
|
26231
|
+
},
|
|
26232
|
+
{
|
|
26233
|
+
id: "locomo-clara",
|
|
26234
|
+
dataset: "locomo",
|
|
26235
|
+
messages: [
|
|
26236
|
+
{
|
|
26237
|
+
speaker: "Clara",
|
|
26238
|
+
text: "My thesis defense is scheduled for March 3rd, not March 13th \u2014 they moved it up a week at my request."
|
|
26239
|
+
},
|
|
26240
|
+
{ speaker: "Assistant", text: "Noted: March 3rd." },
|
|
26241
|
+
{
|
|
26242
|
+
speaker: "Clara",
|
|
26243
|
+
text: "I stopped drinking coffee; my doctor suggested cutting caffeine entirely to help with the afternoon crashes."
|
|
26244
|
+
},
|
|
26245
|
+
{ speaker: "Assistant", text: "Cutting caffeine entirely \u2014 understood." },
|
|
26246
|
+
{
|
|
26247
|
+
speaker: "Clara",
|
|
26248
|
+
text: "I've been learning Japanese for two years now, mostly with podcasts during the morning commute."
|
|
26249
|
+
},
|
|
26250
|
+
{ speaker: "Assistant", text: "Two years of Japanese via podcasts. Impressive." }
|
|
26251
|
+
],
|
|
26252
|
+
facts: [
|
|
26253
|
+
{
|
|
26254
|
+
id: "defense-date",
|
|
26255
|
+
messageIndex: 0,
|
|
26256
|
+
quote: "My thesis defense is scheduled for March 3rd, not March 13th",
|
|
26257
|
+
frame: "Clara's thesis defense date",
|
|
26258
|
+
content: "Clara's thesis defense is scheduled for March 3rd, not March 13th.",
|
|
26259
|
+
restatement: "Clara's thesis defense is scheduled for March 3, not March 13.",
|
|
26260
|
+
category: "correction",
|
|
26261
|
+
confidence: 0.97,
|
|
26262
|
+
tags: ["thesis", "schedule"]
|
|
26263
|
+
},
|
|
26264
|
+
{
|
|
26265
|
+
id: "caffeine",
|
|
26266
|
+
messageIndex: 2,
|
|
26267
|
+
quote: "I stopped drinking coffee; my doctor suggested cutting caffeine entirely",
|
|
26268
|
+
frame: "Clara's caffeine cut",
|
|
26269
|
+
content: "Clara stopped drinking coffee; her doctor suggested cutting caffeine entirely.",
|
|
26270
|
+
restatement: "Clara stopped drinking coffee; her doctor advised cutting caffeine entirely.",
|
|
26271
|
+
category: "fact",
|
|
26272
|
+
confidence: 0.93,
|
|
26273
|
+
tags: ["health", "caffeine"]
|
|
26274
|
+
},
|
|
26275
|
+
{
|
|
26276
|
+
id: "japanese",
|
|
26277
|
+
messageIndex: 4,
|
|
26278
|
+
quote: "I've been learning Japanese for two years now, mostly with podcasts",
|
|
26279
|
+
frame: "Clara's Japanese study",
|
|
26280
|
+
content: "Clara has been learning Japanese for two years now, mostly with podcasts.",
|
|
26281
|
+
restatement: "Clara has been learning Japanese for two years, mostly via podcasts.",
|
|
26282
|
+
category: "skill",
|
|
26283
|
+
confidence: 0.9,
|
|
26284
|
+
tags: ["japanese", "learning"]
|
|
26285
|
+
}
|
|
26286
|
+
]
|
|
26287
|
+
},
|
|
26288
|
+
{
|
|
26289
|
+
id: "lme-notebook",
|
|
26290
|
+
dataset: "longmemeval",
|
|
26291
|
+
messages: [
|
|
26292
|
+
{
|
|
26293
|
+
speaker: "User",
|
|
26294
|
+
text: "I switched my main notebook to a Framework 13 last October after the old one died mid-flight."
|
|
26295
|
+
},
|
|
26296
|
+
{ speaker: "Assistant", text: "Framework 13 it is." },
|
|
26297
|
+
{
|
|
26298
|
+
speaker: "User",
|
|
26299
|
+
text: "I keep all my research notes in Zettelkasten folders now, ever since the markdown sprawl got unusable."
|
|
26300
|
+
},
|
|
26301
|
+
{ speaker: "Assistant", text: "Zettelkasten folders, got it." },
|
|
26302
|
+
{
|
|
26303
|
+
speaker: "User",
|
|
26304
|
+
text: "My partner Priya started a part-time ceramics course in January at the community studio near the library."
|
|
26305
|
+
},
|
|
26306
|
+
{ speaker: "Assistant", text: "Priya's ceramics course \u2014 noted." },
|
|
26307
|
+
{ speaker: "User", text: "I try to read 20 pages a day before bed, and it mostly sticks." },
|
|
26308
|
+
{ speaker: "Assistant", text: "20 pages before bed, noted." }
|
|
26309
|
+
],
|
|
26310
|
+
facts: [
|
|
26311
|
+
{
|
|
26312
|
+
id: "notebook",
|
|
26313
|
+
messageIndex: 0,
|
|
26314
|
+
quote: "I switched my main notebook to a Framework 13 last October",
|
|
26315
|
+
frame: "User's main notebook",
|
|
26316
|
+
content: "The user switched their main notebook to a Framework 13 last October.",
|
|
26317
|
+
restatement: "The user moved their main notebook to a Framework 13 last October.",
|
|
26318
|
+
category: "fact",
|
|
26319
|
+
confidence: 0.95,
|
|
26320
|
+
tags: ["hardware", "notebook"]
|
|
26321
|
+
},
|
|
26322
|
+
{
|
|
26323
|
+
id: "notes-system",
|
|
26324
|
+
messageIndex: 2,
|
|
26325
|
+
quote: "I keep all my research notes in Zettelkasten folders now",
|
|
26326
|
+
frame: "User's research notes",
|
|
26327
|
+
content: "The user keeps all research notes in Zettelkasten folders now.",
|
|
26328
|
+
restatement: "The user keeps research notes in Zettelkasten folders these days.",
|
|
26329
|
+
category: "fact",
|
|
26330
|
+
confidence: 0.92,
|
|
26331
|
+
tags: ["notes", "workflow"]
|
|
26332
|
+
},
|
|
26333
|
+
{
|
|
26334
|
+
id: "priya-ceramics",
|
|
26335
|
+
messageIndex: 4,
|
|
26336
|
+
quote: "My partner Priya started a part-time ceramics course in January",
|
|
26337
|
+
frame: "Priya's ceramics course",
|
|
26338
|
+
content: "The user's partner Priya started a part-time ceramics course in January.",
|
|
26339
|
+
restatement: "The user's partner Priya began a part-time ceramics course in January.",
|
|
26340
|
+
category: "relationship",
|
|
26341
|
+
confidence: 0.91,
|
|
26342
|
+
tags: ["priya", "ceramics"]
|
|
26343
|
+
},
|
|
26344
|
+
{
|
|
26345
|
+
id: "reading-habit",
|
|
26346
|
+
messageIndex: 6,
|
|
26347
|
+
quote: "I try to read 20 pages a day before bed",
|
|
26348
|
+
frame: "User's reading habit",
|
|
26349
|
+
content: "The user tries to read 20 pages a day before bed.",
|
|
26350
|
+
restatement: "The user aims to read 20 pages a day before bed.",
|
|
26351
|
+
category: "preference",
|
|
26352
|
+
confidence: 0.9,
|
|
26353
|
+
tags: ["reading", "habit"]
|
|
26354
|
+
}
|
|
26355
|
+
]
|
|
26356
|
+
},
|
|
26357
|
+
{
|
|
26358
|
+
id: "lme-fitness",
|
|
26359
|
+
dataset: "longmemeval",
|
|
26360
|
+
messages: [
|
|
26361
|
+
{
|
|
26362
|
+
speaker: "User",
|
|
26363
|
+
text: "I canceled my gym membership; I train at home with kettlebells now, three sessions a week."
|
|
26364
|
+
},
|
|
26365
|
+
{ speaker: "Assistant", text: "Home kettlebell training, noted." },
|
|
26366
|
+
{
|
|
26367
|
+
speaker: "User",
|
|
26368
|
+
text: "For invoices I use FreeInvoice, the open-source one, since 2023 when the freelance work picked up."
|
|
26369
|
+
},
|
|
26370
|
+
{ speaker: "Assistant", text: "FreeInvoice since 2023, got it." },
|
|
26371
|
+
{
|
|
26372
|
+
speaker: "User",
|
|
26373
|
+
text: "I'm allergic to walnuts but pecans are fine, so I check bakery labels every time."
|
|
26374
|
+
},
|
|
26375
|
+
{ speaker: "Assistant", text: "Walnut allergy noted; pecans fine." }
|
|
26376
|
+
],
|
|
26377
|
+
facts: [
|
|
26378
|
+
{
|
|
26379
|
+
id: "training",
|
|
26380
|
+
messageIndex: 0,
|
|
26381
|
+
quote: "I canceled my gym membership; I train at home with kettlebells now",
|
|
26382
|
+
frame: "User's home training",
|
|
26383
|
+
content: "The user canceled their gym membership and trains at home with kettlebells now.",
|
|
26384
|
+
restatement: "The user canceled the gym membership and now trains at home with kettlebells.",
|
|
26385
|
+
category: "fact",
|
|
26386
|
+
confidence: 0.94,
|
|
26387
|
+
tags: ["fitness", "kettlebells"]
|
|
26388
|
+
},
|
|
26389
|
+
{
|
|
26390
|
+
id: "invoicing",
|
|
26391
|
+
messageIndex: 2,
|
|
26392
|
+
quote: "For invoices I use FreeInvoice, the open-source one, since 2023",
|
|
26393
|
+
frame: "User's invoicing tool",
|
|
26394
|
+
content: "For invoices the user uses FreeInvoice, the open-source one, since 2023.",
|
|
26395
|
+
restatement: "The user uses open-source FreeInvoice for invoices since 2023.",
|
|
26396
|
+
category: "fact",
|
|
26397
|
+
confidence: 0.93,
|
|
26398
|
+
tags: ["invoicing", "tools"]
|
|
26399
|
+
},
|
|
26400
|
+
{
|
|
26401
|
+
id: "walnuts",
|
|
26402
|
+
messageIndex: 4,
|
|
26403
|
+
quote: "I'm allergic to walnuts but pecans are fine",
|
|
26404
|
+
frame: "User's nut allergy",
|
|
26405
|
+
content: "The user is allergic to walnuts but pecans are fine.",
|
|
26406
|
+
restatement: "The user is allergic to walnuts; pecans are fine.",
|
|
26407
|
+
category: "fact",
|
|
26408
|
+
confidence: 0.96,
|
|
26409
|
+
tags: ["allergy", "walnuts"]
|
|
26410
|
+
}
|
|
26411
|
+
]
|
|
26412
|
+
},
|
|
26413
|
+
{
|
|
26414
|
+
id: "lme-routines",
|
|
26415
|
+
dataset: "longmemeval",
|
|
26416
|
+
messages: [
|
|
26417
|
+
{
|
|
26418
|
+
speaker: "User",
|
|
26419
|
+
text: "My dentist appointments are always the first Tuesday of the month, right when the office opens."
|
|
26420
|
+
},
|
|
26421
|
+
{ speaker: "Assistant", text: "First Tuesday dental visits, noted." },
|
|
26422
|
+
{
|
|
26423
|
+
speaker: "User",
|
|
26424
|
+
text: "I park in the Blue Garage on level 3 when visiting the office, since street parking vanished."
|
|
26425
|
+
},
|
|
26426
|
+
{ speaker: "Assistant", text: "Blue Garage level 3, got it." },
|
|
26427
|
+
{
|
|
26428
|
+
speaker: "User",
|
|
26429
|
+
text: "I compile my Rust projects with nightly, never stable, because of the async trait syntax."
|
|
26430
|
+
},
|
|
26431
|
+
{ speaker: "Assistant", text: "Rust nightly toolchain, noted." }
|
|
26432
|
+
],
|
|
26433
|
+
facts: [
|
|
26434
|
+
{
|
|
26435
|
+
id: "dentist",
|
|
26436
|
+
messageIndex: 0,
|
|
26437
|
+
quote: "My dentist appointments are always the first Tuesday of the month",
|
|
26438
|
+
frame: "User's dentist schedule",
|
|
26439
|
+
content: "The user's dentist appointments are always the first Tuesday of the month.",
|
|
26440
|
+
restatement: "The user's dentist appointments are always on the first Tuesday of the month.",
|
|
26441
|
+
category: "fact",
|
|
26442
|
+
confidence: 0.92,
|
|
26443
|
+
tags: ["dentist", "schedule"]
|
|
26444
|
+
},
|
|
26445
|
+
{
|
|
26446
|
+
id: "parking",
|
|
26447
|
+
messageIndex: 2,
|
|
26448
|
+
quote: "I park in the Blue Garage on level 3 when visiting the office",
|
|
26449
|
+
frame: "User's office parking",
|
|
26450
|
+
content: "The user parks in the Blue Garage on level 3 when visiting the office.",
|
|
26451
|
+
restatement: "The user parks at the Blue Garage, level 3, for office visits.",
|
|
26452
|
+
category: "fact",
|
|
26453
|
+
confidence: 0.93,
|
|
26454
|
+
tags: ["office", "parking"]
|
|
26455
|
+
},
|
|
26456
|
+
{
|
|
26457
|
+
id: "rust-toolchain",
|
|
26458
|
+
messageIndex: 4,
|
|
26459
|
+
quote: "I compile my Rust projects with nightly, never stable",
|
|
26460
|
+
frame: "User's Rust toolchain",
|
|
26461
|
+
content: "The user compiles Rust projects with nightly, never stable.",
|
|
26462
|
+
restatement: "The user builds Rust projects with nightly, never stable.",
|
|
26463
|
+
category: "preference",
|
|
26464
|
+
confidence: 0.95,
|
|
26465
|
+
tags: ["rust", "toolchain"]
|
|
26466
|
+
}
|
|
26467
|
+
]
|
|
26468
|
+
},
|
|
26469
|
+
{
|
|
26470
|
+
id: "locomo-diego",
|
|
26471
|
+
dataset: "locomo",
|
|
26472
|
+
messages: [
|
|
26473
|
+
{
|
|
26474
|
+
speaker: "Diego",
|
|
26475
|
+
text: "I finally finished restoring the grandfather clock my uncle left me; it took nine weekends of shop time."
|
|
26476
|
+
},
|
|
26477
|
+
{ speaker: "Assistant", text: "Nine weekends \u2014 that's dedication." },
|
|
26478
|
+
{
|
|
26479
|
+
speaker: "Diego",
|
|
26480
|
+
text: "My daughter Rosa starts at the maritime academy in September, and she is nervous about the swim test."
|
|
26481
|
+
},
|
|
26482
|
+
{ speaker: "Assistant", text: "Maritime academy in September, noted." },
|
|
26483
|
+
{
|
|
26484
|
+
speaker: "Diego",
|
|
26485
|
+
text: "I stopped coaching the youth league once the schedule collided with my night classes, but I still referee on Sundays."
|
|
26486
|
+
},
|
|
26487
|
+
{ speaker: "Assistant", text: "Refereeing on Sundays only now." }
|
|
26488
|
+
],
|
|
26489
|
+
facts: [
|
|
26490
|
+
{
|
|
26491
|
+
id: "clock-restoration",
|
|
26492
|
+
messageIndex: 0,
|
|
26493
|
+
quote: "I finally finished restoring the grandfather clock my uncle left me",
|
|
26494
|
+
frame: "Diego's clock restoration",
|
|
26495
|
+
content: "Diego finished restoring the grandfather clock his uncle left him.",
|
|
26496
|
+
restatement: "Diego finished restoring the grandfather clock he inherited from his uncle.",
|
|
26497
|
+
category: "moment",
|
|
26498
|
+
confidence: 0.9,
|
|
26499
|
+
tags: ["clock", "restoration"]
|
|
26500
|
+
},
|
|
26501
|
+
{
|
|
26502
|
+
id: "rosa-academy",
|
|
26503
|
+
messageIndex: 2,
|
|
26504
|
+
quote: "My daughter Rosa starts at the maritime academy in September",
|
|
26505
|
+
frame: "Rosa's maritime academy start",
|
|
26506
|
+
content: "Diego's daughter Rosa starts at the maritime academy in September.",
|
|
26507
|
+
restatement: "Diego's daughter Rosa begins at the maritime academy in September.",
|
|
26508
|
+
category: "commitment",
|
|
26509
|
+
confidence: 0.93,
|
|
26510
|
+
tags: ["rosa", "academy"]
|
|
26511
|
+
},
|
|
26512
|
+
{
|
|
26513
|
+
id: "refereeing",
|
|
26514
|
+
messageIndex: 4,
|
|
26515
|
+
quote: "but I still referee on Sundays",
|
|
26516
|
+
frame: "Diego's Sunday refereeing",
|
|
26517
|
+
content: "Diego still referees on Sundays after stopping youth-league coaching.",
|
|
26518
|
+
restatement: "Diego still referees Sundays after quitting youth-league coaching.",
|
|
26519
|
+
category: "commitment",
|
|
26520
|
+
confidence: 0.88,
|
|
26521
|
+
tags: ["referee", "schedule"]
|
|
26522
|
+
}
|
|
26523
|
+
]
|
|
26524
|
+
},
|
|
26525
|
+
{
|
|
26526
|
+
id: "locomo-band",
|
|
26527
|
+
dataset: "locomo",
|
|
26528
|
+
messages: [
|
|
26529
|
+
{
|
|
26530
|
+
speaker: "Nadia",
|
|
26531
|
+
text: "Our band's debut EP drops on Bandcamp the first Friday of October, all five tracks self-recorded."
|
|
26532
|
+
},
|
|
26533
|
+
{ speaker: "Assistant", text: "First Friday of October \u2014 congrats." },
|
|
26534
|
+
{
|
|
26535
|
+
speaker: "Nadia",
|
|
26536
|
+
text: "I switched from bass to synths last winter because the setlist changed direction."
|
|
26537
|
+
},
|
|
26538
|
+
{ speaker: "Assistant", text: "Synths instead of bass, noted." },
|
|
26539
|
+
{
|
|
26540
|
+
speaker: "Nadia",
|
|
26541
|
+
text: "Rehearsals moved to the storage-unit space on Ferry Street since the old room doubled its rent."
|
|
26542
|
+
},
|
|
26543
|
+
{ speaker: "Assistant", text: "Ferry Street rehearsal space, got it." }
|
|
26544
|
+
],
|
|
26545
|
+
facts: [
|
|
26546
|
+
{
|
|
26547
|
+
id: "ep-release",
|
|
26548
|
+
messageIndex: 0,
|
|
26549
|
+
quote: "Our band's debut EP drops on Bandcamp the first Friday of October",
|
|
26550
|
+
frame: "Band's EP release",
|
|
26551
|
+
content: "Nadia's band's debut EP drops on Bandcamp the first Friday of October.",
|
|
26552
|
+
restatement: "Nadia's band releases its debut EP on Bandcamp the first Friday of October.",
|
|
26553
|
+
category: "commitment",
|
|
26554
|
+
confidence: 0.94,
|
|
26555
|
+
tags: ["band", "release"]
|
|
26556
|
+
},
|
|
26557
|
+
{
|
|
26558
|
+
id: "instrument-switch",
|
|
26559
|
+
messageIndex: 2,
|
|
26560
|
+
quote: "I switched from bass to synths last winter",
|
|
26561
|
+
frame: "Nadia's instrument switch",
|
|
26562
|
+
content: "Nadia switched from bass to synths last winter.",
|
|
26563
|
+
restatement: "Nadia moved from bass to synths last winter.",
|
|
26564
|
+
category: "fact",
|
|
26565
|
+
confidence: 0.91,
|
|
26566
|
+
tags: ["band", "instrument"]
|
|
26567
|
+
},
|
|
26568
|
+
{
|
|
26569
|
+
id: "rehearsal-space",
|
|
26570
|
+
messageIndex: 4,
|
|
26571
|
+
quote: "Rehearsals moved to the storage-unit space on Ferry Street",
|
|
26572
|
+
frame: "Band's rehearsal space",
|
|
26573
|
+
content: "Band rehearsals moved to the storage-unit space on Ferry Street.",
|
|
26574
|
+
restatement: "The band now rehearses in the Ferry Street storage-unit space.",
|
|
26575
|
+
category: "fact",
|
|
26576
|
+
confidence: 0.9,
|
|
26577
|
+
tags: ["band", "rehearsal"]
|
|
26578
|
+
}
|
|
26579
|
+
]
|
|
26580
|
+
},
|
|
26581
|
+
{
|
|
26582
|
+
id: "locomo-garden",
|
|
26583
|
+
dataset: "locomo",
|
|
26584
|
+
messages: [
|
|
26585
|
+
{
|
|
26586
|
+
speaker: "Tom",
|
|
26587
|
+
text: "The community garden allotted me plot 12, the shady corner near the compost bins."
|
|
26588
|
+
},
|
|
26589
|
+
{ speaker: "Assistant", text: "Plot 12, the shady corner." },
|
|
26590
|
+
{
|
|
26591
|
+
speaker: "Tom",
|
|
26592
|
+
text: "I'm growing mostly leafy greens this season because the tomatoes failed in the shade two years running."
|
|
26593
|
+
},
|
|
26594
|
+
{ speaker: "Assistant", text: "Leafy greens it is." },
|
|
26595
|
+
{
|
|
26596
|
+
speaker: "Tom",
|
|
26597
|
+
text: "My knee surgery is rescheduled to the 21st, so I'll miss the spring workday for the first time."
|
|
26598
|
+
},
|
|
26599
|
+
{ speaker: "Assistant", text: "Surgery on the 21st \u2014 noted." }
|
|
26600
|
+
],
|
|
26601
|
+
facts: [
|
|
26602
|
+
{
|
|
26603
|
+
id: "garden-plot",
|
|
26604
|
+
messageIndex: 0,
|
|
26605
|
+
quote: "The community garden allotted me plot 12, the shady corner near the compost bins",
|
|
26606
|
+
frame: "Tom's garden plot",
|
|
26607
|
+
content: "The community garden allotted Tom plot 12, the shady corner near the compost bins.",
|
|
26608
|
+
restatement: "Tom's community garden plot is 12, the shady corner by the compost bins.",
|
|
26609
|
+
category: "fact",
|
|
26610
|
+
confidence: 0.92,
|
|
26611
|
+
tags: ["garden", "plot"]
|
|
26612
|
+
},
|
|
26613
|
+
{
|
|
26614
|
+
id: "leafy-greens",
|
|
26615
|
+
messageIndex: 2,
|
|
26616
|
+
quote: "I'm growing mostly leafy greens this season",
|
|
26617
|
+
frame: "Tom's seasonal crops",
|
|
26618
|
+
content: "Tom is growing mostly leafy greens this season.",
|
|
26619
|
+
restatement: "Tom is growing mostly leafy greens this year.",
|
|
26620
|
+
category: "fact",
|
|
26621
|
+
confidence: 0.9,
|
|
26622
|
+
tags: ["garden", "crops"]
|
|
26623
|
+
},
|
|
26624
|
+
{
|
|
26625
|
+
id: "knee-surgery",
|
|
26626
|
+
messageIndex: 4,
|
|
26627
|
+
quote: "My knee surgery is rescheduled to the 21st",
|
|
26628
|
+
frame: "Tom's knee surgery date",
|
|
26629
|
+
content: "Tom's knee surgery is rescheduled to the 21st.",
|
|
26630
|
+
restatement: "Tom's knee surgery got moved to the 21st.",
|
|
26631
|
+
category: "correction",
|
|
26632
|
+
confidence: 0.95,
|
|
26633
|
+
tags: ["health", "surgery"]
|
|
26634
|
+
}
|
|
26635
|
+
]
|
|
26636
|
+
},
|
|
26637
|
+
{
|
|
26638
|
+
id: "lme-tools",
|
|
26639
|
+
dataset: "longmemeval",
|
|
26640
|
+
messages: [
|
|
26641
|
+
{
|
|
26642
|
+
speaker: "User",
|
|
26643
|
+
text: "I migrated all my passwords to a local vault last month after the breach news, and I rotate the master key quarterly."
|
|
26644
|
+
},
|
|
26645
|
+
{ speaker: "Assistant", text: "Local vault with quarterly rotation, noted." },
|
|
26646
|
+
{
|
|
26647
|
+
speaker: "User",
|
|
26648
|
+
text: "I write my standup notes in the shared team doc before 9:30, never in the DM thread."
|
|
26649
|
+
},
|
|
26650
|
+
{ speaker: "Assistant", text: "Standup notes in the shared doc before 9:30." },
|
|
26651
|
+
{
|
|
26652
|
+
speaker: "User",
|
|
26653
|
+
text: "My brother Marco covers my dog-sitting every other Thursday when the late deploy window lands."
|
|
26654
|
+
},
|
|
26655
|
+
{ speaker: "Assistant", text: "Marco's Thursday dog-sitting, noted." }
|
|
26656
|
+
],
|
|
26657
|
+
facts: [
|
|
26658
|
+
{
|
|
26659
|
+
id: "password-vault",
|
|
26660
|
+
messageIndex: 0,
|
|
26661
|
+
quote: "I migrated all my passwords to a local vault last month",
|
|
26662
|
+
frame: "User's password vault",
|
|
26663
|
+
content: "The user migrated all their passwords to a local vault last month.",
|
|
26664
|
+
restatement: "The user moved all passwords into a local vault last month.",
|
|
26665
|
+
category: "fact",
|
|
26666
|
+
confidence: 0.94,
|
|
26667
|
+
tags: ["security", "vault"]
|
|
26668
|
+
},
|
|
26669
|
+
{
|
|
26670
|
+
id: "standup-notes",
|
|
26671
|
+
messageIndex: 2,
|
|
26672
|
+
quote: "I write my standup notes in the shared team doc before 9:30",
|
|
26673
|
+
frame: "User's standup note habit",
|
|
26674
|
+
content: "The user writes standup notes in the shared team doc before 9:30.",
|
|
26675
|
+
restatement: "The user posts standup notes in the shared team doc before 9:30.",
|
|
26676
|
+
category: "preference",
|
|
26677
|
+
confidence: 0.91,
|
|
26678
|
+
tags: ["standup", "workflow"]
|
|
26679
|
+
},
|
|
26680
|
+
{
|
|
26681
|
+
id: "marco-dogsitting",
|
|
26682
|
+
messageIndex: 4,
|
|
26683
|
+
quote: "My brother Marco covers my dog-sitting every other Thursday",
|
|
26684
|
+
frame: "Marco's dog-sitting schedule",
|
|
26685
|
+
content: "The user's brother Marco covers dog-sitting every other Thursday.",
|
|
26686
|
+
restatement: "The user's brother Marco handles dog-sitting every other Thursday.",
|
|
26687
|
+
category: "relationship",
|
|
26688
|
+
confidence: 0.9,
|
|
26689
|
+
tags: ["marco", "dog-sitting"]
|
|
26690
|
+
}
|
|
26691
|
+
]
|
|
26692
|
+
},
|
|
26693
|
+
{
|
|
26694
|
+
id: "lme-study",
|
|
26695
|
+
dataset: "longmemeval",
|
|
26696
|
+
messages: [
|
|
26697
|
+
{
|
|
26698
|
+
speaker: "User",
|
|
26699
|
+
text: "I passed the licensing exam on the second attempt last March, eight points above the cutoff."
|
|
26700
|
+
},
|
|
26701
|
+
{ speaker: "Assistant", text: "Passed on the second attempt \u2014 congrats." },
|
|
26702
|
+
{
|
|
26703
|
+
speaker: "User",
|
|
26704
|
+
text: "I study best at the kitchen counter with noise-canceling headphones, never at the desk."
|
|
26705
|
+
},
|
|
26706
|
+
{ speaker: "Assistant", text: "Kitchen counter with headphones, noted." },
|
|
26707
|
+
{
|
|
26708
|
+
speaker: "User",
|
|
26709
|
+
text: "My tutor Lena charges a sliding scale, and she waived the fee during my exam retake month."
|
|
26710
|
+
},
|
|
26711
|
+
{ speaker: "Assistant", text: "Lena's sliding-scale tutoring, noted." }
|
|
26712
|
+
],
|
|
26713
|
+
facts: [
|
|
26714
|
+
{
|
|
26715
|
+
id: "licensing-exam",
|
|
26716
|
+
messageIndex: 0,
|
|
26717
|
+
quote: "I passed the licensing exam on the second attempt last March",
|
|
26718
|
+
frame: "User's licensing exam result",
|
|
26719
|
+
content: "The user passed the licensing exam on the second attempt last March.",
|
|
26720
|
+
restatement: "The user passed the licensing exam on the second try last March.",
|
|
26721
|
+
category: "moment",
|
|
26722
|
+
confidence: 0.95,
|
|
26723
|
+
tags: ["exam", "licensing"]
|
|
26724
|
+
},
|
|
26725
|
+
{
|
|
26726
|
+
id: "study-setup",
|
|
26727
|
+
messageIndex: 2,
|
|
26728
|
+
quote: "I study best at the kitchen counter with noise-canceling headphones",
|
|
26729
|
+
frame: "User's study setup",
|
|
26730
|
+
content: "The user studies best at the kitchen counter with noise-canceling headphones.",
|
|
26731
|
+
restatement: "The user studies best at the kitchen counter wearing noise-canceling headphones.",
|
|
26732
|
+
category: "preference",
|
|
26733
|
+
confidence: 0.9,
|
|
26734
|
+
tags: ["study", "environment"]
|
|
26735
|
+
},
|
|
26736
|
+
{
|
|
26737
|
+
id: "tutor-lena",
|
|
26738
|
+
messageIndex: 4,
|
|
26739
|
+
quote: "My tutor Lena charges a sliding scale",
|
|
26740
|
+
frame: "Lena's tutoring fee",
|
|
26741
|
+
content: "The user's tutor Lena charges a sliding scale.",
|
|
26742
|
+
restatement: "The user's tutor Lena uses a sliding scale for fees.",
|
|
26743
|
+
category: "fact",
|
|
26744
|
+
confidence: 0.89,
|
|
26745
|
+
tags: ["lena", "tutoring"]
|
|
26746
|
+
}
|
|
26747
|
+
]
|
|
26748
|
+
},
|
|
26749
|
+
{
|
|
26750
|
+
id: "lme-travel",
|
|
26751
|
+
dataset: "longmemeval",
|
|
26752
|
+
messages: [
|
|
26753
|
+
{
|
|
26754
|
+
speaker: "User",
|
|
26755
|
+
text: "I only fly out of the regional airport now; the major hub's security line ate two hours of my life."
|
|
26756
|
+
},
|
|
26757
|
+
{ speaker: "Assistant", text: "Regional airport only, noted." },
|
|
26758
|
+
{
|
|
26759
|
+
speaker: "User",
|
|
26760
|
+
text: "I collect vintage transit maps, and the 1968 one from the city tram network is my favorite piece."
|
|
26761
|
+
},
|
|
26762
|
+
{ speaker: "Assistant", text: "Vintage transit maps, 1968 tram favorite." },
|
|
26763
|
+
{
|
|
26764
|
+
speaker: "User",
|
|
26765
|
+
text: "My passport expires next June, so I renewed it early through the postal service."
|
|
26766
|
+
},
|
|
26767
|
+
{ speaker: "Assistant", text: "Passport renewed early, noted." },
|
|
26768
|
+
{
|
|
26769
|
+
speaker: "User",
|
|
26770
|
+
text: "I book window seats on day flights and aisle seats on red-eyes, no exceptions."
|
|
26771
|
+
},
|
|
26772
|
+
{ speaker: "Assistant", text: "Window by day, aisle by night." }
|
|
26773
|
+
],
|
|
26774
|
+
facts: [
|
|
26775
|
+
{
|
|
26776
|
+
id: "regional-airport",
|
|
26777
|
+
messageIndex: 0,
|
|
26778
|
+
quote: "I only fly out of the regional airport now",
|
|
26779
|
+
frame: "User's airport choice",
|
|
26780
|
+
content: "The user only flies out of the regional airport now.",
|
|
26781
|
+
restatement: "The user flies only out of the regional airport now.",
|
|
26782
|
+
category: "preference",
|
|
26783
|
+
confidence: 0.92,
|
|
26784
|
+
tags: ["travel", "airport"]
|
|
26785
|
+
},
|
|
26786
|
+
{
|
|
26787
|
+
id: "transit-maps",
|
|
26788
|
+
messageIndex: 2,
|
|
26789
|
+
quote: "I collect vintage transit maps",
|
|
26790
|
+
frame: "User's transit map collection",
|
|
26791
|
+
content: "The user collects vintage transit maps.",
|
|
26792
|
+
restatement: "The user collects old transit maps.",
|
|
26793
|
+
category: "preference",
|
|
26794
|
+
confidence: 0.88,
|
|
26795
|
+
tags: ["travel", "collection"]
|
|
26796
|
+
},
|
|
26797
|
+
{
|
|
26798
|
+
id: "passport-renewal",
|
|
26799
|
+
messageIndex: 4,
|
|
26800
|
+
quote: "My passport expires next June, so I renewed it early",
|
|
26801
|
+
frame: "User's passport renewal",
|
|
26802
|
+
content: "The user renewed their passport early; it expires next June.",
|
|
26803
|
+
restatement: "The user renewed the passport early since it expires next June.",
|
|
26804
|
+
category: "fact",
|
|
26805
|
+
confidence: 0.93,
|
|
26806
|
+
tags: ["travel", "passport"]
|
|
26807
|
+
},
|
|
26808
|
+
{
|
|
26809
|
+
id: "seat-preference",
|
|
26810
|
+
messageIndex: 6,
|
|
26811
|
+
quote: "I book window seats on day flights and aisle seats on red-eyes",
|
|
26812
|
+
frame: "User's seat preferences",
|
|
26813
|
+
content: "The user books window seats on day flights and aisle seats on red-eyes.",
|
|
26814
|
+
restatement: "The user takes window seats on day flights, aisle on red-eyes.",
|
|
26815
|
+
category: "preference",
|
|
26816
|
+
confidence: 0.91,
|
|
26817
|
+
tags: ["travel", "seating"]
|
|
26818
|
+
}
|
|
26819
|
+
]
|
|
26820
|
+
}
|
|
26821
|
+
];
|
|
26822
|
+
var SPAN_BENCH_SMOKE_FIXTURE = [
|
|
26823
|
+
SPAN_BENCH_FIXTURE[0],
|
|
26824
|
+
SPAN_BENCH_FIXTURE[3]
|
|
26825
|
+
];
|
|
26826
|
+
|
|
26827
|
+
// src/benchmarks/remnic/extraction-span-mode/segment.ts
|
|
26828
|
+
import { stampSpanSource } from "@remnic/core/extraction-span-source-hash";
|
|
26829
|
+
function renderSegment(conversation) {
|
|
26830
|
+
const lines = [
|
|
26831
|
+
"Numbered conversation segment. Character offsets index each message's text",
|
|
26832
|
+
"after the speaker prefix (offsets are [charStart, charEnd), end-exclusive).",
|
|
26833
|
+
"The printed char count and source hash/length identify that message text.",
|
|
26834
|
+
"For each memory, return sourceMessageIndex plus a verbatim supporting",
|
|
26835
|
+
"span's charStart/charEnd, and a frame of at most 15 words that makes the",
|
|
26836
|
+
"span self-contained (resolve pronouns, name the subject).",
|
|
26837
|
+
""
|
|
26838
|
+
];
|
|
26839
|
+
const messages = conversation.messages.map((message, index) => {
|
|
26840
|
+
const stamp = stampSpanSource(message.text);
|
|
26841
|
+
lines.push(
|
|
26842
|
+
`[${index}] ${message.speaker} (${message.text.length} chars, hash ${stamp.hash}, length ${stamp.length}): ${message.text}`
|
|
26843
|
+
);
|
|
26844
|
+
return {
|
|
26845
|
+
index,
|
|
26846
|
+
speaker: message.speaker,
|
|
26847
|
+
text: message.text,
|
|
26848
|
+
stamp
|
|
26849
|
+
};
|
|
26850
|
+
});
|
|
26851
|
+
return { messages, prompt: lines.join("\n") };
|
|
26852
|
+
}
|
|
26853
|
+
|
|
26854
|
+
// src/benchmarks/remnic/extraction-span-mode/materialize.ts
|
|
26855
|
+
import { verifySpanSource } from "@remnic/core/extraction-span-source-hash";
|
|
26856
|
+
var SPAN_MAX_SLICE_CHARS = 400;
|
|
26857
|
+
var SPAN_MAX_FRAME_WORDS = 15;
|
|
26858
|
+
function reject(reason, fallbackContent) {
|
|
26859
|
+
return { content: fallbackContent, outcome: "fallback", reason };
|
|
26860
|
+
}
|
|
26861
|
+
function materializeSpanFact(fact3, segmentMessages) {
|
|
26862
|
+
const fallbackContent = (fact3.content ?? fact3.span?.frame ?? "").trim();
|
|
26863
|
+
const span = fact3.span ?? null;
|
|
26864
|
+
if (span === null) {
|
|
26865
|
+
return reject("no_span", fallbackContent);
|
|
26866
|
+
}
|
|
26867
|
+
if (fallbackContent === "") {
|
|
26868
|
+
return { content: "", outcome: "fallback", reason: "missing_fallback_content" };
|
|
26869
|
+
}
|
|
26870
|
+
const { sourceMessageIndex, charStart, charEnd, frame } = span;
|
|
26871
|
+
if (!Number.isInteger(sourceMessageIndex) || sourceMessageIndex < 0 || sourceMessageIndex >= segmentMessages.length) {
|
|
26872
|
+
return reject("message_index_out_of_range", fallbackContent);
|
|
26873
|
+
}
|
|
26874
|
+
const message = segmentMessages[sourceMessageIndex];
|
|
26875
|
+
const stampCheck = verifySpanSource(message.text, message.stamp);
|
|
26876
|
+
if (!stampCheck.ok) {
|
|
26877
|
+
return reject(`source_${stampCheck.error}`, fallbackContent);
|
|
26878
|
+
}
|
|
26879
|
+
if (!Number.isInteger(charStart) || !Number.isInteger(charEnd) || charStart < 0 || charEnd > message.text.length) {
|
|
26880
|
+
return reject("offsets_out_of_range", fallbackContent);
|
|
26881
|
+
}
|
|
26882
|
+
if (charStart >= charEnd) {
|
|
26883
|
+
return reject("empty_interval", fallbackContent);
|
|
26884
|
+
}
|
|
26885
|
+
const slice = message.text.slice(charStart, charEnd);
|
|
26886
|
+
if (slice.length > SPAN_MAX_SLICE_CHARS) {
|
|
26887
|
+
return reject("slice_too_long", fallbackContent);
|
|
26888
|
+
}
|
|
26889
|
+
if (slice.trim() === "") {
|
|
26890
|
+
return reject("blank_slice", fallbackContent);
|
|
26891
|
+
}
|
|
26892
|
+
const frameWords = frame.trim().split(/\s+/).filter(Boolean);
|
|
26893
|
+
if (frameWords.length === 0 || frameWords.length > SPAN_MAX_FRAME_WORDS) {
|
|
26894
|
+
return reject("frame_word_count", fallbackContent);
|
|
26895
|
+
}
|
|
26896
|
+
const trimmedFrame = frameWords.join(" ");
|
|
26897
|
+
const content = frameEndsWithPunctuation(trimmedFrame) ? `${trimmedFrame} ${slice}` : `${trimmedFrame}: ${slice}`;
|
|
26898
|
+
return {
|
|
26899
|
+
content,
|
|
26900
|
+
outcome: "span",
|
|
26901
|
+
quote: slice,
|
|
26902
|
+
charStart,
|
|
26903
|
+
charEnd,
|
|
26904
|
+
sourceMessageIndex
|
|
26905
|
+
};
|
|
26906
|
+
}
|
|
26907
|
+
function frameEndsWithPunctuation(frame) {
|
|
26908
|
+
return /[:—-]$/.test(frame);
|
|
26909
|
+
}
|
|
26910
|
+
|
|
26911
|
+
// src/benchmarks/remnic/extraction-span-mode/schema.ts
|
|
26912
|
+
import { z } from "zod";
|
|
26913
|
+
var SpanRefSchema = z.object({
|
|
26914
|
+
sourceMessageIndex: z.number(),
|
|
26915
|
+
charStart: z.number(),
|
|
26916
|
+
charEnd: z.number(),
|
|
26917
|
+
frame: z.string()
|
|
26918
|
+
});
|
|
26919
|
+
var CategorySchema = z.enum([
|
|
26920
|
+
"fact",
|
|
26921
|
+
"preference",
|
|
26922
|
+
"correction",
|
|
26923
|
+
"entity",
|
|
26924
|
+
"decision",
|
|
26925
|
+
"relationship",
|
|
26926
|
+
"principle",
|
|
26927
|
+
"commitment",
|
|
26928
|
+
"moment",
|
|
26929
|
+
"skill",
|
|
26930
|
+
"rule",
|
|
26931
|
+
"procedure",
|
|
26932
|
+
"reasoning_trace"
|
|
26933
|
+
]);
|
|
26934
|
+
var SpanModeFactSchema = z.object({
|
|
26935
|
+
category: CategorySchema,
|
|
26936
|
+
content: z.string().optional().nullable(),
|
|
26937
|
+
confidence: z.number().min(0).max(1),
|
|
26938
|
+
tags: z.array(z.string()),
|
|
26939
|
+
span: SpanRefSchema.optional().nullable()
|
|
26940
|
+
}).superRefine((value, ctx) => {
|
|
26941
|
+
if ((value.span === void 0 || value.span === null) && (value.content ?? "").trim() === "") {
|
|
26942
|
+
ctx.addIssue({
|
|
26943
|
+
code: z.ZodIssueCode.custom,
|
|
26944
|
+
message: "fact needs either a span or non-blank generated content"
|
|
26945
|
+
});
|
|
26946
|
+
}
|
|
26947
|
+
});
|
|
26948
|
+
var CurrentModeFactSchema = z.object({
|
|
26949
|
+
category: CategorySchema,
|
|
26950
|
+
content: z.string(),
|
|
26951
|
+
/**
|
|
26952
|
+
* Production parity: current-mode extraction also emits the verbatim
|
|
26953
|
+
* grounding quote (ExtractedFactSchema.quote, issue #1575). Span mode
|
|
26954
|
+
* replaces BOTH content and quote with offsets + frame.
|
|
26955
|
+
*/
|
|
26956
|
+
quote: z.string().optional().nullable(),
|
|
26957
|
+
confidence: z.number().min(0).max(1),
|
|
26958
|
+
tags: z.array(z.string()),
|
|
26959
|
+
span: SpanRefSchema.optional().nullable()
|
|
26960
|
+
});
|
|
26961
|
+
|
|
26962
|
+
// src/benchmarks/remnic/extraction-span-mode/fake-provider.ts
|
|
26963
|
+
import { estimateGeneratedTokens } from "@remnic/core/extraction-span-tokens";
|
|
26964
|
+
var MS_PER_OUTPUT_TOKEN = 40;
|
|
26965
|
+
var INVALID_SPAN_RATE = 0.04;
|
|
26966
|
+
var DRIFT_SPAN_RATE = 0.02;
|
|
26967
|
+
var DRIFT_CHARS = 3;
|
|
26968
|
+
function goldSpan(gold, conversation) {
|
|
26969
|
+
const text = conversation.messages[gold.messageIndex]?.text;
|
|
26970
|
+
if (text === void 0) {
|
|
26971
|
+
throw new Error(`fixture fact ${gold.id} references missing message ${gold.messageIndex}`);
|
|
26972
|
+
}
|
|
26973
|
+
const charStart = text.indexOf(gold.quote);
|
|
26974
|
+
if (charStart < 0) {
|
|
26975
|
+
throw new Error(`fixture fact ${gold.id} quote is not a verbatim substring of message ${gold.messageIndex}`);
|
|
26976
|
+
}
|
|
26977
|
+
return { charStart, charEnd: charStart + gold.quote.length };
|
|
26978
|
+
}
|
|
26979
|
+
function runFakeExtraction(conversation, mode, seed) {
|
|
26980
|
+
const rng = createSeededRandom((seed * 16777619 + stableConversationSalt(conversation.id)) % 4294967296);
|
|
26981
|
+
const rawFacts = conversation.facts.map((gold) => {
|
|
26982
|
+
const base = {
|
|
26983
|
+
category: gold.category,
|
|
26984
|
+
confidence: gold.confidence,
|
|
26985
|
+
tags: gold.tags
|
|
26986
|
+
};
|
|
26987
|
+
if (mode === "current") {
|
|
26988
|
+
const fact3 = { ...base, content: gold.restatement, quote: gold.quote };
|
|
26989
|
+
return fact3;
|
|
26990
|
+
}
|
|
26991
|
+
return emitSpanFact(gold, conversation, rng);
|
|
26992
|
+
});
|
|
26993
|
+
const responsePayload = JSON.stringify(rawFacts);
|
|
26994
|
+
const outputTokens = estimateGeneratedTokens(responsePayload.length);
|
|
26995
|
+
return {
|
|
26996
|
+
rawFacts,
|
|
26997
|
+
responsePayload,
|
|
26998
|
+
outputTokens,
|
|
26999
|
+
wallClockMs: outputTokens * MS_PER_OUTPUT_TOKEN,
|
|
27000
|
+
memoryEntryCount: rawFacts.length
|
|
27001
|
+
};
|
|
27002
|
+
}
|
|
27003
|
+
function stableConversationSalt(id) {
|
|
27004
|
+
let hash = 2166136261;
|
|
27005
|
+
for (let i = 0; i < id.length; i += 1) {
|
|
27006
|
+
hash ^= id.charCodeAt(i);
|
|
27007
|
+
hash = Math.imul(hash, 16777619);
|
|
27008
|
+
}
|
|
27009
|
+
return hash >>> 0;
|
|
27010
|
+
}
|
|
27011
|
+
function emitSpanFact(gold, conversation, rng) {
|
|
27012
|
+
const { charStart, charEnd } = goldSpan(gold, conversation);
|
|
27013
|
+
const messageLength = conversation.messages[gold.messageIndex].text.length;
|
|
27014
|
+
const roll = rng();
|
|
27015
|
+
const base = {
|
|
27016
|
+
category: gold.category,
|
|
27017
|
+
confidence: gold.confidence,
|
|
27018
|
+
tags: gold.tags,
|
|
27019
|
+
content: gold.frame
|
|
27020
|
+
};
|
|
27021
|
+
if (roll < INVALID_SPAN_RATE) {
|
|
27022
|
+
const variant = Math.floor(roll / INVALID_SPAN_RATE * 3) % 3;
|
|
27023
|
+
if (variant === 0) {
|
|
27024
|
+
return { ...base, span: { sourceMessageIndex: gold.messageIndex, charStart, charEnd: messageLength + 1, frame: gold.frame } };
|
|
27025
|
+
}
|
|
27026
|
+
if (variant === 1) {
|
|
27027
|
+
return { ...base, span: { sourceMessageIndex: gold.messageIndex, charStart, charEnd: charStart, frame: gold.frame } };
|
|
27028
|
+
}
|
|
27029
|
+
return { ...base, span: { sourceMessageIndex: gold.messageIndex, charStart: charEnd, charEnd: charStart, frame: gold.frame } };
|
|
27030
|
+
}
|
|
27031
|
+
if (roll < INVALID_SPAN_RATE + DRIFT_SPAN_RATE && charStart + DRIFT_CHARS < charEnd && charEnd + DRIFT_CHARS <= messageLength) {
|
|
27032
|
+
return {
|
|
27033
|
+
...base,
|
|
27034
|
+
span: {
|
|
27035
|
+
sourceMessageIndex: gold.messageIndex,
|
|
27036
|
+
charStart: charStart + DRIFT_CHARS,
|
|
27037
|
+
charEnd: charEnd + DRIFT_CHARS,
|
|
27038
|
+
frame: gold.frame
|
|
27039
|
+
}
|
|
27040
|
+
};
|
|
27041
|
+
}
|
|
27042
|
+
return {
|
|
27043
|
+
...base,
|
|
27044
|
+
span: { sourceMessageIndex: gold.messageIndex, charStart, charEnd, frame: gold.frame }
|
|
27045
|
+
};
|
|
27046
|
+
}
|
|
27047
|
+
|
|
27048
|
+
// src/benchmarks/remnic/extraction-span-mode/judge.ts
|
|
27049
|
+
function tokenizeForJudge(text) {
|
|
27050
|
+
return text.toLowerCase().split(/[^a-z0-9]+/).filter((token) => token.length > 0);
|
|
27051
|
+
}
|
|
27052
|
+
function judgeMemoryScore(memory, gold) {
|
|
27053
|
+
const goldTokens = tokenizeForJudge(gold);
|
|
27054
|
+
if (goldTokens.length === 0) {
|
|
27055
|
+
return 0;
|
|
27056
|
+
}
|
|
27057
|
+
const memoryTokens = tokenizeForJudge(memory);
|
|
27058
|
+
const available = /* @__PURE__ */ new Map();
|
|
27059
|
+
for (const token of memoryTokens) {
|
|
27060
|
+
available.set(token, (available.get(token) ?? 0) + 1);
|
|
27061
|
+
}
|
|
27062
|
+
let covered = 0;
|
|
27063
|
+
for (const token of goldTokens) {
|
|
27064
|
+
const count = available.get(token) ?? 0;
|
|
27065
|
+
if (count > 0) {
|
|
27066
|
+
covered += 1;
|
|
27067
|
+
available.set(token, count - 1);
|
|
27068
|
+
}
|
|
27069
|
+
}
|
|
27070
|
+
return covered / goldTokens.length * 100;
|
|
27071
|
+
}
|
|
27072
|
+
|
|
27073
|
+
// src/benchmarks/remnic/extraction-span-mode/runner.ts
|
|
27074
|
+
var extractionSpanModeDefinition = {
|
|
27075
|
+
id: "extraction-span-mode",
|
|
27076
|
+
title: "Extraction Span-Mode Phase A Gate",
|
|
27077
|
+
tier: "remnic",
|
|
27078
|
+
status: "ready",
|
|
27079
|
+
runnerAvailable: true,
|
|
27080
|
+
meta: {
|
|
27081
|
+
name: "extraction-span-mode",
|
|
27082
|
+
version: "1.0.0",
|
|
27083
|
+
description: "Deterministic fake-provider A/B of span-mode vs generated extraction with the issue #2333 Phase B gate (wall-clock, judge score, fallback rate).",
|
|
27084
|
+
category: "retrieval",
|
|
27085
|
+
citation: "arXiv 2602.03315 \xA75.2.4 Table 6; Remnic issue #2333"
|
|
27086
|
+
}
|
|
27087
|
+
};
|
|
27088
|
+
function newAccumulator() {
|
|
27089
|
+
return { conversationIds: [], judgeScores: [], wallClockMs: [], outputTokens: [], memoryEntries: [] };
|
|
27090
|
+
}
|
|
27091
|
+
function mean(values) {
|
|
27092
|
+
if (values.length === 0) {
|
|
27093
|
+
throw new Error("cannot average an empty sample");
|
|
27094
|
+
}
|
|
27095
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
27096
|
+
}
|
|
27097
|
+
async function runExtractionSpanModeBenchmark(options) {
|
|
27098
|
+
const seed = options.seed ?? 0;
|
|
27099
|
+
const conversations = loadConversations(options.mode, options.limit);
|
|
27100
|
+
const segmentByConversation = new Map(
|
|
27101
|
+
conversations.map((conversation) => [conversation.id, renderSegment(conversation)])
|
|
27102
|
+
);
|
|
27103
|
+
const tasks = [];
|
|
27104
|
+
const outcomes = [];
|
|
27105
|
+
const acc = { current: newAccumulator(), span: newAccumulator() };
|
|
27106
|
+
const modes = ["current", "span"];
|
|
27107
|
+
for (const conversation of conversations) {
|
|
27108
|
+
const segment = segmentByConversation.get(conversation.id);
|
|
27109
|
+
if (!segment) {
|
|
27110
|
+
throw new Error(`missing rendered segment for conversation ${conversation.id}`);
|
|
27111
|
+
}
|
|
27112
|
+
for (const mode of modes) {
|
|
27113
|
+
const run = runFakeExtraction(conversation, mode, seed);
|
|
27114
|
+
const factScores = [];
|
|
27115
|
+
const modeDetails = {
|
|
27116
|
+
dataset: conversation.dataset,
|
|
27117
|
+
mode,
|
|
27118
|
+
seed
|
|
27119
|
+
};
|
|
27120
|
+
if (mode === "current") {
|
|
27121
|
+
for (const [index, raw] of run.rawFacts.entries()) {
|
|
27122
|
+
const fact3 = CurrentModeFactSchema.parse(raw);
|
|
27123
|
+
const gold = conversation.facts[index];
|
|
27124
|
+
factScores.push(judgeMemoryScore(fact3.content, gold.content));
|
|
27125
|
+
}
|
|
27126
|
+
} else {
|
|
27127
|
+
const perFact = [];
|
|
27128
|
+
for (const [index, raw] of run.rawFacts.entries()) {
|
|
27129
|
+
const fact3 = SpanModeFactSchema.parse(raw);
|
|
27130
|
+
const gold = conversation.facts[index];
|
|
27131
|
+
const materialized = materializeSpanFact(fact3, segment.messages);
|
|
27132
|
+
outcomes.push(materialized.outcome);
|
|
27133
|
+
factScores.push(judgeMemoryScore(materialized.content, gold.content));
|
|
27134
|
+
perFact.push({
|
|
27135
|
+
factId: gold.id,
|
|
27136
|
+
outcome: materialized.outcome,
|
|
27137
|
+
reason: materialized.reason ?? null
|
|
27138
|
+
});
|
|
27139
|
+
}
|
|
27140
|
+
modeDetails.facts = perFact;
|
|
27141
|
+
}
|
|
27142
|
+
const judgeScore = mean(factScores);
|
|
27143
|
+
acc[mode].conversationIds.push(conversation.id);
|
|
27144
|
+
acc[mode].judgeScores.push(judgeScore);
|
|
27145
|
+
acc[mode].wallClockMs.push(run.wallClockMs);
|
|
27146
|
+
acc[mode].outputTokens.push(run.outputTokens);
|
|
27147
|
+
acc[mode].memoryEntries.push(run.memoryEntryCount);
|
|
27148
|
+
tasks.push({
|
|
27149
|
+
taskId: `${conversation.id}:${mode}`,
|
|
27150
|
+
question: `Extract memories from ${conversation.id} (${conversation.dataset}, ${mode} mode)`,
|
|
27151
|
+
expected: "gold-fact coverage",
|
|
27152
|
+
actual: `${mode} extraction via deterministic fake provider`,
|
|
27153
|
+
scores: {
|
|
27154
|
+
[`judge_score_${mode}`]: judgeScore,
|
|
27155
|
+
[`output_tokens_${mode}`]: run.outputTokens,
|
|
27156
|
+
[`wall_clock_ms_${mode}`]: run.wallClockMs,
|
|
27157
|
+
[`memory_entries_${mode}`]: run.memoryEntryCount
|
|
27158
|
+
},
|
|
27159
|
+
latencyMs: run.wallClockMs,
|
|
27160
|
+
tokens: { input: 0, output: run.outputTokens },
|
|
27161
|
+
details: modeDetails
|
|
27162
|
+
});
|
|
27163
|
+
}
|
|
27164
|
+
}
|
|
27165
|
+
const fallbackTally = tallySpanFallbacks(outcomes);
|
|
27166
|
+
if (fallbackTally.fallbackRatePct === null) {
|
|
27167
|
+
throw new Error("span mode produced no span attempts; the gate cannot be evaluated on an unmeasured run");
|
|
27168
|
+
}
|
|
27169
|
+
const wallClockReductionPct = (mean(acc.current.wallClockMs) - mean(acc.span.wallClockMs)) / mean(acc.current.wallClockMs) * 100;
|
|
27170
|
+
const judgeScoreDropPoints = mean(acc.current.judgeScores) - mean(acc.span.judgeScores);
|
|
27171
|
+
const verdict = evaluateSpanPhaseGate({
|
|
27172
|
+
wallClockReductionPct,
|
|
27173
|
+
judgeScoreDropPoints,
|
|
27174
|
+
fallbackRatePct: fallbackTally.fallbackRatePct
|
|
27175
|
+
});
|
|
27176
|
+
const comparison = {
|
|
27177
|
+
model: "deterministic-fake-provider (synthetic; no real model runs)",
|
|
27178
|
+
seed,
|
|
27179
|
+
conversations: conversations.length,
|
|
27180
|
+
perConversation: {
|
|
27181
|
+
judgeScoreCurrent: mean(acc.current.judgeScores),
|
|
27182
|
+
judgeScoreSpan: mean(acc.span.judgeScores),
|
|
27183
|
+
wallClockMsCurrent: mean(acc.current.wallClockMs),
|
|
27184
|
+
wallClockMsSpan: mean(acc.span.wallClockMs),
|
|
27185
|
+
outputTokensCurrent: mean(acc.current.outputTokens),
|
|
27186
|
+
outputTokensSpan: mean(acc.span.outputTokens),
|
|
27187
|
+
memoryEntriesCurrent: mean(acc.current.memoryEntries),
|
|
27188
|
+
memoryEntriesSpan: mean(acc.span.memoryEntries)
|
|
27189
|
+
},
|
|
27190
|
+
wallClockReductionPct,
|
|
27191
|
+
outputTokenReductionPct: (mean(acc.current.outputTokens) - mean(acc.span.outputTokens)) / mean(acc.current.outputTokens) * 100,
|
|
27192
|
+
judgeScoreDropPoints,
|
|
27193
|
+
spanAttempts: fallbackTally.attempts,
|
|
27194
|
+
spanFallbacks: fallbackTally.fallbacks,
|
|
27195
|
+
fallbackRatePct: fallbackTally.fallbackRatePct,
|
|
27196
|
+
costModel: {
|
|
27197
|
+
msPerOutputToken: MS_PER_OUTPUT_TOKEN,
|
|
27198
|
+
decodeBound: true,
|
|
27199
|
+
invalidSpanRate: INVALID_SPAN_RATE,
|
|
27200
|
+
driftSpanRate: DRIFT_SPAN_RATE
|
|
27201
|
+
},
|
|
27202
|
+
gate: {
|
|
27203
|
+
thresholds: {
|
|
27204
|
+
minWallClockReductionPct: 20,
|
|
27205
|
+
maxJudgeDropPoints: 2,
|
|
27206
|
+
maxFallbackRatePct: 15
|
|
27207
|
+
},
|
|
27208
|
+
verdict
|
|
27209
|
+
}
|
|
27210
|
+
};
|
|
27211
|
+
tasks.push({
|
|
27212
|
+
taskId: "span-phase-gate",
|
|
27213
|
+
question: "Does span-mode extraction clear the Phase B gate?",
|
|
27214
|
+
expected: "wall-clock -20%+, judge drop <2, fallback <15%",
|
|
27215
|
+
actual: verdict.pass ? "gate cleared" : `gate failed: ${verdict.failed.join(", ")}`,
|
|
27216
|
+
scores: {
|
|
27217
|
+
gate_pass: verdict.pass ? 1 : 0,
|
|
27218
|
+
wall_clock_reduction_pct: wallClockReductionPct,
|
|
27219
|
+
judge_score_drop_points: judgeScoreDropPoints,
|
|
27220
|
+
fallback_rate_pct: fallbackTally.fallbackRatePct
|
|
27221
|
+
},
|
|
27222
|
+
latencyMs: 0,
|
|
27223
|
+
tokens: { input: 0, output: 0 },
|
|
27224
|
+
goldMemories: [],
|
|
27225
|
+
details: { comparison }
|
|
27226
|
+
});
|
|
27227
|
+
const totalOutputTokens = acc.current.outputTokens.reduce((a, b) => a + b, 0) + acc.span.outputTokens.reduce((a, b) => a + b, 0);
|
|
27228
|
+
const totalWallClockMs = acc.current.wallClockMs.reduce((a, b) => a + b, 0) + acc.span.wallClockMs.reduce((a, b) => a + b, 0);
|
|
27229
|
+
const measuredTaskCount = acc.current.wallClockMs.length + acc.span.wallClockMs.length;
|
|
27230
|
+
return {
|
|
27231
|
+
meta: {
|
|
27232
|
+
id: randomUUID12(),
|
|
27233
|
+
benchmark: options.benchmark.id,
|
|
27234
|
+
benchmarkTier: options.benchmark.tier,
|
|
27235
|
+
version: options.benchmark.meta.version,
|
|
27236
|
+
remnicVersion: await getRemnicVersion(),
|
|
27237
|
+
gitSha: getGitSha(),
|
|
27238
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
27239
|
+
mode: options.mode,
|
|
27240
|
+
runCount: 1,
|
|
27241
|
+
seeds: [seed]
|
|
27242
|
+
},
|
|
27243
|
+
config: {
|
|
27244
|
+
systemProvider: options.systemProvider ?? null,
|
|
27245
|
+
judgeProvider: options.judgeProvider ?? null,
|
|
27246
|
+
adapterMode: options.adapterMode ?? "direct",
|
|
27247
|
+
remnicConfig: {
|
|
27248
|
+
spanBench: {
|
|
27249
|
+
provider: "deterministic-fake",
|
|
27250
|
+
datasets: ["locomo-synthetic", "longmemeval-synthetic"],
|
|
27251
|
+
seed
|
|
27252
|
+
}
|
|
27253
|
+
}
|
|
27254
|
+
},
|
|
27255
|
+
cost: {
|
|
27256
|
+
totalTokens: totalOutputTokens,
|
|
27257
|
+
inputTokens: 0,
|
|
27258
|
+
outputTokens: totalOutputTokens,
|
|
27259
|
+
estimatedCostUsd: 0,
|
|
27260
|
+
totalLatencyMs: totalWallClockMs,
|
|
27261
|
+
meanQueryLatencyMs: measuredTaskCount > 0 ? totalWallClockMs / measuredTaskCount : 0
|
|
27262
|
+
},
|
|
27263
|
+
results: {
|
|
27264
|
+
tasks,
|
|
27265
|
+
aggregates: aggregateTaskScores(tasks.map((task) => task.scores))
|
|
27266
|
+
},
|
|
27267
|
+
environment: {
|
|
27268
|
+
os: process.platform,
|
|
27269
|
+
nodeVersion: process.version,
|
|
27270
|
+
hardware: process.arch
|
|
27271
|
+
}
|
|
27272
|
+
};
|
|
27273
|
+
}
|
|
27274
|
+
function loadConversations(mode, limit) {
|
|
27275
|
+
const base = mode === "quick" ? SPAN_BENCH_SMOKE_FIXTURE : SPAN_BENCH_FIXTURE;
|
|
27276
|
+
if (limit === void 0) {
|
|
27277
|
+
return base;
|
|
27278
|
+
}
|
|
27279
|
+
if (!Number.isInteger(limit) || limit <= 0) {
|
|
27280
|
+
throw new Error("extraction-span-mode limit must be a positive integer");
|
|
27281
|
+
}
|
|
27282
|
+
const limited = base.slice(0, limit);
|
|
27283
|
+
if (limited.length === 0) {
|
|
27284
|
+
throw new Error("extraction-span-mode fixture is empty after applying the requested limit.");
|
|
27285
|
+
}
|
|
27286
|
+
return limited;
|
|
27287
|
+
}
|
|
27288
|
+
|
|
27289
|
+
// src/benchmarks/remnic/enrichment-fidelity/runner.ts
|
|
27290
|
+
import { randomUUID as randomUUID13 } from "crypto";
|
|
26100
27291
|
import { EnrichmentProviderRegistry, runEnrichmentPipeline } from "@remnic/core";
|
|
26101
27292
|
|
|
26102
27293
|
// src/benchmarks/remnic/enrichment-fidelity/fixture.ts
|
|
@@ -26372,7 +27563,7 @@ async function runEnrichmentFidelityBenchmark(options) {
|
|
|
26372
27563
|
);
|
|
26373
27564
|
return {
|
|
26374
27565
|
meta: {
|
|
26375
|
-
id:
|
|
27566
|
+
id: randomUUID13(),
|
|
26376
27567
|
benchmark: options.benchmark.id,
|
|
26377
27568
|
benchmarkTier: options.benchmark.tier,
|
|
26378
27569
|
version: options.benchmark.meta.version,
|
|
@@ -26444,7 +27635,7 @@ function constantAggregate2(value) {
|
|
|
26444
27635
|
// src/benchmarks/remnic/entity-consolidation/runner.ts
|
|
26445
27636
|
import os4 from "os";
|
|
26446
27637
|
import path17 from "path";
|
|
26447
|
-
import { randomUUID as
|
|
27638
|
+
import { randomUUID as randomUUID14 } from "crypto";
|
|
26448
27639
|
import { mkdtemp as mkdtemp4, rm as rm5 } from "fs/promises";
|
|
26449
27640
|
import { StorageManager as StorageManager2 } from "@remnic/core";
|
|
26450
27641
|
|
|
@@ -26555,7 +27746,7 @@ async function runEntityConsolidationBenchmark(options) {
|
|
|
26555
27746
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
26556
27747
|
return {
|
|
26557
27748
|
meta: {
|
|
26558
|
-
id:
|
|
27749
|
+
id: randomUUID14(),
|
|
26559
27750
|
benchmark: options.benchmark.id,
|
|
26560
27751
|
benchmarkTier: options.benchmark.tier,
|
|
26561
27752
|
version: options.benchmark.meta.version,
|
|
@@ -26784,7 +27975,7 @@ function parseNonNegativeInt(rawValue) {
|
|
|
26784
27975
|
}
|
|
26785
27976
|
|
|
26786
27977
|
// src/benchmarks/remnic/page-versioning/runner.ts
|
|
26787
|
-
import { randomUUID as
|
|
27978
|
+
import { randomUUID as randomUUID15 } from "crypto";
|
|
26788
27979
|
import { mkdir as mkdir5, mkdtemp as mkdtemp5, readFile as readFile16, rm as rm6, writeFile as writeFile5 } from "fs/promises";
|
|
26789
27980
|
import os5 from "os";
|
|
26790
27981
|
import path18 from "path";
|
|
@@ -26900,7 +28091,7 @@ async function runPageVersioningBenchmark(options, dependencyOverrides = {}) {
|
|
|
26900
28091
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
26901
28092
|
return {
|
|
26902
28093
|
meta: {
|
|
26903
|
-
id:
|
|
28094
|
+
id: randomUUID15(),
|
|
26904
28095
|
benchmark: options.benchmark.id,
|
|
26905
28096
|
benchmarkTier: options.benchmark.tier,
|
|
26906
28097
|
version: options.benchmark.meta.version,
|
|
@@ -27058,7 +28249,7 @@ function versioningConfig(overrides) {
|
|
|
27058
28249
|
}
|
|
27059
28250
|
|
|
27060
28251
|
// src/benchmarks/remnic/retrieval-personalization/runner.ts
|
|
27061
|
-
import { randomUUID as
|
|
28252
|
+
import { randomUUID as randomUUID16 } from "crypto";
|
|
27062
28253
|
|
|
27063
28254
|
// src/benchmarks/remnic/retrieval-page-ids.ts
|
|
27064
28255
|
function extractRankedPageIds(recallText, pages, options = {}) {
|
|
@@ -27745,7 +28936,7 @@ async function runRetrievalPersonalizationBenchmark(options) {
|
|
|
27745
28936
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
27746
28937
|
return {
|
|
27747
28938
|
meta: {
|
|
27748
|
-
id:
|
|
28939
|
+
id: randomUUID16(),
|
|
27749
28940
|
benchmark: options.benchmark.id,
|
|
27750
28941
|
benchmarkTier: options.benchmark.tier,
|
|
27751
28942
|
version: options.benchmark.meta.version,
|
|
@@ -27793,7 +28984,7 @@ function loadCases6(mode, limit) {
|
|
|
27793
28984
|
}
|
|
27794
28985
|
|
|
27795
28986
|
// src/benchmarks/remnic/retrieval-temporal/runner.ts
|
|
27796
|
-
import { randomUUID as
|
|
28987
|
+
import { randomUUID as randomUUID17 } from "crypto";
|
|
27797
28988
|
|
|
27798
28989
|
// src/benchmarks/remnic/retrieval-temporal/fixture.ts
|
|
27799
28990
|
var RETRIEVAL_TEMPORAL_FIXTURE = buildFixture(SCHEMA_TIER_FIXTURE);
|
|
@@ -27896,7 +29087,7 @@ async function runRetrievalTemporalBenchmark(options) {
|
|
|
27896
29087
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
27897
29088
|
return {
|
|
27898
29089
|
meta: {
|
|
27899
|
-
id:
|
|
29090
|
+
id: randomUUID17(),
|
|
27900
29091
|
benchmark: options.benchmark.id,
|
|
27901
29092
|
benchmarkTier: options.benchmark.tier,
|
|
27902
29093
|
version: options.benchmark.meta.version,
|
|
@@ -28031,7 +29222,7 @@ function matchingPageIds(rankedPageIds, sample) {
|
|
|
28031
29222
|
}
|
|
28032
29223
|
|
|
28033
29224
|
// src/benchmarks/remnic/retrieval-direct-answer/runner.ts
|
|
28034
|
-
import { randomUUID as
|
|
29225
|
+
import { randomUUID as randomUUID18 } from "crypto";
|
|
28035
29226
|
import { isDirectAnswerEligible } from "@remnic/core";
|
|
28036
29227
|
|
|
28037
29228
|
// src/benchmarks/remnic/retrieval-direct-answer/fixture.ts
|
|
@@ -28362,7 +29553,7 @@ async function runRetrievalDirectAnswerBenchmark(options) {
|
|
|
28362
29553
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
28363
29554
|
return {
|
|
28364
29555
|
meta: {
|
|
28365
|
-
id:
|
|
29556
|
+
id: randomUUID18(),
|
|
28366
29557
|
benchmark: options.benchmark.id,
|
|
28367
29558
|
benchmarkTier: options.benchmark.tier,
|
|
28368
29559
|
version: options.benchmark.meta.version,
|
|
@@ -28404,7 +29595,7 @@ async function runRetrievalDirectAnswerBenchmark(options) {
|
|
|
28404
29595
|
}
|
|
28405
29596
|
|
|
28406
29597
|
// src/benchmarks/remnic/retrieval-graph/runner.ts
|
|
28407
|
-
import { randomUUID as
|
|
29598
|
+
import { randomUUID as randomUUID19 } from "crypto";
|
|
28408
29599
|
import {
|
|
28409
29600
|
buildGraphFromMemories,
|
|
28410
29601
|
queryGraph
|
|
@@ -28586,7 +29777,7 @@ async function runRetrievalGraphBenchmark(options) {
|
|
|
28586
29777
|
const meanOff = tasks.length > 0 ? totalOff / tasks.length : 0;
|
|
28587
29778
|
return {
|
|
28588
29779
|
meta: {
|
|
28589
|
-
id:
|
|
29780
|
+
id: randomUUID19(),
|
|
28590
29781
|
benchmark: options.benchmark.id,
|
|
28591
29782
|
benchmarkTier: options.benchmark.tier,
|
|
28592
29783
|
version: options.benchmark.meta.version,
|
|
@@ -28652,7 +29843,7 @@ function loadCases8(mode, limit) {
|
|
|
28652
29843
|
}
|
|
28653
29844
|
|
|
28654
29845
|
// src/benchmarks/remnic/retrieval-reasoning-trace/runner.ts
|
|
28655
|
-
import { randomUUID as
|
|
29846
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
28656
29847
|
import {
|
|
28657
29848
|
applyReasoningTraceBoost,
|
|
28658
29849
|
isReasoningTracePath,
|
|
@@ -28920,7 +30111,7 @@ async function runRetrievalReasoningTraceBenchmark(options) {
|
|
|
28920
30111
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
28921
30112
|
return {
|
|
28922
30113
|
meta: {
|
|
28923
|
-
id:
|
|
30114
|
+
id: randomUUID20(),
|
|
28924
30115
|
benchmark: options.benchmark.id,
|
|
28925
30116
|
benchmarkTier: options.benchmark.tier,
|
|
28926
30117
|
version: options.benchmark.meta.version,
|
|
@@ -28962,7 +30153,7 @@ async function runRetrievalReasoningTraceBenchmark(options) {
|
|
|
28962
30153
|
}
|
|
28963
30154
|
|
|
28964
30155
|
// src/benchmarks/remnic/coding-recall/runner.ts
|
|
28965
|
-
import { randomUUID as
|
|
30156
|
+
import { randomUUID as randomUUID21 } from "crypto";
|
|
28966
30157
|
import { rankReviewCandidates } from "@remnic/core";
|
|
28967
30158
|
|
|
28968
30159
|
// src/benchmarks/remnic/coding-recall/fixture.ts
|
|
@@ -29241,7 +30432,7 @@ async function runCodingRecallBenchmark(options) {
|
|
|
29241
30432
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
29242
30433
|
return {
|
|
29243
30434
|
meta: {
|
|
29244
|
-
id:
|
|
30435
|
+
id: randomUUID21(),
|
|
29245
30436
|
benchmark: options.benchmark.id,
|
|
29246
30437
|
benchmarkTier: options.benchmark.tier,
|
|
29247
30438
|
version: options.benchmark.meta.version,
|
|
@@ -29307,7 +30498,7 @@ function loadCases9(mode, limit) {
|
|
|
29307
30498
|
}
|
|
29308
30499
|
|
|
29309
30500
|
// src/benchmarks/remnic/procedural-recall/runner.ts
|
|
29310
|
-
import { randomUUID as
|
|
30501
|
+
import { randomUUID as randomUUID22 } from "crypto";
|
|
29311
30502
|
import { mkdtemp as mkdtemp6, rm as rm7 } from "fs/promises";
|
|
29312
30503
|
import os6 from "os";
|
|
29313
30504
|
import path19 from "path";
|
|
@@ -29493,7 +30684,7 @@ ${body}`,
|
|
|
29493
30684
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
29494
30685
|
return {
|
|
29495
30686
|
meta: {
|
|
29496
|
-
id:
|
|
30687
|
+
id: randomUUID22(),
|
|
29497
30688
|
benchmark: options.benchmark.id,
|
|
29498
30689
|
benchmarkTier: options.benchmark.tier,
|
|
29499
30690
|
version: options.benchmark.meta.version,
|
|
@@ -29531,7 +30722,7 @@ ${body}`,
|
|
|
29531
30722
|
}
|
|
29532
30723
|
|
|
29533
30724
|
// src/benchmarks/remnic/ingestion-entity-recall/runner.ts
|
|
29534
|
-
import { randomUUID as
|
|
30725
|
+
import { randomUUID as randomUUID23 } from "crypto";
|
|
29535
30726
|
import { mkdtemp as mkdtemp7, writeFile as writeFile6, rm as rm8, mkdir as mkdir6, realpath as realpath5 } from "fs/promises";
|
|
29536
30727
|
import { tmpdir as tmpdir2 } from "os";
|
|
29537
30728
|
import path20 from "path";
|
|
@@ -30130,7 +31321,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
30130
31321
|
const remnicVersion = await getRemnicVersion();
|
|
30131
31322
|
return {
|
|
30132
31323
|
meta: {
|
|
30133
|
-
id:
|
|
31324
|
+
id: randomUUID23(),
|
|
30134
31325
|
benchmark: options.benchmark.id,
|
|
30135
31326
|
benchmarkTier: options.benchmark.tier,
|
|
30136
31327
|
version: options.benchmark.meta.version,
|
|
@@ -30168,7 +31359,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
30168
31359
|
}
|
|
30169
31360
|
|
|
30170
31361
|
// src/benchmarks/remnic/ingestion-schema-completeness/runner.ts
|
|
30171
|
-
import { randomUUID as
|
|
31362
|
+
import { randomUUID as randomUUID24 } from "crypto";
|
|
30172
31363
|
import { mkdtemp as mkdtemp8, writeFile as writeFile7, rm as rm9, mkdir as mkdir7, realpath as realpath6 } from "fs/promises";
|
|
30173
31364
|
import { tmpdir as tmpdir3 } from "os";
|
|
30174
31365
|
import path21 from "path";
|
|
@@ -30227,7 +31418,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
30227
31418
|
const remnicVersion2 = await getRemnicVersion();
|
|
30228
31419
|
return {
|
|
30229
31420
|
meta: {
|
|
30230
|
-
id:
|
|
31421
|
+
id: randomUUID24(),
|
|
30231
31422
|
benchmark: options.benchmark.id,
|
|
30232
31423
|
benchmarkTier: options.benchmark.tier,
|
|
30233
31424
|
version: options.benchmark.meta.version,
|
|
@@ -30300,7 +31491,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
30300
31491
|
const remnicVersion = await getRemnicVersion();
|
|
30301
31492
|
return {
|
|
30302
31493
|
meta: {
|
|
30303
|
-
id:
|
|
31494
|
+
id: randomUUID24(),
|
|
30304
31495
|
benchmark: options.benchmark.id,
|
|
30305
31496
|
benchmarkTier: options.benchmark.tier,
|
|
30306
31497
|
version: options.benchmark.meta.version,
|
|
@@ -30341,7 +31532,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
30341
31532
|
}
|
|
30342
31533
|
|
|
30343
31534
|
// src/benchmarks/remnic/ingestion-backlink-f1/runner.ts
|
|
30344
|
-
import { randomUUID as
|
|
31535
|
+
import { randomUUID as randomUUID25 } from "crypto";
|
|
30345
31536
|
import { mkdtemp as mkdtemp9, writeFile as writeFile8, rm as rm10, mkdir as mkdir8, realpath as realpath7 } from "fs/promises";
|
|
30346
31537
|
import { tmpdir as tmpdir4 } from "os";
|
|
30347
31538
|
import path22 from "path";
|
|
@@ -30401,7 +31592,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
30401
31592
|
const remnicVersion = await getRemnicVersion();
|
|
30402
31593
|
return {
|
|
30403
31594
|
meta: {
|
|
30404
|
-
id:
|
|
31595
|
+
id: randomUUID25(),
|
|
30405
31596
|
benchmark: options.benchmark.id,
|
|
30406
31597
|
benchmarkTier: options.benchmark.tier,
|
|
30407
31598
|
version: options.benchmark.meta.version,
|
|
@@ -30442,7 +31633,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
30442
31633
|
}
|
|
30443
31634
|
|
|
30444
31635
|
// src/benchmarks/remnic/ingestion-setup-friction/runner.ts
|
|
30445
|
-
import { randomUUID as
|
|
31636
|
+
import { randomUUID as randomUUID26 } from "crypto";
|
|
30446
31637
|
import { mkdtemp as mkdtemp10, writeFile as writeFile9, rm as rm11, mkdir as mkdir9, realpath as realpath8 } from "fs/promises";
|
|
30447
31638
|
import { tmpdir as tmpdir5 } from "os";
|
|
30448
31639
|
import path23 from "path";
|
|
@@ -30507,7 +31698,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
30507
31698
|
const remnicVersion = await getRemnicVersion();
|
|
30508
31699
|
return {
|
|
30509
31700
|
meta: {
|
|
30510
|
-
id:
|
|
31701
|
+
id: randomUUID26(),
|
|
30511
31702
|
benchmark: options.benchmark.id,
|
|
30512
31703
|
benchmarkTier: options.benchmark.tier,
|
|
30513
31704
|
version: options.benchmark.meta.version,
|
|
@@ -30548,7 +31739,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
30548
31739
|
}
|
|
30549
31740
|
|
|
30550
31741
|
// src/benchmarks/remnic/ingestion-citation-accuracy/runner.ts
|
|
30551
|
-
import { randomUUID as
|
|
31742
|
+
import { randomUUID as randomUUID27 } from "crypto";
|
|
30552
31743
|
import { mkdtemp as mkdtemp11, writeFile as writeFile10, rm as rm12, mkdir as mkdir10, realpath as realpath9 } from "fs/promises";
|
|
30553
31744
|
import { tmpdir as tmpdir6 } from "os";
|
|
30554
31745
|
import path24 from "path";
|
|
@@ -30691,7 +31882,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
30691
31882
|
const remnicVersion2 = await getRemnicVersion();
|
|
30692
31883
|
return {
|
|
30693
31884
|
meta: {
|
|
30694
|
-
id:
|
|
31885
|
+
id: randomUUID27(),
|
|
30695
31886
|
benchmark: options.benchmark.id,
|
|
30696
31887
|
benchmarkTier: options.benchmark.tier,
|
|
30697
31888
|
version: options.benchmark.meta.version,
|
|
@@ -30809,7 +32000,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
30809
32000
|
const remnicVersion = await getRemnicVersion();
|
|
30810
32001
|
return {
|
|
30811
32002
|
meta: {
|
|
30812
|
-
id:
|
|
32003
|
+
id: randomUUID27(),
|
|
30813
32004
|
benchmark: options.benchmark.id,
|
|
30814
32005
|
benchmarkTier: options.benchmark.tier,
|
|
30815
32006
|
version: options.benchmark.meta.version,
|
|
@@ -31032,7 +32223,7 @@ var ASSISTANT_MORNING_BRIEF_SCENARIOS = [
|
|
|
31032
32223
|
var ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS = ASSISTANT_MORNING_BRIEF_SCENARIOS.slice(0, 2);
|
|
31033
32224
|
|
|
31034
32225
|
// src/benchmarks/remnic/_assistant-common/runner.ts
|
|
31035
|
-
import { randomUUID as
|
|
32226
|
+
import { randomUUID as randomUUID28 } from "crypto";
|
|
31036
32227
|
import path26 from "path";
|
|
31037
32228
|
|
|
31038
32229
|
// src/run-seeds.ts
|
|
@@ -31054,7 +32245,7 @@ function buildBenchmarkRunSeeds(runCount, baseSeed) {
|
|
|
31054
32245
|
// src/stats/bootstrap.ts
|
|
31055
32246
|
var DEFAULT_ITERATIONS = 1e3;
|
|
31056
32247
|
var DEFAULT_LEVEL = 0.95;
|
|
31057
|
-
function
|
|
32248
|
+
function mean2(values) {
|
|
31058
32249
|
if (values.length === 0) {
|
|
31059
32250
|
throw new Error("bootstrap requires at least one value");
|
|
31060
32251
|
}
|
|
@@ -31093,7 +32284,7 @@ function createBootstrapMeans(values, {
|
|
|
31093
32284
|
const pickedIndex = Math.floor(random() * values.length);
|
|
31094
32285
|
sample.push(values[pickedIndex]);
|
|
31095
32286
|
}
|
|
31096
|
-
samples.push(
|
|
32287
|
+
samples.push(mean2(sample));
|
|
31097
32288
|
}
|
|
31098
32289
|
return samples;
|
|
31099
32290
|
}
|
|
@@ -31546,7 +32737,7 @@ async function runAssistantBenchmark(definition, scenarios, resolved, runnerOpti
|
|
|
31546
32737
|
const totalSeedExecutions = tasks.length * runCount;
|
|
31547
32738
|
return {
|
|
31548
32739
|
meta: {
|
|
31549
|
-
id:
|
|
32740
|
+
id: randomUUID28(),
|
|
31550
32741
|
benchmark: definition.id,
|
|
31551
32742
|
benchmarkTier: definition.tier,
|
|
31552
32743
|
version: definition.meta.version,
|
|
@@ -32142,7 +33333,7 @@ async function runAssistantSynthesisBenchmark(options) {
|
|
|
32142
33333
|
}
|
|
32143
33334
|
|
|
32144
33335
|
// src/benchmarks/remnic/buffer-surprise-trigger/runner.ts
|
|
32145
|
-
import { randomUUID as
|
|
33336
|
+
import { randomUUID as randomUUID29 } from "crypto";
|
|
32146
33337
|
import path27 from "path";
|
|
32147
33338
|
import os7 from "os";
|
|
32148
33339
|
import { mkdir as mkdir11, rm as rm13 } from "fs/promises";
|
|
@@ -32376,7 +33567,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
32376
33567
|
const cases = loadCases10(options.mode, options.limit);
|
|
32377
33568
|
const tmpRoot = path27.join(
|
|
32378
33569
|
os7.tmpdir(),
|
|
32379
|
-
`remnic-bench-buffer-surprise-${
|
|
33570
|
+
`remnic-bench-buffer-surprise-${randomUUID29()}`
|
|
32380
33571
|
);
|
|
32381
33572
|
await mkdir11(tmpRoot, { recursive: true });
|
|
32382
33573
|
const tasks = [];
|
|
@@ -32403,7 +33594,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
32403
33594
|
const remnicVersion = await getRemnicVersion();
|
|
32404
33595
|
return {
|
|
32405
33596
|
meta: {
|
|
32406
|
-
id:
|
|
33597
|
+
id: randomUUID29(),
|
|
32407
33598
|
benchmark: options.benchmark.id,
|
|
32408
33599
|
benchmarkTier: options.benchmark.tier,
|
|
32409
33600
|
version: options.benchmark.meta.version,
|
|
@@ -32516,10 +33707,10 @@ async function runSingleCase(caseDef, options) {
|
|
|
32516
33707
|
for (let i = 1; i < flushTurnIndices.length; i += 1) {
|
|
32517
33708
|
turnsBetween.push(flushTurnIndices[i] - flushTurnIndices[i - 1]);
|
|
32518
33709
|
}
|
|
32519
|
-
const
|
|
33710
|
+
const mean5 = turnsBetween.length > 0 ? turnsBetween.reduce((acc, v) => acc + v, 0) / turnsBetween.length : 0;
|
|
32520
33711
|
return {
|
|
32521
33712
|
flushTurnIndices,
|
|
32522
|
-
turnsBetweenFlushesMean:
|
|
33713
|
+
turnsBetweenFlushesMean: mean5,
|
|
32523
33714
|
replayLatencyMs
|
|
32524
33715
|
};
|
|
32525
33716
|
}
|
|
@@ -32630,7 +33821,7 @@ function loadCases10(mode, limit) {
|
|
|
32630
33821
|
}
|
|
32631
33822
|
|
|
32632
33823
|
// src/benchmarks/remnic/contradiction-detection/runner.ts
|
|
32633
|
-
import { randomUUID as
|
|
33824
|
+
import { randomUUID as randomUUID30 } from "crypto";
|
|
32634
33825
|
|
|
32635
33826
|
// src/benchmarks/remnic/contradiction-detection/fixture.ts
|
|
32636
33827
|
var TRUE_CONTRADICTIONS = [
|
|
@@ -32957,7 +34148,7 @@ async function runContradictionDetectionBenchmark(options) {
|
|
|
32957
34148
|
const meanQueryLatencyMs = tasks.length > 0 ? totalLatencyMs / tasks.length : 0;
|
|
32958
34149
|
return {
|
|
32959
34150
|
meta: {
|
|
32960
|
-
id:
|
|
34151
|
+
id: randomUUID30(),
|
|
32961
34152
|
benchmark: options.benchmark.id,
|
|
32962
34153
|
benchmarkTier: options.benchmark.tier,
|
|
32963
34154
|
version: options.benchmark.meta.version,
|
|
@@ -33012,7 +34203,7 @@ function loadCases11(mode, limit) {
|
|
|
33012
34203
|
}
|
|
33013
34204
|
|
|
33014
34205
|
// src/benchmarks/remnic/retention-aged-dataset/runner.ts
|
|
33015
|
-
import { randomUUID as
|
|
34206
|
+
import { randomUUID as randomUUID31 } from "crypto";
|
|
33016
34207
|
import {
|
|
33017
34208
|
decideTierTransition
|
|
33018
34209
|
} from "@remnic/core";
|
|
@@ -33388,7 +34579,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
33388
34579
|
const totalLatencyMs = tasks.reduce((sum, t) => sum + t.latencyMs, 0);
|
|
33389
34580
|
return {
|
|
33390
34581
|
meta: {
|
|
33391
|
-
id:
|
|
34582
|
+
id: randomUUID31(),
|
|
33392
34583
|
benchmark: options.benchmark.id,
|
|
33393
34584
|
benchmarkTier: options.benchmark.tier,
|
|
33394
34585
|
version: options.benchmark.meta.version,
|
|
@@ -33432,7 +34623,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
33432
34623
|
}
|
|
33433
34624
|
|
|
33434
34625
|
// src/benchmarks/remnic/memcorrect/runner.ts
|
|
33435
|
-
import { randomUUID as
|
|
34626
|
+
import { randomUUID as randomUUID32 } from "crypto";
|
|
33436
34627
|
|
|
33437
34628
|
// src/benchmarks/remnic/memcorrect/generator.ts
|
|
33438
34629
|
import { createHash as createHash11 } from "crypto";
|
|
@@ -34043,8 +35234,8 @@ function nonResurrection(log, corrections) {
|
|
|
34043
35234
|
}
|
|
34044
35235
|
function collateralDelta(before, after) {
|
|
34045
35236
|
if (before.length === 0) return 0;
|
|
34046
|
-
const
|
|
34047
|
-
return
|
|
35237
|
+
const mean5 = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
35238
|
+
return mean5(after) - mean5(before);
|
|
34048
35239
|
}
|
|
34049
35240
|
function scopePrecision(log, corrections) {
|
|
34050
35241
|
let scopedCount = 0;
|
|
@@ -34645,7 +35836,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
34645
35836
|
const { adapter: _liveAdapter, ...persistableBenchmarkOptions } = options.benchmarkOptions ?? {};
|
|
34646
35837
|
return {
|
|
34647
35838
|
meta: {
|
|
34648
|
-
id:
|
|
35839
|
+
id: randomUUID32(),
|
|
34649
35840
|
benchmark: options.benchmark.id,
|
|
34650
35841
|
benchmarkTier: options.benchmark.tier,
|
|
34651
35842
|
version: options.benchmark.meta.version,
|
|
@@ -34711,7 +35902,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
34711
35902
|
}
|
|
34712
35903
|
|
|
34713
35904
|
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
34714
|
-
import { randomUUID as
|
|
35905
|
+
import { randomUUID as randomUUID33 } from "crypto";
|
|
34715
35906
|
import { mkdir as mkdir12, writeFile as writeFile11 } from "fs/promises";
|
|
34716
35907
|
import path28 from "path";
|
|
34717
35908
|
|
|
@@ -35556,39 +36747,39 @@ function scoreTaskPair(task, pack, decision) {
|
|
|
35556
36747
|
compression_ratio_vs_raw_transcript: compressionRatio
|
|
35557
36748
|
};
|
|
35558
36749
|
}
|
|
35559
|
-
function
|
|
36750
|
+
function mean3(values) {
|
|
35560
36751
|
if (values.length === 0) return 0;
|
|
35561
36752
|
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
35562
36753
|
}
|
|
35563
36754
|
function aggregateCondition(condition, scored, skillLog) {
|
|
35564
36755
|
const taskCount = scored.length;
|
|
35565
36756
|
const all = scored.map((s) => s.scores);
|
|
35566
|
-
const taskSuccessRate =
|
|
36757
|
+
const taskSuccessRate = mean3(all.map((s) => s.task_success));
|
|
35567
36758
|
const boundaryTasks = scored.filter((s) => s.task.shouldAsk !== void 0);
|
|
35568
36759
|
const askNeeded = scored.filter((s) => s.task.shouldAsk === true);
|
|
35569
36760
|
const actWhenEnough = scored.filter((s) => s.task.shouldAsk === false);
|
|
35570
|
-
const shouldAskAccuracy =
|
|
35571
|
-
const unnecessaryClarificationRate =
|
|
36761
|
+
const shouldAskAccuracy = mean3(boundaryTasks.map((s) => s.scores.should_ask_accuracy));
|
|
36762
|
+
const unnecessaryClarificationRate = mean3(
|
|
35572
36763
|
actWhenEnough.map((s) => s.scores.unnecessary_clarification_rate)
|
|
35573
36764
|
);
|
|
35574
|
-
const actionBoundaryViolationRate =
|
|
36765
|
+
const actionBoundaryViolationRate = mean3(
|
|
35575
36766
|
askNeeded.map((s) => s.scores.action_boundary_violation_rate)
|
|
35576
36767
|
);
|
|
35577
36768
|
const recallTasks = scored.filter(
|
|
35578
36769
|
(s) => (s.task.shouldRecallId !== void 0 || s.task.family === "skill-positive") && s.task.family !== "stale-memory-trap" && s.task.family !== "wrong-scope-trap"
|
|
35579
36770
|
);
|
|
35580
|
-
const relevantMemoryRecall =
|
|
36771
|
+
const relevantMemoryRecall = mean3(recallTasks.map((s) => s.scores.relevant_memory_recall));
|
|
35581
36772
|
const staleTasks = scored.filter((s) => s.task.family === "stale-memory-trap");
|
|
35582
36773
|
const scopeTasks = scored.filter((s) => s.task.family === "wrong-scope-trap");
|
|
35583
|
-
const staleMemoryHarmRate =
|
|
35584
|
-
const wrongScopeRetrievalRate =
|
|
35585
|
-
const supersessionRespectedRate =
|
|
36774
|
+
const staleMemoryHarmRate = mean3(staleTasks.map((s) => s.scores.stale_memory_harm_rate));
|
|
36775
|
+
const wrongScopeRetrievalRate = mean3(scopeTasks.map((s) => s.scores.wrong_scope_retrieval_rate));
|
|
36776
|
+
const supersessionRespectedRate = mean3(staleTasks.map((s) => s.scores.supersession_respected_rate));
|
|
35586
36777
|
const citedTasks = scored.filter((s) => s.scores.retrieved_item_count > 0);
|
|
35587
|
-
const citationCoverage =
|
|
35588
|
-
const meanMemoryTokensInjected =
|
|
35589
|
-
const meanRetrievedItemCount =
|
|
36778
|
+
const citationCoverage = mean3(citedTasks.map((s) => s.scores.citation_coverage));
|
|
36779
|
+
const meanMemoryTokensInjected = mean3(all.map((s) => s.memory_tokens_injected));
|
|
36780
|
+
const meanRetrievedItemCount = mean3(all.map((s) => s.retrieved_item_count));
|
|
35590
36781
|
const compressible = scored.filter((s) => s.task.memoryItems.length > 0);
|
|
35591
|
-
const meanCompressionRatio =
|
|
36782
|
+
const meanCompressionRatio = mean3(compressible.map((s) => s.scores.compression_ratio_vs_raw_transcript));
|
|
35592
36783
|
const considered = skillLog.filter((e) => e.considered);
|
|
35593
36784
|
const injected = considered.filter((e) => e.injected);
|
|
35594
36785
|
const tp = injected.filter((e) => e.outcome === "helped").length;
|
|
@@ -35893,7 +37084,7 @@ async function runBoundedMemoryContractsBenchmark(options) {
|
|
|
35893
37084
|
const skillTriggerLog = c3SkillLog;
|
|
35894
37085
|
return {
|
|
35895
37086
|
meta: {
|
|
35896
|
-
id:
|
|
37087
|
+
id: randomUUID33(),
|
|
35897
37088
|
benchmark: options.benchmark.id,
|
|
35898
37089
|
benchmarkTier: options.benchmark.tier,
|
|
35899
37090
|
version: options.benchmark.meta.version,
|
|
@@ -36069,7 +37260,7 @@ function renderPromptPack(task, condition, pack) {
|
|
|
36069
37260
|
}
|
|
36070
37261
|
|
|
36071
37262
|
// src/benchmarks/remnic/staged-memory/runner.ts
|
|
36072
|
-
import { createHash as createHash16, randomUUID as
|
|
37263
|
+
import { createHash as createHash16, randomUUID as randomUUID34 } from "crypto";
|
|
36073
37264
|
|
|
36074
37265
|
// src/benchmarks/remnic/staged-memory/fixture.ts
|
|
36075
37266
|
import { createHash as createHash15 } from "crypto";
|
|
@@ -37680,7 +38871,7 @@ async function runDriftGenCliCommand(options) {
|
|
|
37680
38871
|
}
|
|
37681
38872
|
|
|
37682
38873
|
// src/benchmarks/remnic/staged-memory/schema.ts
|
|
37683
|
-
import { z } from "zod";
|
|
38874
|
+
import { z as z2 } from "zod";
|
|
37684
38875
|
var STAGED_MEMORY_BENCHMARK_ID = "staged-memory-synthetic-v1";
|
|
37685
38876
|
var STAGED_MEMORY_FIXTURE_NAME = "staged-memory-synthetic";
|
|
37686
38877
|
var STAGED_MEMORY_GENERATOR_VERSION = "1.0.0";
|
|
@@ -37694,38 +38885,38 @@ var STAGED_MEMORY_ARMS = Object.freeze([
|
|
|
37694
38885
|
"staged-memory",
|
|
37695
38886
|
"oracle-retrieval"
|
|
37696
38887
|
]);
|
|
37697
|
-
var hex64 =
|
|
37698
|
-
var nonEmpty =
|
|
37699
|
-
var positiveEpoch =
|
|
37700
|
-
var StagedMemoryFixtureManifestV1Schema =
|
|
37701
|
-
schemaVersion:
|
|
37702
|
-
name:
|
|
38888
|
+
var hex64 = z2.string().regex(/^[0-9a-f]{64}$/, "expected a sha256 hex digest");
|
|
38889
|
+
var nonEmpty = z2.string().min(1);
|
|
38890
|
+
var positiveEpoch = z2.number().int().positive();
|
|
38891
|
+
var StagedMemoryFixtureManifestV1Schema = z2.object({
|
|
38892
|
+
schemaVersion: z2.literal(1),
|
|
38893
|
+
name: z2.literal("staged-memory-synthetic"),
|
|
37703
38894
|
version: nonEmpty,
|
|
37704
38895
|
generatorVersion: nonEmpty,
|
|
37705
|
-
seeds:
|
|
37706
|
-
source:
|
|
37707
|
-
kind:
|
|
38896
|
+
seeds: z2.array(z2.number().int().nonnegative()).min(1),
|
|
38897
|
+
source: z2.object({
|
|
38898
|
+
kind: z2.literal("drift-gen"),
|
|
37708
38899
|
manifestName: nonEmpty,
|
|
37709
38900
|
manifestSha256: hex64
|
|
37710
38901
|
}).strict(),
|
|
37711
|
-
counts:
|
|
37712
|
-
users:
|
|
37713
|
-
cases:
|
|
37714
|
-
distractors:
|
|
38902
|
+
counts: z2.object({
|
|
38903
|
+
users: z2.number().int().nonnegative(),
|
|
38904
|
+
cases: z2.number().int().nonnegative(),
|
|
38905
|
+
distractors: z2.number().int().nonnegative()
|
|
37715
38906
|
}).strict(),
|
|
37716
|
-
files:
|
|
37717
|
-
createdAt:
|
|
37718
|
-
licenses:
|
|
37719
|
-
namespaces:
|
|
38907
|
+
files: z2.record(hex64),
|
|
38908
|
+
createdAt: z2.literal("1970-01-01T00:00:00.000Z"),
|
|
38909
|
+
licenses: z2.array(z2.object({ source: nonEmpty, license: nonEmpty }).strict()).min(1),
|
|
38910
|
+
namespaces: z2.array(nonEmpty).min(2)
|
|
37720
38911
|
}).strict();
|
|
37721
|
-
var StagedMemoryDistractorV1Schema =
|
|
38912
|
+
var StagedMemoryDistractorV1Schema = z2.object({
|
|
37722
38913
|
id: nonEmpty,
|
|
37723
38914
|
sessionId: nonEmpty,
|
|
37724
38915
|
text: nonEmpty,
|
|
37725
|
-
forbiddenFactIds:
|
|
38916
|
+
forbiddenFactIds: z2.array(nonEmpty).min(1),
|
|
37726
38917
|
templateId: nonEmpty
|
|
37727
38918
|
}).strict();
|
|
37728
|
-
var StagedMemoryGoldFactV1Schema =
|
|
38919
|
+
var StagedMemoryGoldFactV1Schema = z2.object({
|
|
37729
38920
|
factId: nonEmpty,
|
|
37730
38921
|
subject: nonEmpty,
|
|
37731
38922
|
attribute: nonEmpty,
|
|
@@ -37733,41 +38924,41 @@ var StagedMemoryGoldFactV1Schema = z.object({
|
|
|
37733
38924
|
statement: nonEmpty,
|
|
37734
38925
|
introducedEpoch: positiveEpoch
|
|
37735
38926
|
}).strict();
|
|
37736
|
-
var StagedMemoryCaseV1Schema =
|
|
37737
|
-
schemaVersion:
|
|
38927
|
+
var StagedMemoryCaseV1Schema = z2.object({
|
|
38928
|
+
schemaVersion: z2.literal(1),
|
|
37738
38929
|
caseId: nonEmpty,
|
|
37739
38930
|
userId: nonEmpty,
|
|
37740
38931
|
namespace: nonEmpty,
|
|
37741
|
-
seed:
|
|
37742
|
-
exposure:
|
|
38932
|
+
seed: z2.number().int().nonnegative(),
|
|
38933
|
+
exposure: z2.object({
|
|
37743
38934
|
sessionId: nonEmpty,
|
|
37744
|
-
sourceSessionRefs:
|
|
38935
|
+
sourceSessionRefs: z2.array(nonEmpty).min(1),
|
|
37745
38936
|
/** Current (non-superseded) fact IDs at the exposure epoch. */
|
|
37746
|
-
salientFactIds:
|
|
37747
|
-
goldFacts:
|
|
38937
|
+
salientFactIds: z2.array(nonEmpty).min(1),
|
|
38938
|
+
goldFacts: z2.array(StagedMemoryGoldFactV1Schema).min(1),
|
|
37748
38939
|
/** Statements of `goldFacts`, in the same order. */
|
|
37749
|
-
goldMemories:
|
|
38940
|
+
goldMemories: z2.array(nonEmpty).min(1),
|
|
37750
38941
|
exposureEpoch: positiveEpoch,
|
|
37751
38942
|
/** Pinned effective timestamp for transition scoring; never wall clock. */
|
|
37752
38943
|
effectiveTimestamp: nonEmpty
|
|
37753
38944
|
}).strict(),
|
|
37754
|
-
transitions:
|
|
37755
|
-
|
|
38945
|
+
transitions: z2.array(
|
|
38946
|
+
z2.object({
|
|
37756
38947
|
oldFactId: nonEmpty,
|
|
37757
38948
|
newFactId: nonEmpty,
|
|
37758
38949
|
epoch: positiveEpoch,
|
|
37759
|
-
kind:
|
|
38950
|
+
kind: z2.enum(["drifting", "contradicted"])
|
|
37760
38951
|
}).strict()
|
|
37761
38952
|
),
|
|
37762
|
-
distractors:
|
|
37763
|
-
task:
|
|
38953
|
+
distractors: z2.array(StagedMemoryDistractorV1Schema),
|
|
38954
|
+
task: z2.object({
|
|
37764
38955
|
question: nonEmpty,
|
|
37765
38956
|
expectedAnswer: nonEmpty,
|
|
37766
|
-
requiredFactIds:
|
|
37767
|
-
forbiddenFactIds:
|
|
37768
|
-
answerFormat:
|
|
38957
|
+
requiredFactIds: z2.array(nonEmpty).min(1),
|
|
38958
|
+
forbiddenFactIds: z2.array(nonEmpty),
|
|
38959
|
+
answerFormat: z2.literal("exact")
|
|
37769
38960
|
}).strict(),
|
|
37770
|
-
scope:
|
|
38961
|
+
scope: z2.object({
|
|
37771
38962
|
principal: nonEmpty,
|
|
37772
38963
|
allowedUserId: nonEmpty,
|
|
37773
38964
|
allowedNamespace: nonEmpty
|
|
@@ -38991,7 +40182,7 @@ async function runStagedMemoryBenchmark(options) {
|
|
|
38991
40182
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
38992
40183
|
const result = {
|
|
38993
40184
|
meta: {
|
|
38994
|
-
id:
|
|
40185
|
+
id: randomUUID34(),
|
|
38995
40186
|
benchmark: options.benchmark.id,
|
|
38996
40187
|
benchmarkTier: options.benchmark.tier,
|
|
38997
40188
|
version: options.benchmark.meta.version,
|
|
@@ -39096,6 +40287,10 @@ var REGISTERED_BENCHMARKS = [
|
|
|
39096
40287
|
...extractionJudgeCalibrationDefinition,
|
|
39097
40288
|
run: runExtractionJudgeCalibrationBenchmark
|
|
39098
40289
|
},
|
|
40290
|
+
{
|
|
40291
|
+
...extractionSpanModeDefinition,
|
|
40292
|
+
run: runExtractionSpanModeBenchmark
|
|
40293
|
+
},
|
|
39099
40294
|
{
|
|
39100
40295
|
...enrichmentFidelityDefinition,
|
|
39101
40296
|
run: runEnrichmentFidelityBenchmark
|
|
@@ -39750,7 +40945,7 @@ function generateReport(results, reportPath) {
|
|
|
39750
40945
|
}
|
|
39751
40946
|
|
|
39752
40947
|
// src/stats/effect-size.ts
|
|
39753
|
-
function
|
|
40948
|
+
function mean4(values) {
|
|
39754
40949
|
if (values.length === 0) {
|
|
39755
40950
|
throw new Error("effect size requires at least one value");
|
|
39756
40951
|
}
|
|
@@ -39766,8 +40961,8 @@ function cohensD(candidateValues, baselineValues) {
|
|
|
39766
40961
|
if (candidateValues.length === 0 || baselineValues.length === 0) {
|
|
39767
40962
|
throw new Error("effect size requires non-empty candidate and baseline arrays");
|
|
39768
40963
|
}
|
|
39769
|
-
const candidateMean =
|
|
39770
|
-
const baselineMean =
|
|
40964
|
+
const candidateMean = mean4(candidateValues);
|
|
40965
|
+
const baselineMean = mean4(baselineValues);
|
|
39771
40966
|
const candidateVariance = sampleVariance(candidateValues, candidateMean);
|
|
39772
40967
|
const baselineVariance = sampleVariance(baselineValues, baselineMean);
|
|
39773
40968
|
const pooledDegreesOfFreedom = candidateValues.length + baselineValues.length - 2;
|
|
@@ -41871,60 +43066,6 @@ function serializeSealedQrels(artifact) {
|
|
|
41871
43066
|
return canonicalJsonStringify(normalized);
|
|
41872
43067
|
}
|
|
41873
43068
|
|
|
41874
|
-
// src/integrity/canary-adapter.ts
|
|
41875
|
-
var CANARY_FIXED_RECALL = "__remnic_canary_response__";
|
|
41876
|
-
var CANARY_SCORE_FLOOR = 0.1;
|
|
41877
|
-
function createCanaryAdapter(options = {}) {
|
|
41878
|
-
const response = options.response ?? CANARY_FIXED_RECALL;
|
|
41879
|
-
const emptySearch = options.emptySearch ?? false;
|
|
41880
|
-
return {
|
|
41881
|
-
async store(_sessionId, _messages) {
|
|
41882
|
-
},
|
|
41883
|
-
async recall(_sessionId, _query, _budgetChars) {
|
|
41884
|
-
return response;
|
|
41885
|
-
},
|
|
41886
|
-
async search(_query, _limit, _sessionId) {
|
|
41887
|
-
if (emptySearch) {
|
|
41888
|
-
return [];
|
|
41889
|
-
}
|
|
41890
|
-
return [
|
|
41891
|
-
{
|
|
41892
|
-
turnIndex: 0,
|
|
41893
|
-
role: "assistant",
|
|
41894
|
-
snippet: response,
|
|
41895
|
-
sessionId: "__canary__",
|
|
41896
|
-
score: 0
|
|
41897
|
-
}
|
|
41898
|
-
];
|
|
41899
|
-
},
|
|
41900
|
-
async reset(_sessionId) {
|
|
41901
|
-
},
|
|
41902
|
-
async getStats(_sessionId) {
|
|
41903
|
-
return {
|
|
41904
|
-
totalMessages: 0,
|
|
41905
|
-
totalSummaryNodes: 0,
|
|
41906
|
-
maxDepth: 0
|
|
41907
|
-
};
|
|
41908
|
-
},
|
|
41909
|
-
async destroy() {
|
|
41910
|
-
}
|
|
41911
|
-
};
|
|
41912
|
-
}
|
|
41913
|
-
function assertCanaryUnderFloor(benchmark, score, floor = CANARY_SCORE_FLOOR) {
|
|
41914
|
-
if (!Number.isFinite(floor) || floor < 0) {
|
|
41915
|
-
throw new Error(`Canary floor must be a non-negative finite number; got ${floor}.`);
|
|
41916
|
-
}
|
|
41917
|
-
if (!Number.isFinite(score)) {
|
|
41918
|
-
return { benchmark, score, floor, passed: false };
|
|
41919
|
-
}
|
|
41920
|
-
return {
|
|
41921
|
-
benchmark,
|
|
41922
|
-
score,
|
|
41923
|
-
floor,
|
|
41924
|
-
passed: score <= floor
|
|
41925
|
-
};
|
|
41926
|
-
}
|
|
41927
|
-
|
|
41928
43069
|
// src/integrity/randomize.ts
|
|
41929
43070
|
function createSeededRng(seed) {
|
|
41930
43071
|
if (!Number.isFinite(seed)) {
|
|
@@ -42316,7 +43457,7 @@ function formatError(error) {
|
|
|
42316
43457
|
}
|
|
42317
43458
|
|
|
42318
43459
|
// src/benchmarks/custom/runner.ts
|
|
42319
|
-
import { randomUUID as
|
|
43460
|
+
import { randomUUID as randomUUID35 } from "crypto";
|
|
42320
43461
|
import path34 from "path";
|
|
42321
43462
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
42322
43463
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
@@ -42404,7 +43545,7 @@ async function runCustomBenchmark(spec, options) {
|
|
|
42404
43545
|
const totalOutputTokens = tasks.reduce((sum, task) => sum + task.tokens.output, 0);
|
|
42405
43546
|
return finalizeBenchmarkResultConfig({
|
|
42406
43547
|
meta: {
|
|
42407
|
-
id:
|
|
43548
|
+
id: randomUUID35(),
|
|
42408
43549
|
benchmark: options.benchmark.id,
|
|
42409
43550
|
benchmarkTier: options.benchmark.tier,
|
|
42410
43551
|
version: options.benchmark.meta.version,
|
|
@@ -45286,10 +46427,10 @@ import path39 from "path";
|
|
|
45286
46427
|
import { hostname } from "os";
|
|
45287
46428
|
import { mkdir as mkdir18, readFile as readFile24, rename as rename7, rm as rm17, stat as stat4, utimes, writeFile as writeFile17 } from "fs/promises";
|
|
45288
46429
|
import path38 from "path";
|
|
45289
|
-
import { randomUUID as
|
|
46430
|
+
import { randomUUID as randomUUID37 } from "crypto";
|
|
45290
46431
|
|
|
45291
46432
|
// src/security/injection-suite/store.ts
|
|
45292
|
-
import { createHash as createHash20, randomUUID as
|
|
46433
|
+
import { createHash as createHash20, randomUUID as randomUUID36 } from "crypto";
|
|
45293
46434
|
import { mkdir as mkdir17, readFile as readFile23, rename as rename6, writeFile as writeFile16 } from "fs/promises";
|
|
45294
46435
|
import path37 from "path";
|
|
45295
46436
|
|
|
@@ -45373,7 +46514,7 @@ var InjectionSuiteRowStore = class {
|
|
|
45373
46514
|
};
|
|
45374
46515
|
await mkdir17(this.checkpointsDir, { recursive: true });
|
|
45375
46516
|
const destination = this.checkpointPath(identity);
|
|
45376
|
-
const tempPath = `${destination}.tmp-${
|
|
46517
|
+
const tempPath = `${destination}.tmp-${randomUUID36()}`;
|
|
45377
46518
|
await writeFile16(tempPath, `${JSON.stringify(checkpoint, null, 2)}
|
|
45378
46519
|
`, "utf8");
|
|
45379
46520
|
await rename6(tempPath, destination);
|
|
@@ -45404,7 +46545,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45404
46545
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
45405
46546
|
const lockPath = this.lockPath(rowKey);
|
|
45406
46547
|
await mkdir18(this.checkpointsDir, { recursive: true });
|
|
45407
|
-
const ownerToken =
|
|
46548
|
+
const ownerToken = randomUUID37();
|
|
45408
46549
|
try {
|
|
45409
46550
|
await mkdir18(lockPath);
|
|
45410
46551
|
} catch (error) {
|
|
@@ -45488,7 +46629,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
45488
46629
|
}
|
|
45489
46630
|
}
|
|
45490
46631
|
if (Date.now() - stampMs < leaseMs) return false;
|
|
45491
|
-
const stalePath = `${lockPath}.stale-${
|
|
46632
|
+
const stalePath = `${lockPath}.stale-${randomUUID37()}`;
|
|
45492
46633
|
try {
|
|
45493
46634
|
await rename7(lockPath, stalePath);
|
|
45494
46635
|
} catch {
|
|
@@ -46483,255 +47624,255 @@ import { constants } from "fs";
|
|
|
46483
47624
|
import { lstat as lstat7, mkdir as mkdir20, open as open3 } from "fs/promises";
|
|
46484
47625
|
import path41 from "path";
|
|
46485
47626
|
import { writeFileAtomically } from "@remnic/core/maintenance/atomic-file";
|
|
46486
|
-
import { z as
|
|
47627
|
+
import { z as z4 } from "zod";
|
|
46487
47628
|
|
|
46488
47629
|
// src/coding-graph/repeated-failure-report-rendering.ts
|
|
46489
47630
|
import { createHash as createHash24 } from "crypto";
|
|
46490
|
-
import { z as
|
|
47631
|
+
import { z as z3 } from "zod";
|
|
46491
47632
|
var SHA2562 = /^[a-f0-9]{64}$/;
|
|
46492
|
-
var IntervalSchema =
|
|
46493
|
-
lower:
|
|
46494
|
-
upper:
|
|
46495
|
-
level:
|
|
47633
|
+
var IntervalSchema = z3.object({
|
|
47634
|
+
lower: z3.number().nullable(),
|
|
47635
|
+
upper: z3.number().nullable(),
|
|
47636
|
+
level: z3.number().min(0).max(1)
|
|
46496
47637
|
});
|
|
46497
|
-
var EffectSchema =
|
|
46498
|
-
taskCount:
|
|
46499
|
-
baselineArm:
|
|
46500
|
-
candidateArm:
|
|
46501
|
-
interpretation:
|
|
46502
|
-
repeatedFailureBenefit:
|
|
47638
|
+
var EffectSchema = z3.object({
|
|
47639
|
+
taskCount: z3.number().int().nonnegative(),
|
|
47640
|
+
baselineArm: z3.enum(REPEATED_FAILURE_ARMS),
|
|
47641
|
+
candidateArm: z3.enum(REPEATED_FAILURE_ARMS),
|
|
47642
|
+
interpretation: z3.enum(["CONFIRMATORY", "EXPLORATORY_COMPLETE_TASKS"]),
|
|
47643
|
+
repeatedFailureBenefit: z3.number().nullable(),
|
|
46503
47644
|
repeatedFailureBenefitInterval: IntervalSchema.nullable(),
|
|
46504
|
-
relativeRiskReduction:
|
|
47645
|
+
relativeRiskReduction: z3.number().nullable(),
|
|
46505
47646
|
relativeRiskReductionInterval: IntervalSchema.nullable(),
|
|
46506
|
-
nonEstimableRrrDraws:
|
|
46507
|
-
repeatedFailureP:
|
|
46508
|
-
taskPassBenefit:
|
|
47647
|
+
nonEstimableRrrDraws: z3.number().int().nonnegative(),
|
|
47648
|
+
repeatedFailureP: z3.number().nullable(),
|
|
47649
|
+
taskPassBenefit: z3.number().nullable(),
|
|
46509
47650
|
taskPassBenefitInterval: IntervalSchema.nullable(),
|
|
46510
|
-
taskPassP:
|
|
47651
|
+
taskPassP: z3.number().nullable()
|
|
46511
47652
|
});
|
|
46512
|
-
var StatisticsSchema =
|
|
46513
|
-
schemaVersion:
|
|
46514
|
-
seed:
|
|
46515
|
-
draws:
|
|
46516
|
-
level:
|
|
46517
|
-
alpha:
|
|
46518
|
-
cuts:
|
|
46519
|
-
hypothesis:
|
|
46520
|
-
taskId:
|
|
46521
|
-
reasons:
|
|
47653
|
+
var StatisticsSchema = z3.object({
|
|
47654
|
+
schemaVersion: z3.literal(1),
|
|
47655
|
+
seed: z3.number().int().nonnegative(),
|
|
47656
|
+
draws: z3.number().int().positive(),
|
|
47657
|
+
level: z3.number().min(0).max(1),
|
|
47658
|
+
alpha: z3.number().min(0).max(1),
|
|
47659
|
+
cuts: z3.array(z3.object({
|
|
47660
|
+
hypothesis: z3.enum(["TIMING", "CONTENT", "TIMIDITY"]),
|
|
47661
|
+
taskId: z3.string().min(1),
|
|
47662
|
+
reasons: z3.array(z3.string().min(1))
|
|
46522
47663
|
})),
|
|
46523
47664
|
timing: EffectSchema,
|
|
46524
47665
|
content: EffectSchema,
|
|
46525
|
-
contentCompoundP:
|
|
46526
|
-
holm:
|
|
46527
|
-
id:
|
|
46528
|
-
rawP:
|
|
46529
|
-
adjustedP:
|
|
46530
|
-
rank:
|
|
47666
|
+
contentCompoundP: z3.number().nullable(),
|
|
47667
|
+
holm: z3.array(z3.object({
|
|
47668
|
+
id: z3.enum(["TIMING", "CONTENT"]),
|
|
47669
|
+
rawP: z3.number(),
|
|
47670
|
+
adjustedP: z3.number(),
|
|
47671
|
+
rank: z3.number().int().positive()
|
|
46531
47672
|
})),
|
|
46532
|
-
decisions:
|
|
46533
|
-
timing:
|
|
46534
|
-
content:
|
|
47673
|
+
decisions: z3.object({
|
|
47674
|
+
timing: z3.enum(["SUPPORTED", "REJECTED", "NOT_ESTIMABLE"]),
|
|
47675
|
+
content: z3.enum(["SUPPORTED", "REJECTED", "NOT_ESTIMABLE"])
|
|
46535
47676
|
}),
|
|
46536
|
-
studyDecision:
|
|
46537
|
-
timidity:
|
|
46538
|
-
taskCount:
|
|
46539
|
-
intervalLevel:
|
|
46540
|
-
passRateDifference:
|
|
47677
|
+
studyDecision: z3.enum(["PASS", "PARTIAL", "REJECT", "NOT_ESTIMABLE"]),
|
|
47678
|
+
timidity: z3.object({
|
|
47679
|
+
taskCount: z3.number().int().nonnegative(),
|
|
47680
|
+
intervalLevel: z3.number().min(0).max(1),
|
|
47681
|
+
passRateDifference: z3.number().nullable(),
|
|
46541
47682
|
passRateInterval: IntervalSchema.nullable(),
|
|
46542
|
-
stepsDifference:
|
|
47683
|
+
stepsDifference: z3.number().nullable(),
|
|
46543
47684
|
stepsInterval: IntervalSchema.nullable(),
|
|
46544
|
-
passMargin:
|
|
46545
|
-
stepsMargin:
|
|
46546
|
-
equivalent:
|
|
47685
|
+
passMargin: z3.number().nonnegative(),
|
|
47686
|
+
stepsMargin: z3.number().nonnegative(),
|
|
47687
|
+
equivalent: z3.boolean().nullable()
|
|
46547
47688
|
})
|
|
46548
47689
|
});
|
|
46549
|
-
var AuditSchema =
|
|
46550
|
-
schemaVersion:
|
|
46551
|
-
runContract:
|
|
46552
|
-
datasetInventoryHash:
|
|
46553
|
-
preregistrationPath:
|
|
46554
|
-
decisionRuleHash:
|
|
46555
|
-
preregistrationHash:
|
|
46556
|
-
analysisVersion:
|
|
46557
|
-
harnessVersion:
|
|
46558
|
-
harnessSourceHash:
|
|
46559
|
-
provenanceHash:
|
|
46560
|
-
modelProfiles:
|
|
46561
|
-
id:
|
|
46562
|
-
hash:
|
|
46563
|
-
modelDigest:
|
|
46564
|
-
tokenizerIdentity:
|
|
46565
|
-
tokenizerImplementation:
|
|
47690
|
+
var AuditSchema = z3.object({
|
|
47691
|
+
schemaVersion: z3.literal(1),
|
|
47692
|
+
runContract: z3.object({
|
|
47693
|
+
datasetInventoryHash: z3.string().regex(SHA2562),
|
|
47694
|
+
preregistrationPath: z3.string().min(1),
|
|
47695
|
+
decisionRuleHash: z3.string().regex(SHA2562),
|
|
47696
|
+
preregistrationHash: z3.string().regex(SHA2562),
|
|
47697
|
+
analysisVersion: z3.string().min(1),
|
|
47698
|
+
harnessVersion: z3.string().min(1),
|
|
47699
|
+
harnessSourceHash: z3.string().regex(SHA2562),
|
|
47700
|
+
provenanceHash: z3.string().regex(SHA2562),
|
|
47701
|
+
modelProfiles: z3.array(z3.object({
|
|
47702
|
+
id: z3.string().min(1),
|
|
47703
|
+
hash: z3.string().regex(SHA2562),
|
|
47704
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47705
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47706
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1")
|
|
46566
47707
|
}).strict()).min(1),
|
|
46567
|
-
trapAudit:
|
|
46568
|
-
minimumTrappedRate:
|
|
46569
|
-
minimumNonFixedRate:
|
|
46570
|
-
maximumInvalidRows:
|
|
46571
|
-
requireCompleteRows:
|
|
47708
|
+
trapAudit: z3.object({
|
|
47709
|
+
minimumTrappedRate: z3.number().min(0).max(1),
|
|
47710
|
+
minimumNonFixedRate: z3.number().min(0).max(1),
|
|
47711
|
+
maximumInvalidRows: z3.literal(0),
|
|
47712
|
+
requireCompleteRows: z3.literal(true)
|
|
46572
47713
|
}).strict()
|
|
46573
47714
|
}).strict(),
|
|
46574
|
-
dataset:
|
|
46575
|
-
inventoryHash:
|
|
46576
|
-
supportArtifactsMatch:
|
|
46577
|
-
taskCount:
|
|
46578
|
-
variantCount:
|
|
46579
|
-
splitCounts:
|
|
47715
|
+
dataset: z3.object({
|
|
47716
|
+
inventoryHash: z3.string().regex(SHA2562),
|
|
47717
|
+
supportArtifactsMatch: z3.boolean(),
|
|
47718
|
+
taskCount: z3.number().int().nonnegative(),
|
|
47719
|
+
variantCount: z3.number().int().nonnegative(),
|
|
47720
|
+
splitCounts: z3.record(z3.string(), z3.number().int().nonnegative())
|
|
46580
47721
|
}).passthrough(),
|
|
46581
|
-
expectedDesign:
|
|
46582
|
-
expectedRows:
|
|
46583
|
-
terminalRows:
|
|
46584
|
-
exactRowSet:
|
|
47722
|
+
expectedDesign: z3.object({
|
|
47723
|
+
expectedRows: z3.number().int().nonnegative(),
|
|
47724
|
+
terminalRows: z3.number().int().nonnegative(),
|
|
47725
|
+
exactRowSet: z3.boolean()
|
|
46585
47726
|
}),
|
|
46586
|
-
factPairs:
|
|
46587
|
-
isolation:
|
|
46588
|
-
allUnique:
|
|
46589
|
-
primaryStartHashesMatchWithinCells:
|
|
47727
|
+
factPairs: z3.object({ pairCount: z3.number().int().nonnegative(), allMatched: z3.boolean() }),
|
|
47728
|
+
isolation: z3.object({
|
|
47729
|
+
allUnique: z3.boolean(),
|
|
47730
|
+
primaryStartHashesMatchWithinCells: z3.boolean()
|
|
46590
47731
|
}).passthrough(),
|
|
46591
|
-
timingEvidence:
|
|
46592
|
-
fakeAgentContract:
|
|
46593
|
-
status:
|
|
46594
|
-
deterministicDriverCount:
|
|
47732
|
+
timingEvidence: z3.object({ allMatched: z3.boolean() }).passthrough(),
|
|
47733
|
+
fakeAgentContract: z3.object({
|
|
47734
|
+
status: z3.enum(["PASS", "FAIL", "NOT_APPLICABLE"]),
|
|
47735
|
+
deterministicDriverCount: z3.number().int().nonnegative()
|
|
46595
47736
|
}),
|
|
46596
|
-
modelProfiles:
|
|
46597
|
-
id:
|
|
46598
|
-
hash:
|
|
46599
|
-
modelDigest:
|
|
46600
|
-
tokenizerIdentity:
|
|
46601
|
-
tokenizerImplementation:
|
|
46602
|
-
driverKind:
|
|
47737
|
+
modelProfiles: z3.array(z3.object({
|
|
47738
|
+
id: z3.string().min(1),
|
|
47739
|
+
hash: z3.string().regex(SHA2562),
|
|
47740
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47741
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47742
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1"),
|
|
47743
|
+
driverKind: z3.enum(["responses", "ollama-chat", "deterministic-fake", "unknown"])
|
|
46603
47744
|
}).strict()).min(1),
|
|
46604
|
-
noTrap:
|
|
46605
|
-
expectedRows:
|
|
46606
|
-
observedRows:
|
|
46607
|
-
allPassed:
|
|
47745
|
+
noTrap: z3.object({
|
|
47746
|
+
expectedRows: z3.number().int().nonnegative(),
|
|
47747
|
+
observedRows: z3.number().int().nonnegative(),
|
|
47748
|
+
allPassed: z3.boolean()
|
|
46608
47749
|
}).strict(),
|
|
46609
|
-
deviations:
|
|
46610
|
-
count:
|
|
46611
|
-
none:
|
|
47750
|
+
deviations: z3.object({
|
|
47751
|
+
count: z3.number().int().nonnegative(),
|
|
47752
|
+
none: z3.boolean()
|
|
46612
47753
|
}).strict(),
|
|
46613
|
-
traces:
|
|
46614
|
-
expectedCount:
|
|
46615
|
-
durableCount:
|
|
46616
|
-
allDurable:
|
|
47754
|
+
traces: z3.object({
|
|
47755
|
+
expectedCount: z3.number().int().nonnegative(),
|
|
47756
|
+
durableCount: z3.number().int().nonnegative(),
|
|
47757
|
+
allDurable: z3.boolean()
|
|
46617
47758
|
}),
|
|
46618
|
-
cuts:
|
|
46619
|
-
decision:
|
|
47759
|
+
cuts: z3.object({ primary: z3.array(z3.unknown()), timidity: z3.array(z3.unknown()) }),
|
|
47760
|
+
decision: z3.enum(["PASS", "PARTIAL", "REJECT", "NOT_ESTIMABLE"])
|
|
46620
47761
|
}).passthrough();
|
|
46621
|
-
var PilotProfileBindingSchema =
|
|
46622
|
-
id:
|
|
46623
|
-
hash:
|
|
46624
|
-
modelDigest:
|
|
46625
|
-
driverKind:
|
|
46626
|
-
tokenizerIdentity:
|
|
46627
|
-
tokenizerImplementation:
|
|
47762
|
+
var PilotProfileBindingSchema = z3.object({
|
|
47763
|
+
id: z3.string().min(1),
|
|
47764
|
+
hash: z3.string().regex(SHA2562),
|
|
47765
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47766
|
+
driverKind: z3.enum(["responses", "ollama-chat", "deterministic-fake", "unknown"]),
|
|
47767
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47768
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1")
|
|
46628
47769
|
}).strict();
|
|
46629
|
-
var PilotTrapReceiptSchema =
|
|
46630
|
-
path:
|
|
46631
|
-
artifactHash:
|
|
46632
|
-
modelProfileId:
|
|
46633
|
-
modelProfileHash:
|
|
46634
|
-
modelDigest:
|
|
46635
|
-
tokenizerIdentity:
|
|
46636
|
-
tokenizerImplementation:
|
|
47770
|
+
var PilotTrapReceiptSchema = z3.object({
|
|
47771
|
+
path: z3.string().min(1),
|
|
47772
|
+
artifactHash: z3.string().regex(SHA2562),
|
|
47773
|
+
modelProfileId: z3.string().min(1),
|
|
47774
|
+
modelProfileHash: z3.string().regex(SHA2562),
|
|
47775
|
+
modelDigest: z3.string().regex(SHA2562),
|
|
47776
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47777
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1")
|
|
46637
47778
|
}).strict();
|
|
46638
|
-
var ComputedPilotPowerSchema =
|
|
46639
|
-
schemaVersion:
|
|
46640
|
-
status:
|
|
46641
|
-
phase:
|
|
46642
|
-
method:
|
|
46643
|
-
draws:
|
|
46644
|
-
analysisDraws:
|
|
46645
|
-
source:
|
|
46646
|
-
episodesHash:
|
|
46647
|
-
expectedDesignHash:
|
|
46648
|
-
decisionRuleHash:
|
|
47779
|
+
var ComputedPilotPowerSchema = z3.object({
|
|
47780
|
+
schemaVersion: z3.literal(1),
|
|
47781
|
+
status: z3.literal("COMPUTED"),
|
|
47782
|
+
phase: z3.literal("pilot"),
|
|
47783
|
+
method: z3.object({ analysisVersion: z3.string().min(1) }).passthrough(),
|
|
47784
|
+
draws: z3.number().int().positive(),
|
|
47785
|
+
analysisDraws: z3.number().int().positive(),
|
|
47786
|
+
source: z3.object({
|
|
47787
|
+
episodesHash: z3.string().regex(SHA2562),
|
|
47788
|
+
expectedDesignHash: z3.string().regex(SHA2562),
|
|
47789
|
+
decisionRuleHash: z3.string().regex(SHA2562)
|
|
46649
47790
|
}).strict(),
|
|
46650
|
-
simulations:
|
|
46651
|
-
timing:
|
|
46652
|
-
content:
|
|
46653
|
-
timidity:
|
|
47791
|
+
simulations: z3.object({
|
|
47792
|
+
timing: z3.object({ power: z3.number().min(0.8).max(1) }).passthrough(),
|
|
47793
|
+
content: z3.object({ power: z3.number().min(0.8).max(1) }).passthrough(),
|
|
47794
|
+
timidity: z3.object({ power: z3.number().min(0.8).max(1) }).passthrough()
|
|
46654
47795
|
}).strict()
|
|
46655
47796
|
}).passthrough();
|
|
46656
|
-
var PilotRowIdentitySchema =
|
|
46657
|
-
suiteVersion:
|
|
46658
|
-
taskId:
|
|
46659
|
-
variantId:
|
|
46660
|
-
modelProfileId:
|
|
46661
|
-
modelProfileHash:
|
|
46662
|
-
seed:
|
|
46663
|
-
arm:
|
|
47797
|
+
var PilotRowIdentitySchema = z3.object({
|
|
47798
|
+
suiteVersion: z3.string().min(1),
|
|
47799
|
+
taskId: z3.string().min(1),
|
|
47800
|
+
variantId: z3.string().min(1),
|
|
47801
|
+
modelProfileId: z3.string().min(1),
|
|
47802
|
+
modelProfileHash: z3.string().regex(SHA2562),
|
|
47803
|
+
seed: z3.number().int().nonnegative().max(4294967295),
|
|
47804
|
+
arm: z3.enum(REPEATED_FAILURE_ARMS)
|
|
46664
47805
|
}).strict();
|
|
46665
|
-
var MainPowerEvidenceSchema =
|
|
46666
|
-
schemaVersion:
|
|
46667
|
-
status:
|
|
46668
|
-
phase:
|
|
46669
|
-
pilotRunId:
|
|
46670
|
-
pilotManifestArtifactHash:
|
|
46671
|
-
pilotPowerArtifactHash:
|
|
47806
|
+
var MainPowerEvidenceSchema = z3.object({
|
|
47807
|
+
schemaVersion: z3.literal(1),
|
|
47808
|
+
status: z3.literal("VERIFIED_PILOT"),
|
|
47809
|
+
phase: z3.literal("main"),
|
|
47810
|
+
pilotRunId: z3.string().min(1),
|
|
47811
|
+
pilotManifestArtifactHash: z3.string().regex(SHA2562),
|
|
47812
|
+
pilotPowerArtifactHash: z3.string().regex(SHA2562),
|
|
46672
47813
|
pilot: ComputedPilotPowerSchema,
|
|
46673
|
-
pilotProfileBindings:
|
|
46674
|
-
pilotTrapAuditReceipts:
|
|
46675
|
-
pilotRunOrder:
|
|
46676
|
-
rowKey:
|
|
46677
|
-
analysis:
|
|
47814
|
+
pilotProfileBindings: z3.array(PilotProfileBindingSchema).min(1),
|
|
47815
|
+
pilotTrapAuditReceipts: z3.array(PilotTrapReceiptSchema).min(1),
|
|
47816
|
+
pilotRunOrder: z3.array(z3.object({
|
|
47817
|
+
rowKey: z3.string().min(1),
|
|
47818
|
+
analysis: z3.enum(["PRIMARY", "TIMIDITY"]),
|
|
46678
47819
|
identity: PilotRowIdentitySchema
|
|
46679
47820
|
}).strict()).min(1),
|
|
46680
|
-
pilotExpectedDesignHash:
|
|
46681
|
-
pilotEpisodesHash:
|
|
47821
|
+
pilotExpectedDesignHash: z3.string().regex(SHA2562),
|
|
47822
|
+
pilotEpisodesHash: z3.string().regex(SHA2562)
|
|
46682
47823
|
}).strict();
|
|
46683
|
-
var FactPairAuditSchema =
|
|
46684
|
-
schemaVersion:
|
|
46685
|
-
pairs:
|
|
47824
|
+
var FactPairAuditSchema = z3.object({
|
|
47825
|
+
schemaVersion: z3.literal(1),
|
|
47826
|
+
pairs: z3.array(z3.object({ status: z3.enum(["MATCHED", "UNMATCHED"]) }).passthrough())
|
|
46686
47827
|
}).passthrough();
|
|
46687
|
-
var RegisteredFactPairAuditSchema =
|
|
46688
|
-
schemaVersion:
|
|
46689
|
-
minimumJaccard:
|
|
46690
|
-
maximumTokenGap:
|
|
46691
|
-
maximumRelativeTokenGap:
|
|
46692
|
-
pairs:
|
|
46693
|
-
pairKey:
|
|
46694
|
-
taskId:
|
|
46695
|
-
variantId:
|
|
46696
|
-
seed:
|
|
46697
|
-
modelProfileId:
|
|
46698
|
-
modelProfileHash:
|
|
46699
|
-
tokenizerIdentity:
|
|
46700
|
-
tokenizerImplementation:
|
|
46701
|
-
historyHash:
|
|
46702
|
-
failureRepoHash:
|
|
46703
|
-
successRepoHash:
|
|
46704
|
-
failureActionFingerprint:
|
|
46705
|
-
successActionFingerprint:
|
|
46706
|
-
failurePathShapeHash:
|
|
46707
|
-
successPathShapeHash:
|
|
46708
|
-
failureActionShapeHash:
|
|
46709
|
-
successActionShapeHash:
|
|
46710
|
-
failureFactId:
|
|
46711
|
-
failureCitationHash:
|
|
46712
|
-
failureFactHash:
|
|
46713
|
-
successFactHash:
|
|
46714
|
-
failureFactCount:
|
|
46715
|
-
successFactCount:
|
|
46716
|
-
failureTokens:
|
|
46717
|
-
successTokens:
|
|
46718
|
-
tokenGap:
|
|
46719
|
-
relativeTokenGap:
|
|
46720
|
-
jaccard:
|
|
46721
|
-
status:
|
|
47828
|
+
var RegisteredFactPairAuditSchema = z3.object({
|
|
47829
|
+
schemaVersion: z3.literal(1),
|
|
47830
|
+
minimumJaccard: z3.literal(0.8),
|
|
47831
|
+
maximumTokenGap: z3.literal(8),
|
|
47832
|
+
maximumRelativeTokenGap: z3.literal(0.05),
|
|
47833
|
+
pairs: z3.array(z3.object({
|
|
47834
|
+
pairKey: z3.string().regex(SHA2562),
|
|
47835
|
+
taskId: z3.string().min(1),
|
|
47836
|
+
variantId: z3.string().min(1),
|
|
47837
|
+
seed: z3.number().int().nonnegative(),
|
|
47838
|
+
modelProfileId: z3.string().min(1),
|
|
47839
|
+
modelProfileHash: z3.string().regex(SHA2562),
|
|
47840
|
+
tokenizerIdentity: z3.string().min(1),
|
|
47841
|
+
tokenizerImplementation: z3.literal("nfkc-whitespace-v1"),
|
|
47842
|
+
historyHash: z3.string().regex(SHA2562),
|
|
47843
|
+
failureRepoHash: z3.string().regex(SHA2562),
|
|
47844
|
+
successRepoHash: z3.string().regex(SHA2562),
|
|
47845
|
+
failureActionFingerprint: z3.string().min(1),
|
|
47846
|
+
successActionFingerprint: z3.string().min(1),
|
|
47847
|
+
failurePathShapeHash: z3.string().regex(SHA2562),
|
|
47848
|
+
successPathShapeHash: z3.string().regex(SHA2562),
|
|
47849
|
+
failureActionShapeHash: z3.string().regex(SHA2562),
|
|
47850
|
+
successActionShapeHash: z3.string().regex(SHA2562),
|
|
47851
|
+
failureFactId: z3.string().min(1),
|
|
47852
|
+
failureCitationHash: z3.string().regex(SHA2562),
|
|
47853
|
+
failureFactHash: z3.string().regex(SHA2562),
|
|
47854
|
+
successFactHash: z3.string().regex(SHA2562),
|
|
47855
|
+
failureFactCount: z3.literal(1),
|
|
47856
|
+
successFactCount: z3.literal(1),
|
|
47857
|
+
failureTokens: z3.number().int().nonnegative(),
|
|
47858
|
+
successTokens: z3.number().int().nonnegative(),
|
|
47859
|
+
tokenGap: z3.number().int().nonnegative(),
|
|
47860
|
+
relativeTokenGap: z3.number().nonnegative(),
|
|
47861
|
+
jaccard: z3.number().min(0).max(1),
|
|
47862
|
+
status: z3.enum(["MATCHED", "UNMATCHED"])
|
|
46722
47863
|
}).strict())
|
|
46723
47864
|
}).strict();
|
|
46724
|
-
var TraceTimingSchema =
|
|
46725
|
-
armAudit:
|
|
46726
|
-
timingPayload:
|
|
46727
|
-
frame:
|
|
46728
|
-
factId:
|
|
46729
|
-
citationHash:
|
|
46730
|
-
factCount:
|
|
46731
|
-
renderedTokenCount:
|
|
47865
|
+
var TraceTimingSchema = z3.object({
|
|
47866
|
+
armAudit: z3.object({
|
|
47867
|
+
timingPayload: z3.object({
|
|
47868
|
+
frame: z3.enum(["TURN_START", "PRE_ACTION"]),
|
|
47869
|
+
factId: z3.string().min(1),
|
|
47870
|
+
citationHash: z3.string().regex(SHA2562),
|
|
47871
|
+
factCount: z3.literal(1),
|
|
47872
|
+
renderedTokenCount: z3.number().int().nonnegative()
|
|
46732
47873
|
}).strict().nullable(),
|
|
46733
|
-
turnStartFactHash:
|
|
46734
|
-
preActionFailureFactHash:
|
|
47874
|
+
turnStartFactHash: z3.string().regex(SHA2562).nullable(),
|
|
47875
|
+
preActionFailureFactHash: z3.string().regex(SHA2562).nullable()
|
|
46735
47876
|
}).passthrough()
|
|
46736
47877
|
}).passthrough();
|
|
46737
47878
|
function registeredProfileBindingsMatch(bindings, expectedProfileCount) {
|
|
@@ -46812,22 +47953,22 @@ function aggregateArmOutcomes(rows) {
|
|
|
46812
47953
|
const first = group[0];
|
|
46813
47954
|
if (!first) throw new Error("arm outcome group cannot be empty");
|
|
46814
47955
|
const valid = group.filter((row) => row.status === "VALID");
|
|
46815
|
-
const
|
|
47956
|
+
const mean5 = (values) => values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
46816
47957
|
return {
|
|
46817
47958
|
modelProfileId: first.identity.modelProfileId,
|
|
46818
47959
|
modelProfileHash: first.identity.modelProfileHash,
|
|
46819
47960
|
arm: first.identity.arm,
|
|
46820
47961
|
validRows: valid.length,
|
|
46821
47962
|
invalidRows: group.length - valid.length,
|
|
46822
|
-
repeatedFailureRate:
|
|
46823
|
-
taskPassRate:
|
|
46824
|
-
meanSteps:
|
|
47963
|
+
repeatedFailureRate: mean5(valid.map((row) => row.repeatedFailure ? 1 : 0)),
|
|
47964
|
+
taskPassRate: mean5(valid.map((row) => row.taskPassed ? 1 : 0)),
|
|
47965
|
+
meanSteps: mean5(valid.map(
|
|
46825
47966
|
(row) => requireNonnegativeInteger(row.steps, "steps", row.rowKey)
|
|
46826
47967
|
)),
|
|
46827
|
-
warningRate:
|
|
47968
|
+
warningRate: mean5(valid.map(
|
|
46828
47969
|
(row) => requireNonnegativeInteger(row.warningCount, "warningCount", row.rowKey) > 0 ? 1 : 0
|
|
46829
47970
|
)),
|
|
46830
|
-
falseWarningRate:
|
|
47971
|
+
falseWarningRate: mean5(valid.map(
|
|
46831
47972
|
(row) => requireNonnegativeInteger(row.falseWarningCount, "falseWarningCount", row.rowKey) > 0 ? 1 : 0
|
|
46832
47973
|
))
|
|
46833
47974
|
};
|
|
@@ -47339,10 +48480,10 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
47339
48480
|
const reproManifest = await verifyRunManifest(runDir);
|
|
47340
48481
|
const source = await readSourceArtifacts(runDir);
|
|
47341
48482
|
const runJson = JSON.parse(source["run.json"]);
|
|
47342
|
-
const runBinding =
|
|
47343
|
-
decisionRuleHash:
|
|
47344
|
-
preregistrationPath:
|
|
47345
|
-
preregistrationHash:
|
|
48483
|
+
const runBinding = z4.object({
|
|
48484
|
+
decisionRuleHash: z4.string().regex(SHA2562),
|
|
48485
|
+
preregistrationPath: z4.string().min(1),
|
|
48486
|
+
preregistrationHash: z4.string().regex(SHA2562)
|
|
47346
48487
|
}).passthrough().parse(runJson);
|
|
47347
48488
|
const decisionRuleBytes = source["decision-rule.json"];
|
|
47348
48489
|
const decisionRule = DecisionRuleSchema.parse(JSON.parse(decisionRuleBytes));
|
|
@@ -47725,7 +48866,7 @@ async function writeRepeatedFailurePaperArtifacts(options) {
|
|
|
47725
48866
|
}
|
|
47726
48867
|
async function runRepeatedFailurePaperReportCliCommand(options) {
|
|
47727
48868
|
try {
|
|
47728
|
-
const { replayRepeatedFailureStatistics: replayRepeatedFailureStatistics2 } = await import("./repeated-failure-suite-runner-
|
|
48869
|
+
const { replayRepeatedFailureStatistics: replayRepeatedFailureStatistics2 } = await import("./repeated-failure-suite-runner-ONGURPYC.js");
|
|
47729
48870
|
const replay = await replayRepeatedFailureStatistics2(options);
|
|
47730
48871
|
if (replay.exitCode !== 0) return replay;
|
|
47731
48872
|
const result = await writeRepeatedFailurePaperArtifacts(options);
|
|
@@ -48312,6 +49453,7 @@ export {
|
|
|
48312
49453
|
orchestrateBenchmarkRuns,
|
|
48313
49454
|
pairedDeltaConfidenceInterval,
|
|
48314
49455
|
parseBenchmarkArtifact,
|
|
49456
|
+
parseCanaryFloor,
|
|
48315
49457
|
parseCodexJsonlUsage,
|
|
48316
49458
|
parseCustomBenchmark,
|
|
48317
49459
|
parseLocalLabManifest,
|
|
@@ -48348,8 +49490,10 @@ export {
|
|
|
48348
49490
|
resolveBenchmarkResultReference,
|
|
48349
49491
|
resolveBenchmarkRunCount,
|
|
48350
49492
|
resolveBenchmarkRunId,
|
|
49493
|
+
resolveCanaryFloorFromEnv,
|
|
48351
49494
|
resolveCodexCreditBudgetConfig,
|
|
48352
49495
|
resolveCommittedH6FixtureDirectory,
|
|
49496
|
+
resolveEffectiveCanaryFloor,
|
|
48353
49497
|
resolveLocalLabJudgeProviderConfig,
|
|
48354
49498
|
resolveLocalLabProfile,
|
|
48355
49499
|
resolveLocalLabRole,
|