@remnic/bench 9.69.56 → 9.69.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +271 -37
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -30763,8 +30763,238 @@ ${body}`,
|
|
|
30763
30763
|
};
|
|
30764
30764
|
}
|
|
30765
30765
|
|
|
30766
|
-
// src/benchmarks/remnic/
|
|
30766
|
+
// src/benchmarks/remnic/say-once/runner.ts
|
|
30767
30767
|
import { randomUUID as randomUUID23 } from "crypto";
|
|
30768
|
+
|
|
30769
|
+
// src/benchmarks/remnic/say-once/fixture.ts
|
|
30770
|
+
var SAY_ONCE_CASES = [
|
|
30771
|
+
// ─── Explicit ───────────────────────────────────────────────────────────
|
|
30772
|
+
{
|
|
30773
|
+
id: "explicit-dark-mode",
|
|
30774
|
+
tier: "explicit",
|
|
30775
|
+
seedUserMessage: "I prefer dark mode for all my editors. Please set it as default.",
|
|
30776
|
+
seedAssistantMessage: "I've noted your preference for dark mode. I'll apply it as the default theme going forward.",
|
|
30777
|
+
preference: "user prefers dark mode",
|
|
30778
|
+
probes: [
|
|
30779
|
+
{
|
|
30780
|
+
prompt: "What theme should I use for the new project?",
|
|
30781
|
+
expectInRecall: "dark mode"
|
|
30782
|
+
}
|
|
30783
|
+
]
|
|
30784
|
+
},
|
|
30785
|
+
{
|
|
30786
|
+
id: "explicit-meeting-format",
|
|
30787
|
+
tier: "explicit",
|
|
30788
|
+
seedUserMessage: "I want all meeting summaries in bullet points, not paragraphs.",
|
|
30789
|
+
seedAssistantMessage: "Understood. I'll format meeting summaries as bullet points rather than paragraphs from now on.",
|
|
30790
|
+
preference: "user prefers bullet-point meeting summaries",
|
|
30791
|
+
probes: [
|
|
30792
|
+
{
|
|
30793
|
+
prompt: "Can you summarize yesterday's standup?",
|
|
30794
|
+
expectInRecall: "bullet"
|
|
30795
|
+
}
|
|
30796
|
+
]
|
|
30797
|
+
},
|
|
30798
|
+
// ─── Casual ─────────────────────────────────────────────────────────────
|
|
30799
|
+
{
|
|
30800
|
+
id: "casual-concise",
|
|
30801
|
+
tier: "casual",
|
|
30802
|
+
seedUserMessage: "Short answers are fine, I don't need a lot of explanation.",
|
|
30803
|
+
seedAssistantMessage: "Got it \u2014 I'll keep responses concise unless you ask for more detail.",
|
|
30804
|
+
preference: "user prefers concise responses",
|
|
30805
|
+
probes: [
|
|
30806
|
+
{
|
|
30807
|
+
prompt: "What's the capital of Finland?",
|
|
30808
|
+
expectInRecall: "concise"
|
|
30809
|
+
}
|
|
30810
|
+
]
|
|
30811
|
+
},
|
|
30812
|
+
{
|
|
30813
|
+
id: "casual-code-examples",
|
|
30814
|
+
tier: "casual",
|
|
30815
|
+
seedUserMessage: "I learn best from code examples, just show me the code.",
|
|
30816
|
+
seedAssistantMessage: "I'll include more code examples in my explanations going forward.",
|
|
30817
|
+
preference: "user prefers code examples in explanations",
|
|
30818
|
+
probes: [
|
|
30819
|
+
{
|
|
30820
|
+
prompt: "How do I use async/await in Python?",
|
|
30821
|
+
expectInRecall: "code examples"
|
|
30822
|
+
}
|
|
30823
|
+
]
|
|
30824
|
+
},
|
|
30825
|
+
// ─── Buried mid-task ────────────────────────────────────────────────────
|
|
30826
|
+
{
|
|
30827
|
+
id: "buried-timezone",
|
|
30828
|
+
tier: "buried-mid-task",
|
|
30829
|
+
seedUserMessage: "Let's deploy the release. By the way, I'm in UTC so schedule everything in my timezone. The build passed, right?",
|
|
30830
|
+
seedAssistantMessage: "The build passed. I'll use UTC for scheduling going forward. Ready to deploy.",
|
|
30831
|
+
preference: "user is in UTC timezone",
|
|
30832
|
+
probes: [
|
|
30833
|
+
{
|
|
30834
|
+
prompt: "What time should I schedule the maintenance window?",
|
|
30835
|
+
expectInRecall: "UTC"
|
|
30836
|
+
}
|
|
30837
|
+
]
|
|
30838
|
+
},
|
|
30839
|
+
{
|
|
30840
|
+
id: "buried-email-style",
|
|
30841
|
+
tier: "buried-mid-task",
|
|
30842
|
+
seedUserMessage: "Can you review the PR? Also I don't like formal email greetings, just get to the point. Oh and the tests pass locally.",
|
|
30843
|
+
seedAssistantMessage: "PR review started. Noted on the informal style \u2014 I'll skip greetings in drafts. Tests passing locally confirmed.",
|
|
30844
|
+
preference: "user prefers informal communication, no greetings",
|
|
30845
|
+
probes: [
|
|
30846
|
+
{
|
|
30847
|
+
prompt: "Draft a response to the client about the delay.",
|
|
30848
|
+
expectInRecall: "greeting"
|
|
30849
|
+
}
|
|
30850
|
+
]
|
|
30851
|
+
}
|
|
30852
|
+
];
|
|
30853
|
+
var SAY_ONCE_SMOKE_FIXTURE = [
|
|
30854
|
+
SAY_ONCE_CASES.find((c) => c.id === "explicit-dark-mode"),
|
|
30855
|
+
SAY_ONCE_CASES.find((c) => c.id === "casual-concise"),
|
|
30856
|
+
SAY_ONCE_CASES.find((c) => c.id === "buried-timezone")
|
|
30857
|
+
];
|
|
30858
|
+
|
|
30859
|
+
// src/benchmarks/remnic/say-once/runner.ts
|
|
30860
|
+
var sayOnceDefinition = {
|
|
30861
|
+
id: "say-once",
|
|
30862
|
+
title: "Say-Once (extraction -> recall round-trip)",
|
|
30863
|
+
tier: "remnic",
|
|
30864
|
+
status: "ready",
|
|
30865
|
+
runnerAvailable: true,
|
|
30866
|
+
meta: {
|
|
30867
|
+
name: "say-once",
|
|
30868
|
+
version: "1.0.0",
|
|
30869
|
+
description: "Scores whether a preference stated once at a given vagueness tier resurfaces in later recall (issue #3036). Replay mode only.",
|
|
30870
|
+
category: "retrieval",
|
|
30871
|
+
citation: "Remnic internal synthetic benchmark for issue #3036"
|
|
30872
|
+
}
|
|
30873
|
+
};
|
|
30874
|
+
var SayOnceHarnessError = class extends Error {
|
|
30875
|
+
constructor(message) {
|
|
30876
|
+
super(message);
|
|
30877
|
+
this.name = "SayOnceHarnessError";
|
|
30878
|
+
}
|
|
30879
|
+
};
|
|
30880
|
+
function takeWithinBudget(cases, budget) {
|
|
30881
|
+
if (!Number.isFinite(budget)) return [...cases];
|
|
30882
|
+
if (budget <= 0) return [];
|
|
30883
|
+
return cases.slice(0, Math.floor(budget));
|
|
30884
|
+
}
|
|
30885
|
+
function resolveSayOnceLimit(raw) {
|
|
30886
|
+
if (raw === void 0 || raw === null) return Number.POSITIVE_INFINITY;
|
|
30887
|
+
const value = typeof raw === "string" ? Number(raw) : raw;
|
|
30888
|
+
if (typeof value !== "number" || !Number.isFinite(value) || !Number.isInteger(value)) {
|
|
30889
|
+
throw new SayOnceHarnessError(
|
|
30890
|
+
`say-once: --limit must be a finite integer; got ${JSON.stringify(raw)}`
|
|
30891
|
+
);
|
|
30892
|
+
}
|
|
30893
|
+
if (value < 0) {
|
|
30894
|
+
throw new SayOnceHarnessError(`say-once: --limit must not be negative; got ${value}`);
|
|
30895
|
+
}
|
|
30896
|
+
return value;
|
|
30897
|
+
}
|
|
30898
|
+
async function runSayOnceBenchmark(options) {
|
|
30899
|
+
const source = options.mode === "quick" ? SAY_ONCE_SMOKE_FIXTURE : SAY_ONCE_CASES;
|
|
30900
|
+
if (source.length === 0) {
|
|
30901
|
+
throw new SayOnceHarnessError("say-once: fixture set is empty");
|
|
30902
|
+
}
|
|
30903
|
+
const cases = takeWithinBudget(source, resolveSayOnceLimit(options.limit));
|
|
30904
|
+
const tasks = [];
|
|
30905
|
+
const system = options.system;
|
|
30906
|
+
if (!system) {
|
|
30907
|
+
throw new SayOnceHarnessError(
|
|
30908
|
+
"say-once: a BenchMemoryAdapter is required (pass --adapter or wire one through the bench runner)"
|
|
30909
|
+
);
|
|
30910
|
+
}
|
|
30911
|
+
let caseIndex = 0;
|
|
30912
|
+
for (const sample of cases) {
|
|
30913
|
+
caseIndex += 1;
|
|
30914
|
+
const startedAt = performance.now();
|
|
30915
|
+
const sessionId = `say-once-${sample.id}-${randomUUID23()}`;
|
|
30916
|
+
try {
|
|
30917
|
+
const result = await runSingleCase(sample, sessionId, system);
|
|
30918
|
+
tasks.push({
|
|
30919
|
+
...result,
|
|
30920
|
+
latencyMs: Math.round(performance.now() - startedAt),
|
|
30921
|
+
tokens: { input: 0, output: 0 }
|
|
30922
|
+
});
|
|
30923
|
+
} finally {
|
|
30924
|
+
await system.reset(sessionId).catch(() => {
|
|
30925
|
+
});
|
|
30926
|
+
}
|
|
30927
|
+
options.onTaskComplete?.(tasks[tasks.length - 1], caseIndex, cases.length);
|
|
30928
|
+
}
|
|
30929
|
+
const remnicVersion = await getRemnicVersion();
|
|
30930
|
+
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
30931
|
+
return {
|
|
30932
|
+
meta: {
|
|
30933
|
+
id: randomUUID23(),
|
|
30934
|
+
benchmark: options.benchmark.id,
|
|
30935
|
+
benchmarkTier: options.benchmark.tier,
|
|
30936
|
+
version: options.benchmark.meta.version,
|
|
30937
|
+
remnicVersion,
|
|
30938
|
+
gitSha: getGitSha(),
|
|
30939
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
30940
|
+
mode: options.mode,
|
|
30941
|
+
runCount: 1,
|
|
30942
|
+
seeds: [options.seed ?? 0]
|
|
30943
|
+
},
|
|
30944
|
+
config: {
|
|
30945
|
+
systemProvider: options.systemProvider ?? null,
|
|
30946
|
+
judgeProvider: options.judgeProvider ?? null,
|
|
30947
|
+
adapterMode: options.adapterMode ?? "direct",
|
|
30948
|
+
remnicConfig: options.remnicConfig ?? {}
|
|
30949
|
+
},
|
|
30950
|
+
cost: {
|
|
30951
|
+
totalTokens: 0,
|
|
30952
|
+
inputTokens: 0,
|
|
30953
|
+
outputTokens: 0,
|
|
30954
|
+
estimatedCostUsd: 0,
|
|
30955
|
+
totalLatencyMs,
|
|
30956
|
+
meanQueryLatencyMs: tasks.length > 0 ? totalLatencyMs / tasks.length : 0
|
|
30957
|
+
},
|
|
30958
|
+
results: {
|
|
30959
|
+
tasks,
|
|
30960
|
+
aggregates: aggregateTaskScores(tasks.map((task) => task.scores))
|
|
30961
|
+
},
|
|
30962
|
+
environment: {
|
|
30963
|
+
os: process.platform,
|
|
30964
|
+
nodeVersion: process.version,
|
|
30965
|
+
hardware: process.arch
|
|
30966
|
+
}
|
|
30967
|
+
};
|
|
30968
|
+
}
|
|
30969
|
+
async function runSingleCase(sample, sessionId, system) {
|
|
30970
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
30971
|
+
await system.store(sessionId, [
|
|
30972
|
+
{ role: "user", content: sample.seedUserMessage, timestamp: now },
|
|
30973
|
+
{ role: "assistant", content: sample.seedAssistantMessage, timestamp: now }
|
|
30974
|
+
]);
|
|
30975
|
+
let recalled = 0;
|
|
30976
|
+
const probeNotes = [];
|
|
30977
|
+
for (const probe of sample.probes) {
|
|
30978
|
+
const recalledContext = await system.recall(sessionId, probe.prompt, 2e3);
|
|
30979
|
+
const hit = recalledContext.toLowerCase().includes(probe.expectInRecall.toLowerCase());
|
|
30980
|
+
if (hit) recalled += 1;
|
|
30981
|
+
probeNotes.push(
|
|
30982
|
+
`${probe.prompt} -> ${hit ? "recalled" : "missed"} (expected "${probe.expectInRecall}")`
|
|
30983
|
+
);
|
|
30984
|
+
}
|
|
30985
|
+
const rate = sample.probes.length > 0 ? recalled / sample.probes.length : 0;
|
|
30986
|
+
return {
|
|
30987
|
+
taskId: sample.id,
|
|
30988
|
+
question: sample.seedUserMessage,
|
|
30989
|
+
expected: `all ${sample.probes.length} probe(s) recall the preference`,
|
|
30990
|
+
actual: `${recalled}/${sample.probes.length} recalled`,
|
|
30991
|
+
scores: { recall: rate },
|
|
30992
|
+
details: { tier: sample.tier, probes: probeNotes }
|
|
30993
|
+
};
|
|
30994
|
+
}
|
|
30995
|
+
|
|
30996
|
+
// src/benchmarks/remnic/ingestion-entity-recall/runner.ts
|
|
30997
|
+
import { randomUUID as randomUUID24 } from "crypto";
|
|
30768
30998
|
import { mkdtemp as mkdtemp7, writeFile as writeFile6, rm as rm8, mkdir as mkdir6, realpath as realpath5 } from "fs/promises";
|
|
30769
30999
|
import { tmpdir as tmpdir2 } from "os";
|
|
30770
31000
|
import path21 from "path";
|
|
@@ -31363,7 +31593,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
31363
31593
|
const remnicVersion = await getRemnicVersion();
|
|
31364
31594
|
return {
|
|
31365
31595
|
meta: {
|
|
31366
|
-
id:
|
|
31596
|
+
id: randomUUID24(),
|
|
31367
31597
|
benchmark: options.benchmark.id,
|
|
31368
31598
|
benchmarkTier: options.benchmark.tier,
|
|
31369
31599
|
version: options.benchmark.meta.version,
|
|
@@ -31401,7 +31631,7 @@ async function buildResult(options, tasks, totalLatencyMs) {
|
|
|
31401
31631
|
}
|
|
31402
31632
|
|
|
31403
31633
|
// src/benchmarks/remnic/ingestion-schema-completeness/runner.ts
|
|
31404
|
-
import { randomUUID as
|
|
31634
|
+
import { randomUUID as randomUUID25 } from "crypto";
|
|
31405
31635
|
import { mkdtemp as mkdtemp8, writeFile as writeFile7, rm as rm9, mkdir as mkdir7, realpath as realpath6 } from "fs/promises";
|
|
31406
31636
|
import { tmpdir as tmpdir3 } from "os";
|
|
31407
31637
|
import path22 from "path";
|
|
@@ -31460,7 +31690,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31460
31690
|
const remnicVersion2 = await getRemnicVersion();
|
|
31461
31691
|
return {
|
|
31462
31692
|
meta: {
|
|
31463
|
-
id:
|
|
31693
|
+
id: randomUUID25(),
|
|
31464
31694
|
benchmark: options.benchmark.id,
|
|
31465
31695
|
benchmarkTier: options.benchmark.tier,
|
|
31466
31696
|
version: options.benchmark.meta.version,
|
|
@@ -31533,7 +31763,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31533
31763
|
const remnicVersion = await getRemnicVersion();
|
|
31534
31764
|
return {
|
|
31535
31765
|
meta: {
|
|
31536
|
-
id:
|
|
31766
|
+
id: randomUUID25(),
|
|
31537
31767
|
benchmark: options.benchmark.id,
|
|
31538
31768
|
benchmarkTier: options.benchmark.tier,
|
|
31539
31769
|
version: options.benchmark.meta.version,
|
|
@@ -31574,7 +31804,7 @@ async function runIngestionSchemaCompletenessBenchmark(options) {
|
|
|
31574
31804
|
}
|
|
31575
31805
|
|
|
31576
31806
|
// src/benchmarks/remnic/ingestion-backlink-f1/runner.ts
|
|
31577
|
-
import { randomUUID as
|
|
31807
|
+
import { randomUUID as randomUUID26 } from "crypto";
|
|
31578
31808
|
import { mkdtemp as mkdtemp9, writeFile as writeFile8, rm as rm10, mkdir as mkdir8, realpath as realpath7 } from "fs/promises";
|
|
31579
31809
|
import { tmpdir as tmpdir4 } from "os";
|
|
31580
31810
|
import path23 from "path";
|
|
@@ -31634,7 +31864,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
31634
31864
|
const remnicVersion = await getRemnicVersion();
|
|
31635
31865
|
return {
|
|
31636
31866
|
meta: {
|
|
31637
|
-
id:
|
|
31867
|
+
id: randomUUID26(),
|
|
31638
31868
|
benchmark: options.benchmark.id,
|
|
31639
31869
|
benchmarkTier: options.benchmark.tier,
|
|
31640
31870
|
version: options.benchmark.meta.version,
|
|
@@ -31675,7 +31905,7 @@ async function runIngestionBacklinkF1Benchmark(options) {
|
|
|
31675
31905
|
}
|
|
31676
31906
|
|
|
31677
31907
|
// src/benchmarks/remnic/ingestion-setup-friction/runner.ts
|
|
31678
|
-
import { randomUUID as
|
|
31908
|
+
import { randomUUID as randomUUID27 } from "crypto";
|
|
31679
31909
|
import { mkdtemp as mkdtemp10, writeFile as writeFile9, rm as rm11, mkdir as mkdir9, realpath as realpath8 } from "fs/promises";
|
|
31680
31910
|
import { tmpdir as tmpdir5 } from "os";
|
|
31681
31911
|
import path24 from "path";
|
|
@@ -31740,7 +31970,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
31740
31970
|
const remnicVersion = await getRemnicVersion();
|
|
31741
31971
|
return {
|
|
31742
31972
|
meta: {
|
|
31743
|
-
id:
|
|
31973
|
+
id: randomUUID27(),
|
|
31744
31974
|
benchmark: options.benchmark.id,
|
|
31745
31975
|
benchmarkTier: options.benchmark.tier,
|
|
31746
31976
|
version: options.benchmark.meta.version,
|
|
@@ -31781,7 +32011,7 @@ async function runIngestionSetupFrictionBenchmark(options) {
|
|
|
31781
32011
|
}
|
|
31782
32012
|
|
|
31783
32013
|
// src/benchmarks/remnic/ingestion-citation-accuracy/runner.ts
|
|
31784
|
-
import { randomUUID as
|
|
32014
|
+
import { randomUUID as randomUUID28 } from "crypto";
|
|
31785
32015
|
import { mkdtemp as mkdtemp11, writeFile as writeFile10, rm as rm12, mkdir as mkdir10, realpath as realpath9 } from "fs/promises";
|
|
31786
32016
|
import { tmpdir as tmpdir6 } from "os";
|
|
31787
32017
|
import path25 from "path";
|
|
@@ -31924,7 +32154,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
31924
32154
|
const remnicVersion2 = await getRemnicVersion();
|
|
31925
32155
|
return {
|
|
31926
32156
|
meta: {
|
|
31927
|
-
id:
|
|
32157
|
+
id: randomUUID28(),
|
|
31928
32158
|
benchmark: options.benchmark.id,
|
|
31929
32159
|
benchmarkTier: options.benchmark.tier,
|
|
31930
32160
|
version: options.benchmark.meta.version,
|
|
@@ -32042,7 +32272,7 @@ async function runIngestionCitationAccuracyBenchmark(options) {
|
|
|
32042
32272
|
const remnicVersion = await getRemnicVersion();
|
|
32043
32273
|
return {
|
|
32044
32274
|
meta: {
|
|
32045
|
-
id:
|
|
32275
|
+
id: randomUUID28(),
|
|
32046
32276
|
benchmark: options.benchmark.id,
|
|
32047
32277
|
benchmarkTier: options.benchmark.tier,
|
|
32048
32278
|
version: options.benchmark.meta.version,
|
|
@@ -32265,7 +32495,7 @@ var ASSISTANT_MORNING_BRIEF_SCENARIOS = [
|
|
|
32265
32495
|
var ASSISTANT_MORNING_BRIEF_SMOKE_SCENARIOS = ASSISTANT_MORNING_BRIEF_SCENARIOS.slice(0, 2);
|
|
32266
32496
|
|
|
32267
32497
|
// src/benchmarks/remnic/_assistant-common/runner.ts
|
|
32268
|
-
import { randomUUID as
|
|
32498
|
+
import { randomUUID as randomUUID29 } from "crypto";
|
|
32269
32499
|
import path27 from "path";
|
|
32270
32500
|
|
|
32271
32501
|
// src/run-seeds.ts
|
|
@@ -32779,7 +33009,7 @@ async function runAssistantBenchmark(definition, scenarios, resolved, runnerOpti
|
|
|
32779
33009
|
const totalSeedExecutions = tasks.length * runCount;
|
|
32780
33010
|
return {
|
|
32781
33011
|
meta: {
|
|
32782
|
-
id:
|
|
33012
|
+
id: randomUUID29(),
|
|
32783
33013
|
benchmark: definition.id,
|
|
32784
33014
|
benchmarkTier: definition.tier,
|
|
32785
33015
|
version: definition.meta.version,
|
|
@@ -33375,7 +33605,7 @@ async function runAssistantSynthesisBenchmark(options) {
|
|
|
33375
33605
|
}
|
|
33376
33606
|
|
|
33377
33607
|
// src/benchmarks/remnic/buffer-surprise-trigger/runner.ts
|
|
33378
|
-
import { randomUUID as
|
|
33608
|
+
import { randomUUID as randomUUID30 } from "crypto";
|
|
33379
33609
|
import path28 from "path";
|
|
33380
33610
|
import os7 from "os";
|
|
33381
33611
|
import { mkdir as mkdir11, rm as rm13 } from "fs/promises";
|
|
@@ -33609,19 +33839,19 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
33609
33839
|
const cases = loadCases10(options.mode, options.limit);
|
|
33610
33840
|
const tmpRoot = path28.join(
|
|
33611
33841
|
os7.tmpdir(),
|
|
33612
|
-
`remnic-bench-buffer-surprise-${
|
|
33842
|
+
`remnic-bench-buffer-surprise-${randomUUID30()}`
|
|
33613
33843
|
);
|
|
33614
33844
|
await mkdir11(tmpRoot, { recursive: true });
|
|
33615
33845
|
const tasks = [];
|
|
33616
33846
|
const startedAt = performance.now();
|
|
33617
33847
|
try {
|
|
33618
33848
|
for (const caseDef of cases) {
|
|
33619
|
-
const control = await
|
|
33849
|
+
const control = await runSingleCase2(caseDef, {
|
|
33620
33850
|
surpriseEnabled: false,
|
|
33621
33851
|
tmpRoot,
|
|
33622
33852
|
label: "control"
|
|
33623
33853
|
});
|
|
33624
|
-
const candidate = await
|
|
33854
|
+
const candidate = await runSingleCase2(caseDef, {
|
|
33625
33855
|
surpriseEnabled: true,
|
|
33626
33856
|
tmpRoot,
|
|
33627
33857
|
label: "candidate"
|
|
@@ -33636,7 +33866,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
33636
33866
|
const remnicVersion = await getRemnicVersion();
|
|
33637
33867
|
return {
|
|
33638
33868
|
meta: {
|
|
33639
|
-
id:
|
|
33869
|
+
id: randomUUID30(),
|
|
33640
33870
|
benchmark: options.benchmark.id,
|
|
33641
33871
|
benchmarkTier: options.benchmark.tier,
|
|
33642
33872
|
version: options.benchmark.meta.version,
|
|
@@ -33675,7 +33905,7 @@ async function runBufferSurpriseTriggerBenchmark(options) {
|
|
|
33675
33905
|
}
|
|
33676
33906
|
};
|
|
33677
33907
|
}
|
|
33678
|
-
async function
|
|
33908
|
+
async function runSingleCase2(caseDef, options) {
|
|
33679
33909
|
const memoryDir = path28.join(
|
|
33680
33910
|
options.tmpRoot,
|
|
33681
33911
|
`${caseDef.id}-${options.label}`
|
|
@@ -33863,7 +34093,7 @@ function loadCases10(mode, limit) {
|
|
|
33863
34093
|
}
|
|
33864
34094
|
|
|
33865
34095
|
// src/benchmarks/remnic/contradiction-detection/runner.ts
|
|
33866
|
-
import { randomUUID as
|
|
34096
|
+
import { randomUUID as randomUUID31 } from "crypto";
|
|
33867
34097
|
|
|
33868
34098
|
// src/benchmarks/remnic/contradiction-detection/fixture.ts
|
|
33869
34099
|
var TRUE_CONTRADICTIONS = [
|
|
@@ -34190,7 +34420,7 @@ async function runContradictionDetectionBenchmark(options) {
|
|
|
34190
34420
|
const meanQueryLatencyMs = tasks.length > 0 ? totalLatencyMs / tasks.length : 0;
|
|
34191
34421
|
return {
|
|
34192
34422
|
meta: {
|
|
34193
|
-
id:
|
|
34423
|
+
id: randomUUID31(),
|
|
34194
34424
|
benchmark: options.benchmark.id,
|
|
34195
34425
|
benchmarkTier: options.benchmark.tier,
|
|
34196
34426
|
version: options.benchmark.meta.version,
|
|
@@ -34245,7 +34475,7 @@ function loadCases11(mode, limit) {
|
|
|
34245
34475
|
}
|
|
34246
34476
|
|
|
34247
34477
|
// src/benchmarks/remnic/retention-aged-dataset/runner.ts
|
|
34248
|
-
import { randomUUID as
|
|
34478
|
+
import { randomUUID as randomUUID32 } from "crypto";
|
|
34249
34479
|
import {
|
|
34250
34480
|
decideTierTransition
|
|
34251
34481
|
} from "@remnic/core";
|
|
@@ -34621,7 +34851,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
34621
34851
|
const totalLatencyMs = tasks.reduce((sum, t) => sum + t.latencyMs, 0);
|
|
34622
34852
|
return {
|
|
34623
34853
|
meta: {
|
|
34624
|
-
id:
|
|
34854
|
+
id: randomUUID32(),
|
|
34625
34855
|
benchmark: options.benchmark.id,
|
|
34626
34856
|
benchmarkTier: options.benchmark.tier,
|
|
34627
34857
|
version: options.benchmark.meta.version,
|
|
@@ -34665,7 +34895,7 @@ async function runRetentionAgedDatasetBenchmark(options) {
|
|
|
34665
34895
|
}
|
|
34666
34896
|
|
|
34667
34897
|
// src/benchmarks/remnic/memcorrect/runner.ts
|
|
34668
|
-
import { randomUUID as
|
|
34898
|
+
import { randomUUID as randomUUID33 } from "crypto";
|
|
34669
34899
|
|
|
34670
34900
|
// src/benchmarks/remnic/memcorrect/generator.ts
|
|
34671
34901
|
import { createHash as createHash11 } from "crypto";
|
|
@@ -35878,7 +36108,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
35878
36108
|
const { adapter: _liveAdapter, ...persistableBenchmarkOptions } = options.benchmarkOptions ?? {};
|
|
35879
36109
|
return {
|
|
35880
36110
|
meta: {
|
|
35881
|
-
id:
|
|
36111
|
+
id: randomUUID33(),
|
|
35882
36112
|
benchmark: options.benchmark.id,
|
|
35883
36113
|
benchmarkTier: options.benchmark.tier,
|
|
35884
36114
|
version: options.benchmark.meta.version,
|
|
@@ -35944,7 +36174,7 @@ async function runMemCorrectBenchmark(options) {
|
|
|
35944
36174
|
}
|
|
35945
36175
|
|
|
35946
36176
|
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
35947
|
-
import { randomUUID as
|
|
36177
|
+
import { randomUUID as randomUUID34 } from "crypto";
|
|
35948
36178
|
import { mkdir as mkdir12, writeFile as writeFile11 } from "fs/promises";
|
|
35949
36179
|
import path29 from "path";
|
|
35950
36180
|
|
|
@@ -37126,7 +37356,7 @@ async function runBoundedMemoryContractsBenchmark(options) {
|
|
|
37126
37356
|
const skillTriggerLog = c3SkillLog;
|
|
37127
37357
|
return {
|
|
37128
37358
|
meta: {
|
|
37129
|
-
id:
|
|
37359
|
+
id: randomUUID34(),
|
|
37130
37360
|
benchmark: options.benchmark.id,
|
|
37131
37361
|
benchmarkTier: options.benchmark.tier,
|
|
37132
37362
|
version: options.benchmark.meta.version,
|
|
@@ -37302,7 +37532,7 @@ function renderPromptPack(task, condition, pack) {
|
|
|
37302
37532
|
}
|
|
37303
37533
|
|
|
37304
37534
|
// src/benchmarks/remnic/staged-memory/runner.ts
|
|
37305
|
-
import { createHash as createHash16, randomUUID as
|
|
37535
|
+
import { createHash as createHash16, randomUUID as randomUUID35 } from "crypto";
|
|
37306
37536
|
|
|
37307
37537
|
// src/benchmarks/remnic/staged-memory/fixture.ts
|
|
37308
37538
|
import { createHash as createHash15 } from "crypto";
|
|
@@ -40224,7 +40454,7 @@ async function runStagedMemoryBenchmark(options) {
|
|
|
40224
40454
|
const totalLatencyMs = tasks.reduce((sum, task) => sum + task.latencyMs, 0);
|
|
40225
40455
|
const result = {
|
|
40226
40456
|
meta: {
|
|
40227
|
-
id:
|
|
40457
|
+
id: randomUUID35(),
|
|
40228
40458
|
benchmark: options.benchmark.id,
|
|
40229
40459
|
benchmarkTier: options.benchmark.tier,
|
|
40230
40460
|
version: options.benchmark.meta.version,
|
|
@@ -40373,6 +40603,10 @@ var REGISTERED_BENCHMARKS = [
|
|
|
40373
40603
|
...proceduralRecallDefinition,
|
|
40374
40604
|
run: runProceduralRecallBenchmark
|
|
40375
40605
|
},
|
|
40606
|
+
{
|
|
40607
|
+
...sayOnceDefinition,
|
|
40608
|
+
run: runSayOnceBenchmark
|
|
40609
|
+
},
|
|
40376
40610
|
{
|
|
40377
40611
|
...ingestionEntityRecallDefinition,
|
|
40378
40612
|
run: runIngestionEntityRecallBenchmark
|
|
@@ -43499,7 +43733,7 @@ function formatError(error) {
|
|
|
43499
43733
|
}
|
|
43500
43734
|
|
|
43501
43735
|
// src/benchmarks/custom/runner.ts
|
|
43502
|
-
import { randomUUID as
|
|
43736
|
+
import { randomUUID as randomUUID36 } from "crypto";
|
|
43503
43737
|
import path35 from "path";
|
|
43504
43738
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
43505
43739
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
@@ -43587,7 +43821,7 @@ async function runCustomBenchmark(spec, options) {
|
|
|
43587
43821
|
const totalOutputTokens = tasks.reduce((sum, task) => sum + task.tokens.output, 0);
|
|
43588
43822
|
return finalizeBenchmarkResultConfig({
|
|
43589
43823
|
meta: {
|
|
43590
|
-
id:
|
|
43824
|
+
id: randomUUID36(),
|
|
43591
43825
|
benchmark: options.benchmark.id,
|
|
43592
43826
|
benchmarkTier: options.benchmark.tier,
|
|
43593
43827
|
version: options.benchmark.meta.version,
|
|
@@ -46469,10 +46703,10 @@ import path40 from "path";
|
|
|
46469
46703
|
import { hostname } from "os";
|
|
46470
46704
|
import { mkdir as mkdir18, readFile as readFile25, rename as rename7, rm as rm17, stat as stat4, utimes, writeFile as writeFile17 } from "fs/promises";
|
|
46471
46705
|
import path39 from "path";
|
|
46472
|
-
import { randomUUID as
|
|
46706
|
+
import { randomUUID as randomUUID38 } from "crypto";
|
|
46473
46707
|
|
|
46474
46708
|
// src/security/injection-suite/store.ts
|
|
46475
|
-
import { createHash as createHash20, randomUUID as
|
|
46709
|
+
import { createHash as createHash20, randomUUID as randomUUID37 } from "crypto";
|
|
46476
46710
|
import { mkdir as mkdir17, readFile as readFile24, rename as rename6, writeFile as writeFile16 } from "fs/promises";
|
|
46477
46711
|
import path38 from "path";
|
|
46478
46712
|
|
|
@@ -46556,7 +46790,7 @@ var InjectionSuiteRowStore = class {
|
|
|
46556
46790
|
};
|
|
46557
46791
|
await mkdir17(this.checkpointsDir, { recursive: true });
|
|
46558
46792
|
const destination = this.checkpointPath(identity);
|
|
46559
|
-
const tempPath = `${destination}.tmp-${
|
|
46793
|
+
const tempPath = `${destination}.tmp-${randomUUID37()}`;
|
|
46560
46794
|
await writeFile16(tempPath, `${JSON.stringify(checkpoint, null, 2)}
|
|
46561
46795
|
`, "utf8");
|
|
46562
46796
|
await rename6(tempPath, destination);
|
|
@@ -46587,7 +46821,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46587
46821
|
const rowKey = buildInjectionSuiteRowKey(identity);
|
|
46588
46822
|
const lockPath = this.lockPath(rowKey);
|
|
46589
46823
|
await mkdir18(this.checkpointsDir, { recursive: true });
|
|
46590
|
-
const ownerToken =
|
|
46824
|
+
const ownerToken = randomUUID38();
|
|
46591
46825
|
try {
|
|
46592
46826
|
await mkdir18(lockPath);
|
|
46593
46827
|
} catch (error) {
|
|
@@ -46671,7 +46905,7 @@ var InjectionSuiteClaimLock = class {
|
|
|
46671
46905
|
}
|
|
46672
46906
|
}
|
|
46673
46907
|
if (Date.now() - stampMs < leaseMs) return false;
|
|
46674
|
-
const stalePath = `${lockPath}.stale-${
|
|
46908
|
+
const stalePath = `${lockPath}.stale-${randomUUID38()}`;
|
|
46675
46909
|
try {
|
|
46676
46910
|
await rename7(lockPath, stalePath);
|
|
46677
46911
|
} catch {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/bench",
|
|
3
|
-
"version": "9.69.
|
|
3
|
+
"version": "9.69.58",
|
|
4
4
|
"description": "Retrieval latency ladder benchmarks + CI regression gates for @remnic/core",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -41,8 +41,8 @@
|
|
|
41
41
|
"hyparquet": "^1.25.7",
|
|
42
42
|
"yaml": "^2.4.2",
|
|
43
43
|
"zod": "^3.24.0",
|
|
44
|
-
"@remnic/coding-graph": "^9.69.
|
|
45
|
-
"@remnic/core": "^9.69.
|
|
44
|
+
"@remnic/coding-graph": "^9.69.58",
|
|
45
|
+
"@remnic/core": "^9.69.58"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"tsup": "^8.5.1",
|