@remnic/bench 9.3.725 → 9.3.727
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1452 -80
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -545,11 +545,11 @@ function benchEntityStructuredFactSourcePath(memoryDir, sessionId) {
|
|
|
545
545
|
`${benchCoreMemorySource(sessionId)}.json`
|
|
546
546
|
);
|
|
547
547
|
}
|
|
548
|
-
function normalizeBenchEntityStructuredFact(
|
|
549
|
-
return
|
|
548
|
+
function normalizeBenchEntityStructuredFact(fact3) {
|
|
549
|
+
return fact3.replace(/\s+/g, " ").trim();
|
|
550
550
|
}
|
|
551
|
-
function addBenchEntityStructuredFactSource(sources, entityName, sectionKey,
|
|
552
|
-
const normalizedFact = normalizeBenchEntityStructuredFact(
|
|
551
|
+
function addBenchEntityStructuredFactSource(sources, entityName, sectionKey, fact3) {
|
|
552
|
+
const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
|
|
553
553
|
if (!entityName || !sectionKey || !normalizedFact) return;
|
|
554
554
|
let entitySources = sources.get(entityName);
|
|
555
555
|
if (!entitySources) {
|
|
@@ -566,8 +566,8 @@ function addBenchEntityStructuredFactSource(sources, entityName, sectionKey, fac
|
|
|
566
566
|
function mergeBenchEntityStructuredFactSources(target, source) {
|
|
567
567
|
for (const [entityName, sections] of source) {
|
|
568
568
|
for (const [sectionKey, facts] of sections) {
|
|
569
|
-
for (const
|
|
570
|
-
addBenchEntityStructuredFactSource(target, entityName, sectionKey,
|
|
569
|
+
for (const fact3 of facts) {
|
|
570
|
+
addBenchEntityStructuredFactSource(target, entityName, sectionKey, fact3);
|
|
571
571
|
}
|
|
572
572
|
}
|
|
573
573
|
}
|
|
@@ -607,13 +607,13 @@ async function readBenchEntityStructuredFactSourceFile(filePath, expectedSession
|
|
|
607
607
|
if (typeof candidate.entityName !== "string" || typeof candidate.sectionKey !== "string" || !Array.isArray(candidate.facts)) {
|
|
608
608
|
continue;
|
|
609
609
|
}
|
|
610
|
-
for (const
|
|
611
|
-
if (typeof
|
|
610
|
+
for (const fact3 of candidate.facts) {
|
|
611
|
+
if (typeof fact3 !== "string") continue;
|
|
612
612
|
addBenchEntityStructuredFactSource(
|
|
613
613
|
sources,
|
|
614
614
|
candidate.entityName,
|
|
615
615
|
candidate.sectionKey,
|
|
616
|
-
|
|
616
|
+
fact3
|
|
617
617
|
);
|
|
618
618
|
}
|
|
619
619
|
}
|
|
@@ -674,9 +674,9 @@ async function removeBenchEntityStructuredFactSources(memoryDir, sessionId) {
|
|
|
674
674
|
async function captureBenchEntityStructuredFactWrite(orchestrator, sources, entityName, structuredSections, beforeSources) {
|
|
675
675
|
const incomingFacts = /* @__PURE__ */ new Set();
|
|
676
676
|
for (const section of structuredSections ?? []) {
|
|
677
|
-
for (const
|
|
678
|
-
if (typeof
|
|
679
|
-
const normalizedFact = normalizeBenchEntityStructuredFact(
|
|
677
|
+
for (const fact3 of section.facts ?? []) {
|
|
678
|
+
if (typeof fact3 !== "string") continue;
|
|
679
|
+
const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
|
|
680
680
|
if (normalizedFact) incomingFacts.add(normalizedFact);
|
|
681
681
|
}
|
|
682
682
|
}
|
|
@@ -686,8 +686,8 @@ async function captureBenchEntityStructuredFactWrite(orchestrator, sources, enti
|
|
|
686
686
|
if (!raw) return;
|
|
687
687
|
const entity = parseEntityFile(raw, entityStorage.entitySchemas);
|
|
688
688
|
for (const section of entity.structuredSections ?? []) {
|
|
689
|
-
for (const
|
|
690
|
-
const normalizedFact = normalizeBenchEntityStructuredFact(
|
|
689
|
+
for (const fact3 of section.facts) {
|
|
690
|
+
const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
|
|
691
691
|
if (!incomingFacts.has(normalizedFact)) continue;
|
|
692
692
|
if (beforeSources.get(entityName)?.get(section.key)?.has(normalizedFact)) {
|
|
693
693
|
continue;
|
|
@@ -710,8 +710,8 @@ async function readBenchEntityStructuredFactSnapshot(orchestrator) {
|
|
|
710
710
|
if (!raw) continue;
|
|
711
711
|
const entity = parseEntityFile(raw, entityStorage.entitySchemas);
|
|
712
712
|
for (const section of entity.structuredSections ?? []) {
|
|
713
|
-
for (const
|
|
714
|
-
addBenchEntityStructuredFactSource(sources, entityName, section.key,
|
|
713
|
+
for (const fact3 of section.facts) {
|
|
714
|
+
addBenchEntityStructuredFactSource(sources, entityName, section.key, fact3);
|
|
715
715
|
}
|
|
716
716
|
}
|
|
717
717
|
}
|
|
@@ -720,7 +720,7 @@ async function readBenchEntityStructuredFactSnapshot(orchestrator) {
|
|
|
720
720
|
function hasBenchEntityStructuredFacts(structuredSections) {
|
|
721
721
|
return (structuredSections ?? []).some(
|
|
722
722
|
(section) => (section.facts ?? []).some(
|
|
723
|
-
(
|
|
723
|
+
(fact3) => typeof fact3 === "string" && normalizeBenchEntityStructuredFact(fact3).length > 0
|
|
724
724
|
)
|
|
725
725
|
);
|
|
726
726
|
}
|
|
@@ -859,17 +859,17 @@ function compileBenchEntityFacts(entity) {
|
|
|
859
859
|
const facts = [];
|
|
860
860
|
const seen = /* @__PURE__ */ new Set();
|
|
861
861
|
for (const entry of entity.timeline) {
|
|
862
|
-
const
|
|
863
|
-
if (!
|
|
864
|
-
seen.add(
|
|
865
|
-
facts.push(
|
|
862
|
+
const fact3 = entry.text.trim();
|
|
863
|
+
if (!fact3 || seen.has(fact3)) continue;
|
|
864
|
+
seen.add(fact3);
|
|
865
|
+
facts.push(fact3);
|
|
866
866
|
}
|
|
867
867
|
for (const section of entity.structuredSections ?? []) {
|
|
868
868
|
for (const rawFact of section.facts) {
|
|
869
|
-
const
|
|
870
|
-
if (!
|
|
871
|
-
seen.add(
|
|
872
|
-
facts.push(
|
|
869
|
+
const fact3 = rawFact.replace(/\s+/g, " ").trim();
|
|
870
|
+
if (!fact3 || seen.has(fact3)) continue;
|
|
871
|
+
seen.add(fact3);
|
|
872
|
+
facts.push(fact3);
|
|
873
873
|
}
|
|
874
874
|
}
|
|
875
875
|
return facts;
|
|
@@ -892,8 +892,8 @@ function pruneBenchEntityStructuredFacts(structuredSections, targetSources, prot
|
|
|
892
892
|
for (const section of sections) {
|
|
893
893
|
const targetFacts = targetSources?.get(section.key);
|
|
894
894
|
const protectedFacts = protectedSources?.get(section.key);
|
|
895
|
-
const nextFacts = section.facts.filter((
|
|
896
|
-
const normalizedFact = normalizeBenchEntityStructuredFact(
|
|
895
|
+
const nextFacts = section.facts.filter((fact3) => {
|
|
896
|
+
const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
|
|
897
897
|
if (!targetFacts?.has(normalizedFact)) {
|
|
898
898
|
return true;
|
|
899
899
|
}
|
|
@@ -11198,8 +11198,8 @@ async function resolveLocalLabRuntimeProfile(options) {
|
|
|
11198
11198
|
|
|
11199
11199
|
// src/benchmark.ts
|
|
11200
11200
|
import fs2 from "fs";
|
|
11201
|
-
import
|
|
11202
|
-
import { createHash as
|
|
11201
|
+
import path33 from "path";
|
|
11202
|
+
import { createHash as createHash11 } from "crypto";
|
|
11203
11203
|
import { expandTildePath as expandTildePath3 } from "@remnic/core";
|
|
11204
11204
|
|
|
11205
11205
|
// src/judges/judge-cache.ts
|
|
@@ -11700,11 +11700,11 @@ function collectMetricValues(metricsList) {
|
|
|
11700
11700
|
return metricValues;
|
|
11701
11701
|
}
|
|
11702
11702
|
function summarizeMetricValues(values) {
|
|
11703
|
-
const
|
|
11703
|
+
const mean4 = values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
11704
11704
|
const median = values.length % 2 === 0 ? (values[values.length / 2 - 1] + values[values.length / 2]) / 2 : values[Math.floor(values.length / 2)];
|
|
11705
|
-
const variance = values.reduce((sum, value) => sum + (value -
|
|
11705
|
+
const variance = values.reduce((sum, value) => sum + (value - mean4) ** 2, 0) / values.length;
|
|
11706
11706
|
return {
|
|
11707
|
-
mean:
|
|
11707
|
+
mean: mean4,
|
|
11708
11708
|
median,
|
|
11709
11709
|
stdDev: Math.sqrt(variance),
|
|
11710
11710
|
min: values[0],
|
|
@@ -28213,8 +28213,8 @@ function renderMemoryViewForAgent(graph) {
|
|
|
28213
28213
|
}
|
|
28214
28214
|
lines.push("");
|
|
28215
28215
|
lines.push("Recent memory items:");
|
|
28216
|
-
for (const
|
|
28217
|
-
lines.push(`- [${
|
|
28216
|
+
for (const fact3 of graph.facts) {
|
|
28217
|
+
lines.push(`- [${fact3.id}] ${fact3.summary}`);
|
|
28218
28218
|
}
|
|
28219
28219
|
if (graph.stances.length > 0) {
|
|
28220
28220
|
lines.push("");
|
|
@@ -28240,9 +28240,9 @@ function renderMemorySummaryForJudge(graph) {
|
|
|
28240
28240
|
if (graph.currentDate) {
|
|
28241
28241
|
sections.push(`CURRENT_DATE: ${graph.currentDate}`);
|
|
28242
28242
|
}
|
|
28243
|
-
const facts = graph.facts.map((
|
|
28244
|
-
const tags = Array.isArray(
|
|
28245
|
-
return ` - ${
|
|
28243
|
+
const facts = graph.facts.map((fact3) => {
|
|
28244
|
+
const tags = Array.isArray(fact3.tags) && fact3.tags.length > 0 ? ` [tags: ${fact3.tags.join(", ")}]` : "";
|
|
28245
|
+
return ` - ${fact3.id}: ${fact3.summary}${tags}`;
|
|
28246
28246
|
}).join("\n");
|
|
28247
28247
|
if (facts.length > 0) sections.push(`FACTS:
|
|
28248
28248
|
${facts}`);
|
|
@@ -29004,10 +29004,10 @@ async function runSingleCase(caseDef, options) {
|
|
|
29004
29004
|
for (let i = 1; i < flushTurnIndices.length; i += 1) {
|
|
29005
29005
|
turnsBetween.push(flushTurnIndices[i] - flushTurnIndices[i - 1]);
|
|
29006
29006
|
}
|
|
29007
|
-
const
|
|
29007
|
+
const mean4 = turnsBetween.length > 0 ? turnsBetween.reduce((acc, v) => acc + v, 0) / turnsBetween.length : 0;
|
|
29008
29008
|
return {
|
|
29009
29009
|
flushTurnIndices,
|
|
29010
|
-
turnsBetweenFlushesMean:
|
|
29010
|
+
turnsBetweenFlushesMean: mean4,
|
|
29011
29011
|
replayLatencyMs
|
|
29012
29012
|
};
|
|
29013
29013
|
}
|
|
@@ -30531,8 +30531,8 @@ function nonResurrection(log, corrections) {
|
|
|
30531
30531
|
}
|
|
30532
30532
|
function collateralDelta(before, after) {
|
|
30533
30533
|
if (before.length === 0) return 0;
|
|
30534
|
-
const
|
|
30535
|
-
return
|
|
30534
|
+
const mean4 = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
|
|
30535
|
+
return mean4(after) - mean4(before);
|
|
30536
30536
|
}
|
|
30537
30537
|
function scopePrecision(log, corrections) {
|
|
30538
30538
|
let scopedCount = 0;
|
|
@@ -30975,6 +30975,1374 @@ async function runMemCorrectBenchmark(options) {
|
|
|
30975
30975
|
};
|
|
30976
30976
|
}
|
|
30977
30977
|
|
|
30978
|
+
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
30979
|
+
import { randomUUID as randomUUID32 } from "crypto";
|
|
30980
|
+
import { mkdir as mkdir16, writeFile as writeFile15 } from "fs/promises";
|
|
30981
|
+
import path32 from "path";
|
|
30982
|
+
|
|
30983
|
+
// src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
|
|
30984
|
+
import { createHash as createHash10 } from "crypto";
|
|
30985
|
+
var SCOPE_ACME = "project:acme";
|
|
30986
|
+
var SCOPE_BETA = "project:beta";
|
|
30987
|
+
var SCOPE_ALICE = "user:alice";
|
|
30988
|
+
var DEPLOY_GATEWAY_SKILL = {
|
|
30989
|
+
id: "skill:deploy-gateway",
|
|
30990
|
+
title: "Production gateway deploy runbook",
|
|
30991
|
+
trigger: "deploy gateway to production",
|
|
30992
|
+
appliesWhen: ["deploy", "gateway", "production"],
|
|
30993
|
+
doesNotApplyWhen: ["staging", "rollback", "what", "process", "explain"],
|
|
30994
|
+
steps: [
|
|
30995
|
+
"Run the production deploy checks for the gateway",
|
|
30996
|
+
"Push the release tag after CI is green",
|
|
30997
|
+
"Notify on-call in #deployments"
|
|
30998
|
+
],
|
|
30999
|
+
status: "active",
|
|
31000
|
+
sourceMemoryIds: ["decision:deploy-runbook"],
|
|
31001
|
+
confidence: 0.92,
|
|
31002
|
+
tokens: 60
|
|
31003
|
+
};
|
|
31004
|
+
var ROTATE_KEYS_SKILL = {
|
|
31005
|
+
id: "skill:rotate-api-keys",
|
|
31006
|
+
title: "API key rotation procedure",
|
|
31007
|
+
trigger: "rotate api keys",
|
|
31008
|
+
appliesWhen: ["rotate", "api", "keys"],
|
|
31009
|
+
doesNotApplyWhen: ["read", "view", "list", "audit", "what"],
|
|
31010
|
+
steps: [
|
|
31011
|
+
"Generate new key pair in the vault",
|
|
31012
|
+
"Update the service to dual-load old + new",
|
|
31013
|
+
"Retire the old key after one full rotation window"
|
|
31014
|
+
],
|
|
31015
|
+
status: "active",
|
|
31016
|
+
sourceMemoryIds: ["decision:key-rotation-policy"],
|
|
31017
|
+
confidence: 0.88,
|
|
31018
|
+
tokens: 55
|
|
31019
|
+
};
|
|
31020
|
+
function fact2(id, scope, content, keywords, answerToken, turn, tokens = 24, overrides = {}) {
|
|
31021
|
+
return {
|
|
31022
|
+
id,
|
|
31023
|
+
category: "fact",
|
|
31024
|
+
scope,
|
|
31025
|
+
status: "active",
|
|
31026
|
+
content,
|
|
31027
|
+
subjectKeywords: keywords,
|
|
31028
|
+
answerToken,
|
|
31029
|
+
tokens,
|
|
31030
|
+
turn,
|
|
31031
|
+
...overrides
|
|
31032
|
+
};
|
|
31033
|
+
}
|
|
31034
|
+
var recallNeededTasks = [
|
|
31035
|
+
{
|
|
31036
|
+
id: "recall-framework-choice",
|
|
31037
|
+
family: "recall-needed",
|
|
31038
|
+
prompt: "Which web framework did we settle on for the Acme dashboard?",
|
|
31039
|
+
scope: SCOPE_ACME,
|
|
31040
|
+
subjectKeywords: ["framework", "dashboard", "acme"],
|
|
31041
|
+
expectedAnswer: "remix",
|
|
31042
|
+
shouldRecallId: "fact:acme-framework",
|
|
31043
|
+
shouldExcludeIds: [],
|
|
31044
|
+
memoryItems: [
|
|
31045
|
+
fact2(
|
|
31046
|
+
"fact:acme-framework",
|
|
31047
|
+
SCOPE_ACME,
|
|
31048
|
+
"We chose Remix for the Acme dashboard frontend.",
|
|
31049
|
+
["framework", "dashboard", "acme"],
|
|
31050
|
+
"remix",
|
|
31051
|
+
3,
|
|
31052
|
+
30
|
|
31053
|
+
),
|
|
31054
|
+
fact2(
|
|
31055
|
+
"fact:acme-css",
|
|
31056
|
+
SCOPE_ACME,
|
|
31057
|
+
"The Acme dashboard uses Tailwind for styling.",
|
|
31058
|
+
["styling", "css", "acme"],
|
|
31059
|
+
"tailwind",
|
|
31060
|
+
4,
|
|
31061
|
+
20
|
|
31062
|
+
)
|
|
31063
|
+
],
|
|
31064
|
+
skills: []
|
|
31065
|
+
},
|
|
31066
|
+
{
|
|
31067
|
+
id: "recall-meeting-day",
|
|
31068
|
+
family: "recall-needed",
|
|
31069
|
+
prompt: "What day is our recurring Acme sync?",
|
|
31070
|
+
scope: SCOPE_ACME,
|
|
31071
|
+
subjectKeywords: ["sync", "meeting", "day", "acme"],
|
|
31072
|
+
expectedAnswer: "tuesday",
|
|
31073
|
+
shouldRecallId: "fact:acme-sync-day",
|
|
31074
|
+
shouldExcludeIds: [],
|
|
31075
|
+
memoryItems: [
|
|
31076
|
+
fact2(
|
|
31077
|
+
"fact:acme-sync-day",
|
|
31078
|
+
SCOPE_ACME,
|
|
31079
|
+
"The Acme team sync is on Tuesdays.",
|
|
31080
|
+
["sync", "meeting", "day", "acme"],
|
|
31081
|
+
"tuesday",
|
|
31082
|
+
2,
|
|
31083
|
+
28
|
|
31084
|
+
)
|
|
31085
|
+
],
|
|
31086
|
+
skills: []
|
|
31087
|
+
},
|
|
31088
|
+
{
|
|
31089
|
+
id: "recall-alice-timezone",
|
|
31090
|
+
family: "recall-needed",
|
|
31091
|
+
prompt: "What timezone is Alice in?",
|
|
31092
|
+
scope: SCOPE_ALICE,
|
|
31093
|
+
subjectKeywords: ["alice", "timezone"],
|
|
31094
|
+
expectedAnswer: "aest",
|
|
31095
|
+
shouldRecallId: "fact:alice-timezone",
|
|
31096
|
+
shouldExcludeIds: [],
|
|
31097
|
+
memoryItems: [
|
|
31098
|
+
fact2(
|
|
31099
|
+
"fact:alice-timezone",
|
|
31100
|
+
SCOPE_ALICE,
|
|
31101
|
+
"Alice is based in AEST (UTC+10).",
|
|
31102
|
+
["alice", "timezone"],
|
|
31103
|
+
"aest",
|
|
31104
|
+
1,
|
|
31105
|
+
26
|
|
31106
|
+
)
|
|
31107
|
+
],
|
|
31108
|
+
skills: []
|
|
31109
|
+
}
|
|
31110
|
+
];
|
|
31111
|
+
var staleTrapTasks = [
|
|
31112
|
+
{
|
|
31113
|
+
id: "stale-ci-provider",
|
|
31114
|
+
family: "stale-memory-trap",
|
|
31115
|
+
prompt: "Which CI provider does Acme use?",
|
|
31116
|
+
scope: SCOPE_ACME,
|
|
31117
|
+
subjectKeywords: ["ci", "provider", "acme"],
|
|
31118
|
+
expectedAnswer: "github-actions",
|
|
31119
|
+
shouldRecallId: "fact:acme-ci-corrected",
|
|
31120
|
+
shouldExcludeIds: ["fact:acme-ci-stale"],
|
|
31121
|
+
memoryItems: [
|
|
31122
|
+
{
|
|
31123
|
+
id: "fact:acme-ci-stale",
|
|
31124
|
+
category: "fact",
|
|
31125
|
+
scope: SCOPE_ACME,
|
|
31126
|
+
status: "superseded",
|
|
31127
|
+
supersededBy: "fact:acme-ci-corrected",
|
|
31128
|
+
content: "Acme uses CircleCI for continuous integration. CircleCI is configured with the orb for builds. CircleCI pipelines run on every push.",
|
|
31129
|
+
subjectKeywords: ["ci", "provider", "acme"],
|
|
31130
|
+
answerToken: "circleci",
|
|
31131
|
+
tokens: 70,
|
|
31132
|
+
turn: 1
|
|
31133
|
+
},
|
|
31134
|
+
{
|
|
31135
|
+
id: "fact:acme-ci-corrected",
|
|
31136
|
+
category: "correction",
|
|
31137
|
+
scope: SCOPE_ACME,
|
|
31138
|
+
status: "active",
|
|
31139
|
+
content: "Correction: Acme migrated to GitHub Actions for CI.",
|
|
31140
|
+
subjectKeywords: ["ci", "provider", "acme"],
|
|
31141
|
+
answerToken: "github-actions",
|
|
31142
|
+
tokens: 22,
|
|
31143
|
+
turn: 8
|
|
31144
|
+
}
|
|
31145
|
+
],
|
|
31146
|
+
skills: []
|
|
31147
|
+
},
|
|
31148
|
+
{
|
|
31149
|
+
id: "stale-feature-flag",
|
|
31150
|
+
family: "stale-memory-trap",
|
|
31151
|
+
prompt: "Is the Acme dark-mode flag enabled in production?",
|
|
31152
|
+
scope: SCOPE_ACME,
|
|
31153
|
+
subjectKeywords: ["dark-mode", "flag", "production", "acme"],
|
|
31154
|
+
expectedAnswer: "no",
|
|
31155
|
+
shouldRecallId: "fact:acme-darkmode-corrected",
|
|
31156
|
+
shouldExcludeIds: ["fact:acme-darkmode-stale"],
|
|
31157
|
+
memoryItems: [
|
|
31158
|
+
{
|
|
31159
|
+
id: "fact:acme-darkmode-stale",
|
|
31160
|
+
category: "fact",
|
|
31161
|
+
scope: SCOPE_ACME,
|
|
31162
|
+
status: "superseded",
|
|
31163
|
+
supersededBy: "fact:acme-darkmode-corrected",
|
|
31164
|
+
content: "Dark mode is enabled everywhere in Acme production. The dark-mode flag is on for all users. We shipped dark mode to production last month.",
|
|
31165
|
+
subjectKeywords: ["dark-mode", "flag", "production", "acme"],
|
|
31166
|
+
answerToken: "yes",
|
|
31167
|
+
tokens: 74,
|
|
31168
|
+
turn: 2
|
|
31169
|
+
},
|
|
31170
|
+
{
|
|
31171
|
+
id: "fact:acme-darkmode-corrected",
|
|
31172
|
+
category: "correction",
|
|
31173
|
+
scope: SCOPE_ACME,
|
|
31174
|
+
status: "active",
|
|
31175
|
+
content: "Correction: the Acme dark-mode flag was rolled back to off in production.",
|
|
31176
|
+
subjectKeywords: ["dark-mode", "flag", "production", "acme"],
|
|
31177
|
+
answerToken: "no",
|
|
31178
|
+
tokens: 26,
|
|
31179
|
+
turn: 9
|
|
31180
|
+
}
|
|
31181
|
+
],
|
|
31182
|
+
skills: []
|
|
31183
|
+
}
|
|
31184
|
+
];
|
|
31185
|
+
var wrongScopeTasks = [
|
|
31186
|
+
{
|
|
31187
|
+
id: "scope-acme-vs-beta-db",
|
|
31188
|
+
family: "wrong-scope-trap",
|
|
31189
|
+
prompt: "Which database does Acme use?",
|
|
31190
|
+
scope: SCOPE_ACME,
|
|
31191
|
+
subjectKeywords: ["database", "acme"],
|
|
31192
|
+
expectedAnswer: "postgres",
|
|
31193
|
+
shouldRecallId: "fact:acme-db",
|
|
31194
|
+
shouldExcludeIds: ["fact:beta-db"],
|
|
31195
|
+
memoryItems: [
|
|
31196
|
+
fact2(
|
|
31197
|
+
"fact:acme-db",
|
|
31198
|
+
SCOPE_ACME,
|
|
31199
|
+
"Acme uses Postgres for its primary database.",
|
|
31200
|
+
["database", "acme"],
|
|
31201
|
+
"postgres",
|
|
31202
|
+
5,
|
|
31203
|
+
24
|
|
31204
|
+
),
|
|
31205
|
+
fact2(
|
|
31206
|
+
"fact:beta-db",
|
|
31207
|
+
SCOPE_BETA,
|
|
31208
|
+
"The Beta project database is MySQL with a read replica. MySQL is tuned for the Beta workload.",
|
|
31209
|
+
["database", "acme"],
|
|
31210
|
+
"mysql",
|
|
31211
|
+
6,
|
|
31212
|
+
66,
|
|
31213
|
+
{ wrongScope: true }
|
|
31214
|
+
)
|
|
31215
|
+
],
|
|
31216
|
+
skills: []
|
|
31217
|
+
},
|
|
31218
|
+
{
|
|
31219
|
+
id: "scope-acme-vs-beta-cache",
|
|
31220
|
+
family: "wrong-scope-trap",
|
|
31221
|
+
prompt: "What cache does Acme use?",
|
|
31222
|
+
scope: SCOPE_ACME,
|
|
31223
|
+
subjectKeywords: ["cache", "acme"],
|
|
31224
|
+
expectedAnswer: "redis",
|
|
31225
|
+
shouldRecallId: "fact:acme-cache",
|
|
31226
|
+
shouldExcludeIds: ["fact:beta-cache"],
|
|
31227
|
+
memoryItems: [
|
|
31228
|
+
fact2(
|
|
31229
|
+
"fact:acme-cache",
|
|
31230
|
+
SCOPE_ACME,
|
|
31231
|
+
"Acme uses Redis for caching.",
|
|
31232
|
+
["cache", "acme"],
|
|
31233
|
+
"redis",
|
|
31234
|
+
3,
|
|
31235
|
+
20
|
|
31236
|
+
),
|
|
31237
|
+
fact2(
|
|
31238
|
+
"fact:beta-cache",
|
|
31239
|
+
SCOPE_BETA,
|
|
31240
|
+
"Beta uses Memcached for its cache layer. Memcached is sharded across the Beta fleet.",
|
|
31241
|
+
["cache", "acme"],
|
|
31242
|
+
"memcached",
|
|
31243
|
+
4,
|
|
31244
|
+
60,
|
|
31245
|
+
{ wrongScope: true }
|
|
31246
|
+
)
|
|
31247
|
+
],
|
|
31248
|
+
skills: []
|
|
31249
|
+
}
|
|
31250
|
+
];
|
|
31251
|
+
var skillPositiveTasks = [
|
|
31252
|
+
{
|
|
31253
|
+
id: "skill-deploy-gateway",
|
|
31254
|
+
family: "skill-positive",
|
|
31255
|
+
prompt: "Let's deploy the gateway to production now.",
|
|
31256
|
+
scope: SCOPE_ACME,
|
|
31257
|
+
subjectKeywords: ["deploy", "gateway", "production"],
|
|
31258
|
+
expectedAnswer: "run-deploy-checks-then-tag",
|
|
31259
|
+
shouldUseSkillId: "skill:deploy-gateway",
|
|
31260
|
+
shouldExcludeIds: [],
|
|
31261
|
+
memoryItems: [
|
|
31262
|
+
fact2(
|
|
31263
|
+
"decision:deploy-runbook",
|
|
31264
|
+
SCOPE_ACME,
|
|
31265
|
+
"There is a production gateway deploy runbook.",
|
|
31266
|
+
["deploy", "gateway", "production"],
|
|
31267
|
+
"runbook-exists",
|
|
31268
|
+
7,
|
|
31269
|
+
22
|
|
31270
|
+
)
|
|
31271
|
+
],
|
|
31272
|
+
skills: [DEPLOY_GATEWAY_SKILL]
|
|
31273
|
+
},
|
|
31274
|
+
{
|
|
31275
|
+
id: "skill-rotate-keys",
|
|
31276
|
+
family: "skill-positive",
|
|
31277
|
+
prompt: "Please rotate the Acme api keys.",
|
|
31278
|
+
scope: SCOPE_ACME,
|
|
31279
|
+
subjectKeywords: ["rotate", "api", "keys"],
|
|
31280
|
+
expectedAnswer: "dual-load-then-retire",
|
|
31281
|
+
shouldUseSkillId: "skill:rotate-api-keys",
|
|
31282
|
+
shouldExcludeIds: [],
|
|
31283
|
+
memoryItems: [
|
|
31284
|
+
fact2(
|
|
31285
|
+
"decision:key-rotation-policy",
|
|
31286
|
+
SCOPE_ACME,
|
|
31287
|
+
"Acme has an api key rotation procedure.",
|
|
31288
|
+
["rotate", "api", "keys"],
|
|
31289
|
+
"rotation-policy-exists",
|
|
31290
|
+
6,
|
|
31291
|
+
20
|
|
31292
|
+
)
|
|
31293
|
+
],
|
|
31294
|
+
skills: [ROTATE_KEYS_SKILL]
|
|
31295
|
+
}
|
|
31296
|
+
];
|
|
31297
|
+
var skillNegativeTasks = [
|
|
31298
|
+
{
|
|
31299
|
+
id: "skill-deploy-gateway-question",
|
|
31300
|
+
family: "skill-negative",
|
|
31301
|
+
prompt: "What is our usual process for gateway deploys?",
|
|
31302
|
+
scope: SCOPE_ACME,
|
|
31303
|
+
subjectKeywords: ["deploy", "gateway"],
|
|
31304
|
+
expectedAnswer: "describe-runbook",
|
|
31305
|
+
shouldNotUseSkillId: "skill:deploy-gateway",
|
|
31306
|
+
shouldRecallId: "fact:deploy-process-note",
|
|
31307
|
+
shouldExcludeIds: [],
|
|
31308
|
+
memoryItems: [
|
|
31309
|
+
fact2(
|
|
31310
|
+
"fact:deploy-process-note",
|
|
31311
|
+
SCOPE_ACME,
|
|
31312
|
+
"The gateway deploy process is documented in the runbook.",
|
|
31313
|
+
["deploy", "gateway"],
|
|
31314
|
+
"describe-runbook",
|
|
31315
|
+
7,
|
|
31316
|
+
22
|
|
31317
|
+
)
|
|
31318
|
+
],
|
|
31319
|
+
skills: [DEPLOY_GATEWAY_SKILL]
|
|
31320
|
+
},
|
|
31321
|
+
{
|
|
31322
|
+
id: "skill-rotate-keys-audit",
|
|
31323
|
+
family: "skill-negative",
|
|
31324
|
+
prompt: "Audit the Acme api keys for leakage.",
|
|
31325
|
+
scope: SCOPE_ACME,
|
|
31326
|
+
subjectKeywords: ["api", "keys"],
|
|
31327
|
+
expectedAnswer: "audit-only-no-rotation",
|
|
31328
|
+
shouldNotUseSkillId: "skill:rotate-api-keys",
|
|
31329
|
+
shouldExcludeIds: [],
|
|
31330
|
+
memoryItems: [
|
|
31331
|
+
fact2(
|
|
31332
|
+
"fact:acme-key-audit",
|
|
31333
|
+
SCOPE_ACME,
|
|
31334
|
+
"Acme key audits are read-only; no rotation.",
|
|
31335
|
+
["api", "keys", "audit"],
|
|
31336
|
+
"audit-only-no-rotation",
|
|
31337
|
+
5,
|
|
31338
|
+
24
|
|
31339
|
+
)
|
|
31340
|
+
],
|
|
31341
|
+
skills: [ROTATE_KEYS_SKILL]
|
|
31342
|
+
}
|
|
31343
|
+
];
|
|
31344
|
+
var askNeededTasks = [
|
|
31345
|
+
{
|
|
31346
|
+
id: "ask-which-project-deploy",
|
|
31347
|
+
family: "ask-needed",
|
|
31348
|
+
prompt: "Go ahead and deploy it.",
|
|
31349
|
+
scope: SCOPE_ACME,
|
|
31350
|
+
subjectKeywords: ["deploy"],
|
|
31351
|
+
expectedAnswer: "ask:which-target",
|
|
31352
|
+
shouldAsk: true,
|
|
31353
|
+
shouldExcludeIds: [],
|
|
31354
|
+
memoryItems: [
|
|
31355
|
+
{
|
|
31356
|
+
id: "boundary:confirm-deploy-target",
|
|
31357
|
+
category: "boundary",
|
|
31358
|
+
scope: SCOPE_ACME,
|
|
31359
|
+
status: "active",
|
|
31360
|
+
content: "Boundary: confirm the deploy target and project before any production deploy.",
|
|
31361
|
+
subjectKeywords: ["deploy", "boundary"],
|
|
31362
|
+
tokens: 26,
|
|
31363
|
+
turn: 10
|
|
31364
|
+
}
|
|
31365
|
+
],
|
|
31366
|
+
skills: []
|
|
31367
|
+
},
|
|
31368
|
+
{
|
|
31369
|
+
id: "ask-which-user-delete",
|
|
31370
|
+
family: "ask-needed",
|
|
31371
|
+
prompt: "Delete the user record.",
|
|
31372
|
+
scope: SCOPE_ALICE,
|
|
31373
|
+
subjectKeywords: ["delete", "user"],
|
|
31374
|
+
expectedAnswer: "ask:which-user",
|
|
31375
|
+
shouldAsk: true,
|
|
31376
|
+
shouldExcludeIds: [],
|
|
31377
|
+
memoryItems: [
|
|
31378
|
+
{
|
|
31379
|
+
id: "boundary:confirm-delete-target",
|
|
31380
|
+
category: "boundary",
|
|
31381
|
+
scope: SCOPE_ALICE,
|
|
31382
|
+
status: "active",
|
|
31383
|
+
content: "Boundary: confirm which user record before any destructive delete.",
|
|
31384
|
+
subjectKeywords: ["delete", "user", "boundary"],
|
|
31385
|
+
tokens: 24,
|
|
31386
|
+
turn: 11
|
|
31387
|
+
}
|
|
31388
|
+
],
|
|
31389
|
+
skills: []
|
|
31390
|
+
}
|
|
31391
|
+
];
|
|
31392
|
+
var actWhenEnoughTasks = [
|
|
31393
|
+
{
|
|
31394
|
+
id: "act-self-contained-greeting",
|
|
31395
|
+
family: "act-when-enough",
|
|
31396
|
+
prompt: "Say hello to the Acme team.",
|
|
31397
|
+
scope: SCOPE_ACME,
|
|
31398
|
+
subjectKeywords: ["hello", "acme"],
|
|
31399
|
+
expectedAnswer: "hello-acme",
|
|
31400
|
+
shouldAsk: false,
|
|
31401
|
+
shouldExcludeIds: [],
|
|
31402
|
+
memoryItems: [],
|
|
31403
|
+
skills: []
|
|
31404
|
+
},
|
|
31405
|
+
{
|
|
31406
|
+
id: "act-self-contained-summarize",
|
|
31407
|
+
family: "act-when-enough",
|
|
31408
|
+
prompt: "Summarize: the sky is blue.",
|
|
31409
|
+
scope: SCOPE_ACME,
|
|
31410
|
+
subjectKeywords: ["summarize", "sky"],
|
|
31411
|
+
expectedAnswer: "the-sky-is-blue",
|
|
31412
|
+
shouldAsk: false,
|
|
31413
|
+
shouldExcludeIds: [],
|
|
31414
|
+
memoryItems: [],
|
|
31415
|
+
skills: []
|
|
31416
|
+
},
|
|
31417
|
+
{
|
|
31418
|
+
id: "act-recall-based-status",
|
|
31419
|
+
family: "act-when-enough",
|
|
31420
|
+
prompt: "Give me the Acme production status, no need to confirm.",
|
|
31421
|
+
scope: SCOPE_ACME,
|
|
31422
|
+
subjectKeywords: ["production", "status", "acme"],
|
|
31423
|
+
expectedAnswer: "green",
|
|
31424
|
+
shouldAsk: false,
|
|
31425
|
+
shouldRecallId: "fact:acme-status",
|
|
31426
|
+
shouldExcludeIds: [],
|
|
31427
|
+
memoryItems: [
|
|
31428
|
+
fact2(
|
|
31429
|
+
"fact:acme-status",
|
|
31430
|
+
SCOPE_ACME,
|
|
31431
|
+
"Acme production status is green.",
|
|
31432
|
+
["production", "status", "acme"],
|
|
31433
|
+
"green",
|
|
31434
|
+
4,
|
|
31435
|
+
24
|
|
31436
|
+
)
|
|
31437
|
+
],
|
|
31438
|
+
skills: []
|
|
31439
|
+
}
|
|
31440
|
+
];
|
|
31441
|
+
var BOUNDED_MEMORY_FIXTURE = [
|
|
31442
|
+
...recallNeededTasks,
|
|
31443
|
+
...staleTrapTasks,
|
|
31444
|
+
...wrongScopeTasks,
|
|
31445
|
+
...skillPositiveTasks,
|
|
31446
|
+
...skillNegativeTasks,
|
|
31447
|
+
...askNeededTasks,
|
|
31448
|
+
...actWhenEnoughTasks
|
|
31449
|
+
];
|
|
31450
|
+
var BOUNDED_MEMORY_SMOKE_FIXTURE = [
|
|
31451
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "recall-framework-choice"),
|
|
31452
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "stale-ci-provider"),
|
|
31453
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "scope-acme-vs-beta-db"),
|
|
31454
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "skill-deploy-gateway"),
|
|
31455
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "skill-deploy-gateway-question"),
|
|
31456
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "ask-which-project-deploy"),
|
|
31457
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "act-self-contained-greeting"),
|
|
31458
|
+
// Extra coverage: a second recall + an act-recall so quick mode sees both
|
|
31459
|
+
// self-contained and recall-driven act-when-enough paths.
|
|
31460
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "recall-meeting-day"),
|
|
31461
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "act-recall-based-status"),
|
|
31462
|
+
BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "stale-feature-flag")
|
|
31463
|
+
];
|
|
31464
|
+
function fixtureHash(tasks) {
|
|
31465
|
+
const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
|
|
31466
|
+
const payload = source.map(
|
|
31467
|
+
(t) => [
|
|
31468
|
+
t.id,
|
|
31469
|
+
t.family,
|
|
31470
|
+
t.expectedAnswer,
|
|
31471
|
+
t.scope,
|
|
31472
|
+
t.shouldAsk === void 0 ? "-" : String(t.shouldAsk),
|
|
31473
|
+
t.memoryItems.map((m) => `${m.id}:${m.status}:${m.scope}`).join(","),
|
|
31474
|
+
t.skills.map((s) => s.id).join(",")
|
|
31475
|
+
].join("|")
|
|
31476
|
+
).join("\n");
|
|
31477
|
+
return createHash10("sha256").update(payload, "utf8").digest("hex");
|
|
31478
|
+
}
|
|
31479
|
+
|
|
31480
|
+
// src/benchmarks/remnic/bounded-memory-contracts/agent.ts
|
|
31481
|
+
var BOUNDED_MEMORY_CONTRACT = {
|
|
31482
|
+
id: "bounded-memory-default",
|
|
31483
|
+
description: "Fresh bounded prompt assembled from typed, scoped, citable memory slots. No historical raw transcript is appended.",
|
|
31484
|
+
maxTotalTokens: 320,
|
|
31485
|
+
slots: [
|
|
31486
|
+
{
|
|
31487
|
+
id: "active_scope",
|
|
31488
|
+
memoryCategories: ["fact", "decision"],
|
|
31489
|
+
maxItems: 4,
|
|
31490
|
+
required: true,
|
|
31491
|
+
excludeIfSuperseded: true,
|
|
31492
|
+
requireCitation: true
|
|
31493
|
+
},
|
|
31494
|
+
{
|
|
31495
|
+
id: "relevant_facts",
|
|
31496
|
+
memoryCategories: ["fact", "decision", "preference", "principle", "entity"],
|
|
31497
|
+
maxItems: 6,
|
|
31498
|
+
required: true,
|
|
31499
|
+
excludeIfSuperseded: true,
|
|
31500
|
+
requireCitation: true
|
|
31501
|
+
},
|
|
31502
|
+
{
|
|
31503
|
+
id: "conflicts_and_supersessions",
|
|
31504
|
+
memoryCategories: ["correction"],
|
|
31505
|
+
maxItems: 3,
|
|
31506
|
+
required: false,
|
|
31507
|
+
excludeIfSuperseded: false,
|
|
31508
|
+
requireCitation: true
|
|
31509
|
+
},
|
|
31510
|
+
{
|
|
31511
|
+
id: "boundaries",
|
|
31512
|
+
memoryCategories: ["boundary"],
|
|
31513
|
+
maxItems: 2,
|
|
31514
|
+
required: false,
|
|
31515
|
+
excludeIfSuperseded: true,
|
|
31516
|
+
requireCitation: true
|
|
31517
|
+
}
|
|
31518
|
+
]
|
|
31519
|
+
};
|
|
31520
|
+
function toPackItem(item, exposedMetadata) {
|
|
31521
|
+
return {
|
|
31522
|
+
itemId: item.id,
|
|
31523
|
+
category: item.category,
|
|
31524
|
+
scope: item.scope,
|
|
31525
|
+
status: item.status,
|
|
31526
|
+
content: item.content,
|
|
31527
|
+
subjectKeywords: item.subjectKeywords,
|
|
31528
|
+
citation: exposedMetadata ? `mem:${item.id}` : "",
|
|
31529
|
+
tokens: item.tokens,
|
|
31530
|
+
superseded: item.status === "superseded",
|
|
31531
|
+
wrongScope: item.wrongScope === true,
|
|
31532
|
+
exposedMetadata
|
|
31533
|
+
};
|
|
31534
|
+
}
|
|
31535
|
+
function keywordOverlap(a, b) {
|
|
31536
|
+
const setB = new Set(b);
|
|
31537
|
+
let n = 0;
|
|
31538
|
+
for (const kw of a) {
|
|
31539
|
+
if (setB.has(kw)) n += 1;
|
|
31540
|
+
}
|
|
31541
|
+
return n;
|
|
31542
|
+
}
|
|
31543
|
+
function assemblePack(task, condition, contract, injectSkills) {
|
|
31544
|
+
const fullTranscriptTokens = task.memoryItems.reduce((s, m) => s + m.tokens, 0);
|
|
31545
|
+
if (condition === "no-memory") {
|
|
31546
|
+
return {
|
|
31547
|
+
condition,
|
|
31548
|
+
slots: [],
|
|
31549
|
+
transcriptBlock: null,
|
|
31550
|
+
boundaryItem: null,
|
|
31551
|
+
totalTokens: 0,
|
|
31552
|
+
fullTranscriptTokens
|
|
31553
|
+
};
|
|
31554
|
+
}
|
|
31555
|
+
if (condition === "raw-transcript") {
|
|
31556
|
+
const ordered = task.memoryItems.slice().sort((a, b) => b.turn - a.turn);
|
|
31557
|
+
const items = [];
|
|
31558
|
+
let budget2 = contract.maxTotalTokens;
|
|
31559
|
+
for (const m of ordered) {
|
|
31560
|
+
if (budget2 <= 0) break;
|
|
31561
|
+
if (m.tokens > budget2) continue;
|
|
31562
|
+
items.push(toPackItem(m, false));
|
|
31563
|
+
budget2 -= m.tokens;
|
|
31564
|
+
}
|
|
31565
|
+
const transcriptBlock = items.map((it) => `- [turn ${task.memoryItems.find((m) => m.id === it.itemId).turn}] ${it.content}`).join("\n");
|
|
31566
|
+
return {
|
|
31567
|
+
condition,
|
|
31568
|
+
slots: [{ id: "transcript", items }],
|
|
31569
|
+
transcriptBlock,
|
|
31570
|
+
// Raw transcript buries any boundary prose; it is NOT surfaced as a
|
|
31571
|
+
// structured boundary item, so the agent cannot reliably act on it.
|
|
31572
|
+
boundaryItem: null,
|
|
31573
|
+
totalTokens: items.reduce((s, it) => s + it.tokens, 0),
|
|
31574
|
+
fullTranscriptTokens
|
|
31575
|
+
};
|
|
31576
|
+
}
|
|
31577
|
+
const inScope = task.memoryItems.filter((m) => m.scope === task.scope);
|
|
31578
|
+
const boundaryFixture = inScope.find(
|
|
31579
|
+
(m) => m.category === "boundary" && m.status === "active"
|
|
31580
|
+
);
|
|
31581
|
+
const boundaryItem = boundaryFixture ? toPackItem(boundaryFixture, true) : null;
|
|
31582
|
+
const rankedCandidates = inScope.filter((m) => m.status !== "pending_review").slice().sort((a, b) => {
|
|
31583
|
+
const oa = keywordOverlap(a.subjectKeywords, task.subjectKeywords);
|
|
31584
|
+
const ob = keywordOverlap(b.subjectKeywords, task.subjectKeywords);
|
|
31585
|
+
if (ob !== oa) return ob - oa;
|
|
31586
|
+
return b.turn - a.turn;
|
|
31587
|
+
});
|
|
31588
|
+
const alreadyPicked = /* @__PURE__ */ new Set();
|
|
31589
|
+
let budget = contract.maxTotalTokens;
|
|
31590
|
+
if (boundaryFixture) {
|
|
31591
|
+
alreadyPicked.add(boundaryFixture.id);
|
|
31592
|
+
budget -= boundaryFixture.tokens;
|
|
31593
|
+
}
|
|
31594
|
+
const slots = contract.slots.map((slot) => {
|
|
31595
|
+
const items = [];
|
|
31596
|
+
for (const m of rankedCandidates) {
|
|
31597
|
+
if (budget <= 0) break;
|
|
31598
|
+
if (alreadyPicked.has(m.id)) continue;
|
|
31599
|
+
if (!slot.memoryCategories.includes(m.category)) continue;
|
|
31600
|
+
if (slot.excludeIfSuperseded && m.status === "superseded") continue;
|
|
31601
|
+
if (items.length >= slot.maxItems) break;
|
|
31602
|
+
if (m.tokens > budget) continue;
|
|
31603
|
+
items.push(toPackItem(m, true));
|
|
31604
|
+
alreadyPicked.add(m.id);
|
|
31605
|
+
budget -= m.tokens;
|
|
31606
|
+
}
|
|
31607
|
+
return { id: slot.id, items };
|
|
31608
|
+
});
|
|
31609
|
+
let totalTokens = contract.maxTotalTokens - budget;
|
|
31610
|
+
if (condition === "typed-plus-skills" && injectSkills.length > 0) {
|
|
31611
|
+
const skillItems = [];
|
|
31612
|
+
for (const skill of injectSkills) {
|
|
31613
|
+
if (totalTokens + skill.tokens > contract.maxTotalTokens) break;
|
|
31614
|
+
skillItems.push({
|
|
31615
|
+
itemId: skill.id,
|
|
31616
|
+
category: "skill",
|
|
31617
|
+
scope: task.scope,
|
|
31618
|
+
status: skill.status,
|
|
31619
|
+
content: `${skill.title}: ${skill.steps.join(" \u2192 ")}`,
|
|
31620
|
+
subjectKeywords: skill.appliesWhen,
|
|
31621
|
+
citation: `skill:${skill.id}`,
|
|
31622
|
+
tokens: skill.tokens,
|
|
31623
|
+
superseded: false,
|
|
31624
|
+
wrongScope: false,
|
|
31625
|
+
exposedMetadata: true
|
|
31626
|
+
});
|
|
31627
|
+
totalTokens += skill.tokens;
|
|
31628
|
+
}
|
|
31629
|
+
if (skillItems.length > 0) {
|
|
31630
|
+
slots.push({ id: "triggered_skills", items: skillItems });
|
|
31631
|
+
}
|
|
31632
|
+
}
|
|
31633
|
+
return {
|
|
31634
|
+
condition,
|
|
31635
|
+
slots,
|
|
31636
|
+
transcriptBlock: null,
|
|
31637
|
+
boundaryItem,
|
|
31638
|
+
totalTokens,
|
|
31639
|
+
fullTranscriptTokens
|
|
31640
|
+
};
|
|
31641
|
+
}
|
|
31642
|
+
function classifySkillTrigger(skill, task) {
|
|
31643
|
+
const taskTokens = tokenize4(task.prompt + " " + task.subjectKeywords.join(" "));
|
|
31644
|
+
const taskSet = new Set(taskTokens);
|
|
31645
|
+
const appliesHits = skill.appliesWhen.filter((k) => taskSet.has(k));
|
|
31646
|
+
const blocksHits = skill.doesNotApplyWhen.filter((k) => taskSet.has(k));
|
|
31647
|
+
const considered = appliesHits.length > 0 || blocksHits.length > 0;
|
|
31648
|
+
if (appliesHits.length > 0 && blocksHits.length === 0) {
|
|
31649
|
+
return {
|
|
31650
|
+
considered: true,
|
|
31651
|
+
injected: true,
|
|
31652
|
+
reason: `appliesWhen matched [${appliesHits.join(", ")}]; no doesNotApplyWhen hit`
|
|
31653
|
+
};
|
|
31654
|
+
}
|
|
31655
|
+
if (blocksHits.length > 0) {
|
|
31656
|
+
return {
|
|
31657
|
+
considered: true,
|
|
31658
|
+
injected: false,
|
|
31659
|
+
reason: `blocked by doesNotApplyWhen [${blocksHits.join(", ")}]`
|
|
31660
|
+
};
|
|
31661
|
+
}
|
|
31662
|
+
return { considered, injected: false, reason: "no trigger overlap" };
|
|
31663
|
+
}
|
|
31664
|
+
function tokenize4(text) {
|
|
31665
|
+
return text.toLowerCase().replace(/[^a-z0-9\s-]/g, " ").split(/\s+/).filter((t) => t.length > 0);
|
|
31666
|
+
}
|
|
31667
|
+
function rankCandidates(pack, task, sourceItems) {
|
|
31668
|
+
const candidates = [];
|
|
31669
|
+
const exposedMetadata = pack.condition !== "raw-transcript";
|
|
31670
|
+
for (const slot of pack.slots) {
|
|
31671
|
+
for (const it of slot.items) {
|
|
31672
|
+
const overlap = keywordOverlap(it.subjectKeywords, task.subjectKeywords);
|
|
31673
|
+
if (overlap === 0) continue;
|
|
31674
|
+
const source = sourceItems.find((m) => m.id === it.itemId);
|
|
31675
|
+
const turn = source ? source.turn : it.tokens;
|
|
31676
|
+
candidates.push({ item: it, overlap, tokens: it.tokens, turn });
|
|
31677
|
+
}
|
|
31678
|
+
}
|
|
31679
|
+
if (candidates.length === 0) return null;
|
|
31680
|
+
candidates.sort((a, b) => {
|
|
31681
|
+
if (b.overlap !== a.overlap) return b.overlap - a.overlap;
|
|
31682
|
+
return exposedMetadata ? b.turn - a.turn : b.tokens - a.tokens;
|
|
31683
|
+
});
|
|
31684
|
+
return candidates[0].item;
|
|
31685
|
+
}
|
|
31686
|
+
function simulateAgent(task, pack, injectedSkills) {
|
|
31687
|
+
const allItems = task.memoryItems;
|
|
31688
|
+
const consideredSkillIds = [];
|
|
31689
|
+
const usedSkillIds = [];
|
|
31690
|
+
if (task.shouldAsk === true) {
|
|
31691
|
+
const asked = pack.boundaryItem !== null;
|
|
31692
|
+
return {
|
|
31693
|
+
answer: asked ? task.expectedAnswer : "act-without-confirmation",
|
|
31694
|
+
askedClarification: asked,
|
|
31695
|
+
acted: !asked,
|
|
31696
|
+
recalledItemIds: [],
|
|
31697
|
+
usedSkillIds,
|
|
31698
|
+
consideredSkillIds,
|
|
31699
|
+
wrongScopeLeakedIds: [],
|
|
31700
|
+
staleLeakedIds: []
|
|
31701
|
+
};
|
|
31702
|
+
}
|
|
31703
|
+
if (task.shouldAsk === false) {
|
|
31704
|
+
const best2 = rankCandidates(pack, task, allItems);
|
|
31705
|
+
const answer2 = best2?.itemId ? deriveAnswer(best2.itemId, allItems, injectedSkills, task) : SELF_CONTAINED_ANSWERS[task.id] ?? "unknown";
|
|
31706
|
+
return {
|
|
31707
|
+
answer: answer2,
|
|
31708
|
+
askedClarification: false,
|
|
31709
|
+
acted: true,
|
|
31710
|
+
recalledItemIds: best2 ? [best2.itemId] : [],
|
|
31711
|
+
usedSkillIds,
|
|
31712
|
+
consideredSkillIds,
|
|
31713
|
+
wrongScopeLeakedIds: best2?.wrongScope ? [best2.itemId] : [],
|
|
31714
|
+
staleLeakedIds: best2?.superseded ? [best2.itemId] : []
|
|
31715
|
+
};
|
|
31716
|
+
}
|
|
31717
|
+
for (const skill of injectedSkills) {
|
|
31718
|
+
consideredSkillIds.push(skill.id);
|
|
31719
|
+
usedSkillIds.push(skill.id);
|
|
31720
|
+
}
|
|
31721
|
+
const best = rankCandidates(pack, task, allItems);
|
|
31722
|
+
if (task.family === "skill-positive" && injectedSkills.length > 0) {
|
|
31723
|
+
const answer2 = deriveAnswer(task.shouldUseSkillId ?? "", allItems, injectedSkills, task);
|
|
31724
|
+
return {
|
|
31725
|
+
answer: answer2,
|
|
31726
|
+
askedClarification: false,
|
|
31727
|
+
acted: true,
|
|
31728
|
+
recalledItemIds: best ? [best.itemId] : [],
|
|
31729
|
+
usedSkillIds,
|
|
31730
|
+
consideredSkillIds,
|
|
31731
|
+
wrongScopeLeakedIds: best?.wrongScope ? [best.itemId] : [],
|
|
31732
|
+
staleLeakedIds: best?.superseded ? [best.itemId] : []
|
|
31733
|
+
};
|
|
31734
|
+
}
|
|
31735
|
+
const answer = best ? deriveAnswer(best.itemId, allItems, injectedSkills, task) : SELF_CONTAINED_ANSWERS[task.id] ?? "unknown";
|
|
31736
|
+
return {
|
|
31737
|
+
answer,
|
|
31738
|
+
askedClarification: false,
|
|
31739
|
+
acted: true,
|
|
31740
|
+
recalledItemIds: best ? [best.itemId] : [],
|
|
31741
|
+
usedSkillIds,
|
|
31742
|
+
consideredSkillIds,
|
|
31743
|
+
wrongScopeLeakedIds: best?.wrongScope ? [best.itemId] : [],
|
|
31744
|
+
staleLeakedIds: best?.superseded ? [best.itemId] : []
|
|
31745
|
+
};
|
|
31746
|
+
}
|
|
31747
|
+
function deriveAnswer(itemId, items, skills, task) {
|
|
31748
|
+
if (task.family === "skill-positive") {
|
|
31749
|
+
const skill = skills.find((sx) => sx.id === task.shouldUseSkillId);
|
|
31750
|
+
if (skill) {
|
|
31751
|
+
return SKILL_ANSWER_TOKENS[skill.id] ?? "procedure-applied";
|
|
31752
|
+
}
|
|
31753
|
+
}
|
|
31754
|
+
const item = items.find((m) => m.id === itemId);
|
|
31755
|
+
return item?.answerToken ?? "unknown";
|
|
31756
|
+
}
|
|
31757
|
+
var SKILL_ANSWER_TOKENS = {
|
|
31758
|
+
"skill:deploy-gateway": "run-deploy-checks-then-tag",
|
|
31759
|
+
"skill:rotate-api-keys": "dual-load-then-retire"
|
|
31760
|
+
};
|
|
31761
|
+
var SELF_CONTAINED_ANSWERS = {
|
|
31762
|
+
"act-self-contained-greeting": "hello-acme",
|
|
31763
|
+
"act-self-contained-summarize": "the-sky-is-blue"
|
|
31764
|
+
};
|
|
31765
|
+
|
|
31766
|
+
// src/benchmarks/remnic/bounded-memory-contracts/scoring.ts
|
|
31767
|
+
function normalizeAnswer(value) {
|
|
31768
|
+
return value.trim().toLowerCase().replace(/\s+/g, "-");
|
|
31769
|
+
}
|
|
31770
|
+
function visiblePackItems(pack) {
|
|
31771
|
+
const out = [];
|
|
31772
|
+
for (const slot of pack.slots) {
|
|
31773
|
+
for (const it of slot.items) {
|
|
31774
|
+
out.push({
|
|
31775
|
+
itemId: it.itemId,
|
|
31776
|
+
citation: it.citation,
|
|
31777
|
+
superseded: it.superseded,
|
|
31778
|
+
wrongScope: it.wrongScope
|
|
31779
|
+
});
|
|
31780
|
+
}
|
|
31781
|
+
}
|
|
31782
|
+
if (pack.boundaryItem) {
|
|
31783
|
+
out.push({
|
|
31784
|
+
itemId: pack.boundaryItem.itemId,
|
|
31785
|
+
citation: pack.boundaryItem.citation,
|
|
31786
|
+
superseded: pack.boundaryItem.superseded,
|
|
31787
|
+
wrongScope: pack.boundaryItem.wrongScope
|
|
31788
|
+
});
|
|
31789
|
+
}
|
|
31790
|
+
return out;
|
|
31791
|
+
}
|
|
31792
|
+
function scoreTaskPair(task, pack, decision) {
|
|
31793
|
+
const visible = visiblePackItems(pack);
|
|
31794
|
+
const visibleIds = new Set(visible.map((v) => v.itemId));
|
|
31795
|
+
const taskSuccess = normalizeAnswer(decision.answer) === normalizeAnswer(task.expectedAnswer) ? 1 : 0;
|
|
31796
|
+
const isBoundaryTask = task.shouldAsk !== void 0;
|
|
31797
|
+
const shouldAskAccuracy = isBoundaryTask ? decision.askedClarification === task.shouldAsk ? 1 : 0 : 0;
|
|
31798
|
+
const unnecessaryClarification = task.shouldAsk === false ? decision.askedClarification ? 1 : 0 : 0;
|
|
31799
|
+
const actionBoundaryViolation = task.shouldAsk === true ? decision.acted ? 1 : 0 : 0;
|
|
31800
|
+
const hasRecallTarget = task.shouldRecallId !== void 0 || task.family === "skill-positive";
|
|
31801
|
+
let relevantMemoryRecall = 0;
|
|
31802
|
+
if (hasRecallTarget) {
|
|
31803
|
+
if (task.family === "skill-positive") {
|
|
31804
|
+
relevantMemoryRecall = taskSuccess;
|
|
31805
|
+
} else if (task.shouldRecallId) {
|
|
31806
|
+
relevantMemoryRecall = visibleIds.has(task.shouldRecallId) && decision.recalledItemIds.includes(task.shouldRecallId) ? 1 : 0;
|
|
31807
|
+
}
|
|
31808
|
+
}
|
|
31809
|
+
const stalePresent = visible.some((v) => v.superseded);
|
|
31810
|
+
const wrongScopePresent = visible.some((v) => v.wrongScope);
|
|
31811
|
+
const staleMemoryHarm = stalePresent ? 1 : 0;
|
|
31812
|
+
const wrongScopeRetrieval = wrongScopePresent ? 1 : 0;
|
|
31813
|
+
const hasSupersededInTrace = task.memoryItems.some((m) => m.status === "superseded");
|
|
31814
|
+
const supersessionRespected = hasSupersededInTrace ? stalePresent ? 0 : 1 : 1;
|
|
31815
|
+
const citationCoverage = visible.length > 0 ? visible.filter((v) => v.citation.length > 0).length / visible.length : 1;
|
|
31816
|
+
const memoryTokensInjected = pack.totalTokens;
|
|
31817
|
+
const retrievedItemCount = visible.length;
|
|
31818
|
+
const compressionRatio = pack.fullTranscriptTokens > 0 ? pack.fullTranscriptTokens / Math.max(pack.totalTokens, 1) : 1;
|
|
31819
|
+
return {
|
|
31820
|
+
task_success: taskSuccess,
|
|
31821
|
+
should_ask_accuracy: shouldAskAccuracy,
|
|
31822
|
+
unnecessary_clarification_rate: unnecessaryClarification,
|
|
31823
|
+
action_boundary_violation_rate: actionBoundaryViolation,
|
|
31824
|
+
relevant_memory_recall: relevantMemoryRecall,
|
|
31825
|
+
stale_memory_harm_rate: staleMemoryHarm,
|
|
31826
|
+
wrong_scope_retrieval_rate: wrongScopeRetrieval,
|
|
31827
|
+
supersession_respected_rate: supersessionRespected,
|
|
31828
|
+
citation_coverage: citationCoverage,
|
|
31829
|
+
memory_tokens_injected: memoryTokensInjected,
|
|
31830
|
+
retrieved_item_count: retrievedItemCount,
|
|
31831
|
+
compression_ratio_vs_raw_transcript: compressionRatio
|
|
31832
|
+
};
|
|
31833
|
+
}
|
|
31834
|
+
function mean2(values) {
|
|
31835
|
+
if (values.length === 0) return 0;
|
|
31836
|
+
return values.reduce((s, v) => s + v, 0) / values.length;
|
|
31837
|
+
}
|
|
31838
|
+
function aggregateCondition(condition, scored, skillLog) {
|
|
31839
|
+
const taskCount = scored.length;
|
|
31840
|
+
const all = scored.map((s) => s.scores);
|
|
31841
|
+
const taskSuccessRate = mean2(all.map((s) => s.task_success));
|
|
31842
|
+
const boundaryTasks = scored.filter((s) => s.task.shouldAsk !== void 0);
|
|
31843
|
+
const askNeeded = scored.filter((s) => s.task.shouldAsk === true);
|
|
31844
|
+
const actWhenEnough = scored.filter((s) => s.task.shouldAsk === false);
|
|
31845
|
+
const shouldAskAccuracy = mean2(boundaryTasks.map((s) => s.scores.should_ask_accuracy));
|
|
31846
|
+
const unnecessaryClarificationRate = mean2(
|
|
31847
|
+
actWhenEnough.map((s) => s.scores.unnecessary_clarification_rate)
|
|
31848
|
+
);
|
|
31849
|
+
const actionBoundaryViolationRate = mean2(
|
|
31850
|
+
askNeeded.map((s) => s.scores.action_boundary_violation_rate)
|
|
31851
|
+
);
|
|
31852
|
+
const recallTasks = scored.filter(
|
|
31853
|
+
(s) => (s.task.shouldRecallId !== void 0 || s.task.family === "skill-positive") && s.task.family !== "stale-memory-trap" && s.task.family !== "wrong-scope-trap"
|
|
31854
|
+
);
|
|
31855
|
+
const relevantMemoryRecall = mean2(recallTasks.map((s) => s.scores.relevant_memory_recall));
|
|
31856
|
+
const staleTasks = scored.filter((s) => s.task.family === "stale-memory-trap");
|
|
31857
|
+
const scopeTasks = scored.filter((s) => s.task.family === "wrong-scope-trap");
|
|
31858
|
+
const staleMemoryHarmRate = mean2(staleTasks.map((s) => s.scores.stale_memory_harm_rate));
|
|
31859
|
+
const wrongScopeRetrievalRate = mean2(scopeTasks.map((s) => s.scores.wrong_scope_retrieval_rate));
|
|
31860
|
+
const supersessionRespectedRate = mean2(staleTasks.map((s) => s.scores.supersession_respected_rate));
|
|
31861
|
+
const citedTasks = scored.filter((s) => s.scores.retrieved_item_count > 0);
|
|
31862
|
+
const citationCoverage = mean2(citedTasks.map((s) => s.scores.citation_coverage));
|
|
31863
|
+
const meanMemoryTokensInjected = mean2(all.map((s) => s.memory_tokens_injected));
|
|
31864
|
+
const meanRetrievedItemCount = mean2(all.map((s) => s.retrieved_item_count));
|
|
31865
|
+
const compressible = scored.filter((s) => s.task.memoryItems.length > 0);
|
|
31866
|
+
const meanCompressionRatio = mean2(compressible.map((s) => s.scores.compression_ratio_vs_raw_transcript));
|
|
31867
|
+
const considered = skillLog.filter((e) => e.considered);
|
|
31868
|
+
const injected = considered.filter((e) => e.injected);
|
|
31869
|
+
const tp = injected.filter((e) => e.outcome === "helped").length;
|
|
31870
|
+
const fp = injected.filter((e) => e.outcome === "harmed").length;
|
|
31871
|
+
const notInjected = considered.filter((e) => !e.injected);
|
|
31872
|
+
const fn = notInjected.filter((e) => e.outcome === "harmed").length;
|
|
31873
|
+
const tn = notInjected.filter((e) => e.outcome === "irrelevant").length;
|
|
31874
|
+
return {
|
|
31875
|
+
condition,
|
|
31876
|
+
taskCount,
|
|
31877
|
+
taskSuccessRate,
|
|
31878
|
+
shouldAskAccuracy,
|
|
31879
|
+
unnecessaryClarificationRate,
|
|
31880
|
+
actionBoundaryViolationRate,
|
|
31881
|
+
relevantMemoryRecall,
|
|
31882
|
+
staleMemoryHarmRate,
|
|
31883
|
+
wrongScopeRetrievalRate,
|
|
31884
|
+
supersessionRespectedRate,
|
|
31885
|
+
citationCoverage,
|
|
31886
|
+
meanMemoryTokensInjected,
|
|
31887
|
+
meanRetrievedItemCount,
|
|
31888
|
+
meanCompressionRatio,
|
|
31889
|
+
skillTriggerPrecision: tp + fp > 0 ? tp / (tp + fp) : 0,
|
|
31890
|
+
skillTriggerRecall: tp + fn > 0 ? tp / (tp + fn) : 0,
|
|
31891
|
+
skillFalsePositiveRate: fp + tn > 0 ? fp / (fp + tn) : 0,
|
|
31892
|
+
skillFalseNegativeRate: tp + fn > 0 ? fn / (tp + fn) : 0,
|
|
31893
|
+
skillHelpedCount: tp,
|
|
31894
|
+
skillHarmedCount: fp,
|
|
31895
|
+
skillIrrelevantCount: considered.filter((e) => e.outcome === "irrelevant").length
|
|
31896
|
+
};
|
|
31897
|
+
}
|
|
31898
|
+
|
|
31899
|
+
// src/benchmarks/remnic/bounded-memory-contracts/types.ts
|
|
31900
|
+
var BOUNDED_MEMORY_CONDITIONS = [
|
|
31901
|
+
"no-memory",
|
|
31902
|
+
"raw-transcript",
|
|
31903
|
+
"typed-contract",
|
|
31904
|
+
"typed-plus-skills"
|
|
31905
|
+
];
|
|
31906
|
+
var BOUNDED_MEMORY_CONDITION_LABELS = {
|
|
31907
|
+
"no-memory": "C0 no-memory",
|
|
31908
|
+
"raw-transcript": "C1 raw-transcript",
|
|
31909
|
+
"typed-contract": "C2 typed-contract",
|
|
31910
|
+
"typed-plus-skills": "C3 typed-plus-skills"
|
|
31911
|
+
};
|
|
31912
|
+
|
|
31913
|
+
// src/benchmarks/remnic/bounded-memory-contracts/report.ts
|
|
31914
|
+
var CONDITION_ORDER = [
|
|
31915
|
+
"no-memory",
|
|
31916
|
+
"raw-transcript",
|
|
31917
|
+
"typed-contract",
|
|
31918
|
+
"typed-plus-skills"
|
|
31919
|
+
];
|
|
31920
|
+
function pct(value) {
|
|
31921
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
31922
|
+
}
|
|
31923
|
+
function num(value) {
|
|
31924
|
+
return value.toFixed(2);
|
|
31925
|
+
}
|
|
31926
|
+
function renderReportMarkdown(tasks, aggregates) {
|
|
31927
|
+
const lines = [];
|
|
31928
|
+
lines.push("# Bounded Memory Contracts \u2014 Benchmark Report");
|
|
31929
|
+
lines.push("");
|
|
31930
|
+
lines.push("> Generated by `@remnic/bench` (issue #1708). Fully synthetic, offline, deterministic. See the *Safe vs unsupported claims* section before citing any number.");
|
|
31931
|
+
lines.push("");
|
|
31932
|
+
lines.push("## Configuration");
|
|
31933
|
+
lines.push("");
|
|
31934
|
+
lines.push("- **Benchmark**: `bounded-memory-contracts` v1.0.0");
|
|
31935
|
+
lines.push("- **Model / provider**: none (deterministic offline simulation; no LLM called)");
|
|
31936
|
+
lines.push("- **Dataset source**: synthetic smoke fixture (no real user/client data)");
|
|
31937
|
+
lines.push(`- **Task count**: ${tasks.length} across 7 families`);
|
|
31938
|
+
lines.push("- **Conditions**: C0 no-memory \xB7 C1 raw-transcript \xB7 C2 typed-contract \xB7 C3 typed-plus-skills");
|
|
31939
|
+
lines.push("- **Budget**: shared token budget (budget-normalized comparison per the issue's mitigation for an unfair baseline)");
|
|
31940
|
+
lines.push("");
|
|
31941
|
+
lines.push("## Condition summary");
|
|
31942
|
+
lines.push("");
|
|
31943
|
+
lines.push("| Condition | Task success | Recall | Stale harm | Wrong-scope | Supersession respected | Citations | Mean tokens | Compression |");
|
|
31944
|
+
lines.push("|---|---|---|---|---|---|---|---|---|");
|
|
31945
|
+
for (const cond of CONDITION_ORDER) {
|
|
31946
|
+
const a = aggregates[cond];
|
|
31947
|
+
if (!a) continue;
|
|
31948
|
+
lines.push(
|
|
31949
|
+
`| ${BOUNDED_MEMORY_CONDITION_LABELS[cond]} | ${pct(a.taskSuccessRate)} | ${pct(a.relevantMemoryRecall)} | ${pct(a.staleMemoryHarmRate)} | ${pct(a.wrongScopeRetrievalRate)} | ${pct(a.supersessionRespectedRate)} | ${pct(a.citationCoverage)} | ${num(a.meanMemoryTokensInjected)} | ${num(a.meanCompressionRatio)}\xD7 |`
|
|
31950
|
+
);
|
|
31951
|
+
}
|
|
31952
|
+
lines.push("");
|
|
31953
|
+
lines.push("## Boundary behavior (ask vs act)");
|
|
31954
|
+
lines.push("");
|
|
31955
|
+
lines.push("| Condition | Should-ask accuracy | Unnecessary clarification | Boundary violation |");
|
|
31956
|
+
lines.push("|---|---|---|---|");
|
|
31957
|
+
for (const cond of CONDITION_ORDER) {
|
|
31958
|
+
const a = aggregates[cond];
|
|
31959
|
+
if (!a) continue;
|
|
31960
|
+
lines.push(
|
|
31961
|
+
`| ${BOUNDED_MEMORY_CONDITION_LABELS[cond]} | ${pct(a.shouldAskAccuracy)} | ${pct(a.unnecessaryClarificationRate)} | ${pct(a.actionBoundaryViolationRate)} |`
|
|
31962
|
+
);
|
|
31963
|
+
}
|
|
31964
|
+
lines.push("");
|
|
31965
|
+
const c3 = aggregates["typed-plus-skills"];
|
|
31966
|
+
lines.push("## Skill-trigger behavior (C3 only)");
|
|
31967
|
+
lines.push("");
|
|
31968
|
+
if (c3) {
|
|
31969
|
+
lines.push(`- **Trigger precision**: ${pct(c3.skillTriggerPrecision)}`);
|
|
31970
|
+
lines.push(`- **Trigger recall**: ${pct(c3.skillTriggerRecall)}`);
|
|
31971
|
+
lines.push(`- **False-positive rate**: ${pct(c3.skillFalsePositiveRate)}`);
|
|
31972
|
+
lines.push(`- **False-negative rate**: ${pct(c3.skillFalseNegativeRate)}`);
|
|
31973
|
+
lines.push(`- **Helped / harmed / irrelevant**: ${c3.skillHelpedCount} / ${c3.skillHarmedCount} / ${c3.skillIrrelevantCount}`);
|
|
31974
|
+
} else {
|
|
31975
|
+
lines.push("_C3 not run._");
|
|
31976
|
+
}
|
|
31977
|
+
lines.push("");
|
|
31978
|
+
lines.push("## Biggest wins (typed retrieval vs raw transcript)");
|
|
31979
|
+
lines.push("");
|
|
31980
|
+
const c1 = aggregates["raw-transcript"];
|
|
31981
|
+
const c2 = aggregates["typed-contract"];
|
|
31982
|
+
if (c1 && c2) {
|
|
31983
|
+
lines.push(`- **Stale-memory harm**: C1 ${pct(c1.staleMemoryHarmRate)} \u2192 C2 ${pct(c2.staleMemoryHarmRate)}. The typed contract excludes superseded facts; raw transcript cannot.`);
|
|
31984
|
+
lines.push(`- **Wrong-scope leakage**: C1 ${pct(c1.wrongScopeRetrievalRate)} \u2192 C2 ${pct(c2.wrongScopeRetrievalRate)}. Scope filtering removes cross-project decoys that raw transcript surfaces.`);
|
|
31985
|
+
lines.push(`- **Boundary violations**: C1 ${pct(c1.actionBoundaryViolationRate)} \u2192 C2 ${pct(c2.actionBoundaryViolationRate)}. Structured boundary notes are reliably surfaced; buried transcript prose is not.`);
|
|
31986
|
+
lines.push(`- **Citation coverage**: C1 ${pct(c1.citationCoverage)} \u2192 C2 ${pct(c2.citationCoverage)}. Typed slots carry citations; raw text does not.`);
|
|
31987
|
+
lines.push(`- **Compression**: C1 ${num(c1.meanCompressionRatio)}\xD7 \u2192 C2 ${num(c2.meanCompressionRatio)}\xD7 of the raw transcript. Typed packs are smaller for the same budget.`);
|
|
31988
|
+
}
|
|
31989
|
+
lines.push("");
|
|
31990
|
+
lines.push("## Biggest failures / honest weaknesses");
|
|
31991
|
+
lines.push("");
|
|
31992
|
+
lines.push("- **Recall parity on pure-recall tasks**: raw transcript (C1) is a *fair* baseline \u2014 when the needed fact is present and unambiguous, C1 recalls it as well as C2. Typed retrieval's advantage is governance, not brute recall.");
|
|
31993
|
+
lines.push("- **C0 no-memory**: fails every recall-dependent task by construction; it is the floor, not a competitor.");
|
|
31994
|
+
lines.push("- **Skill overfitting risk**: the smoke fixture is small and hand-authored around two skills; positive/negative trigger balance is preliminary.");
|
|
31995
|
+
lines.push("");
|
|
31996
|
+
lines.push("## Worked examples");
|
|
31997
|
+
lines.push("");
|
|
31998
|
+
const staleExample = tasks.find((t) => t.family === "stale-memory-trap");
|
|
31999
|
+
if (staleExample) {
|
|
32000
|
+
lines.push(`- **Stale-memory trap** (\`${staleExample.id}\`): a superseded fact (more repeated in the transcript) crowds out the one-line correction under C1, while C2 excludes it via \`excludeIfSuperseded\`.`);
|
|
32001
|
+
}
|
|
32002
|
+
const scopeExample = tasks.find((t) => t.family === "wrong-scope-trap");
|
|
32003
|
+
if (scopeExample) {
|
|
32004
|
+
lines.push(`- **Wrong-scope trap** (\`${scopeExample.id}\`): a same-subject fact from a different project leaks under C1 (no scope metadata) but is filtered out under C2/C3.`);
|
|
32005
|
+
}
|
|
32006
|
+
lines.push("");
|
|
32007
|
+
lines.push("## Token / cost / latency");
|
|
32008
|
+
lines.push("");
|
|
32009
|
+
lines.push("- **Estimated cost**: $0 \u2014 no model was called. Token counts reflect injected memory only.");
|
|
32010
|
+
lines.push("| Condition | Mean injected tokens | Mean retrieved items |");
|
|
32011
|
+
lines.push("|---|---|---|");
|
|
32012
|
+
for (const cond of CONDITION_ORDER) {
|
|
32013
|
+
const a = aggregates[cond];
|
|
32014
|
+
if (!a) continue;
|
|
32015
|
+
lines.push(`| ${BOUNDED_MEMORY_CONDITION_LABELS[cond]} | ${num(a.meanMemoryTokensInjected)} | ${num(a.meanRetrievedItemCount)} |`);
|
|
32016
|
+
}
|
|
32017
|
+
lines.push("");
|
|
32018
|
+
lines.push("## Safe vs unsupported claims");
|
|
32019
|
+
lines.push("");
|
|
32020
|
+
lines.push("**Safe to claim from this run:**");
|
|
32021
|
+
lines.push("");
|
|
32022
|
+
lines.push("- Typed retrieval contracts exclude superseded and wrong-scope memories that raw transcript stuffing surfaces, *under this deterministic simulation*.");
|
|
32023
|
+
lines.push("- Structured boundary notes are reliably surfaced by the typed contract and not by raw transcript, *under this simulation*.");
|
|
32024
|
+
lines.push("- Typed packs are smaller than the raw transcript for the same task, under the shared budget.");
|
|
32025
|
+
lines.push("");
|
|
32026
|
+
lines.push("**NOT supported by this run (do not claim):**");
|
|
32027
|
+
lines.push("");
|
|
32028
|
+
lines.push("- That Remnic beats Mem0 / Zep / Letta or any other system. No cross-system comparison was performed.");
|
|
32029
|
+
lines.push("- That these numbers generalize to real users. The fixture is small, synthetic, and hand-authored.");
|
|
32030
|
+
lines.push("- Any frontier-model quality figures. No LLM was called; the agent is a deterministic decision procedure.");
|
|
32031
|
+
lines.push("- That skill-triggered memory helps on real procedural workloads. Two hand-authored skills are not evidence.");
|
|
32032
|
+
lines.push("");
|
|
32033
|
+
lines.push("## Recommended next experiment");
|
|
32034
|
+
lines.push("");
|
|
32035
|
+
lines.push("1. Scale to the full fixture (100+ tasks) with de-identified or synthetic traces.");
|
|
32036
|
+
lines.push("2. Swap the deterministic agent for a real responder under a blinded judge (condition names hidden).");
|
|
32037
|
+
lines.push("3. Add a `model-context-max` raw-transcript variant as a *separate, non-primary* condition.");
|
|
32038
|
+
lines.push("4. Expand the skill library and report false-positive/negative rates over a larger trigger set.");
|
|
32039
|
+
lines.push("");
|
|
32040
|
+
return lines.join("\n");
|
|
32041
|
+
}
|
|
32042
|
+
|
|
32043
|
+
// src/benchmarks/remnic/bounded-memory-contracts/runner.ts
|
|
32044
|
+
var boundedMemoryContractsDefinition = {
|
|
32045
|
+
id: "bounded-memory-contracts",
|
|
32046
|
+
title: "Bounded Memory Contracts",
|
|
32047
|
+
tier: "remnic",
|
|
32048
|
+
status: "ready",
|
|
32049
|
+
runnerAvailable: true,
|
|
32050
|
+
meta: {
|
|
32051
|
+
name: "bounded-memory-contracts",
|
|
32052
|
+
version: "1.0.0",
|
|
32053
|
+
description: "Ablates raw transcript stuffing vs typed retrieval contracts vs skill-triggered memory under a shared token budget (issue #1708).",
|
|
32054
|
+
category: "agentic",
|
|
32055
|
+
citation: "Remnic internal synthetic benchmark for issue #1708"
|
|
32056
|
+
}
|
|
32057
|
+
};
|
|
32058
|
+
function resolveInjectedSkills(task) {
|
|
32059
|
+
const injected = [];
|
|
32060
|
+
for (const skill of task.skills) {
|
|
32061
|
+
const verdict = classifySkillTrigger(skill, task);
|
|
32062
|
+
if (verdict.injected) {
|
|
32063
|
+
injected.push(skill);
|
|
32064
|
+
}
|
|
32065
|
+
}
|
|
32066
|
+
return injected;
|
|
32067
|
+
}
|
|
32068
|
+
function skillOutcome(task, skill, injected) {
|
|
32069
|
+
if (task.family === "skill-positive") {
|
|
32070
|
+
const expected = task.shouldUseSkillId === skill.id;
|
|
32071
|
+
if (injected && expected) return "helped";
|
|
32072
|
+
if (injected !== expected) return "harmed";
|
|
32073
|
+
return "irrelevant";
|
|
32074
|
+
}
|
|
32075
|
+
if (task.family === "skill-negative") {
|
|
32076
|
+
return injected ? "harmed" : "irrelevant";
|
|
32077
|
+
}
|
|
32078
|
+
return "irrelevant";
|
|
32079
|
+
}
|
|
32080
|
+
async function runBoundedMemoryContractsBenchmark(options) {
|
|
32081
|
+
const fixtureSource = options.mode === "quick" ? BOUNDED_MEMORY_SMOKE_FIXTURE : BOUNDED_MEMORY_FIXTURE;
|
|
32082
|
+
const tasks = options.mode === "full" && typeof options.limit === "number" && options.limit > 0 && Number.isFinite(options.limit) ? fixtureSource.slice(0, Math.floor(options.limit)) : fixtureSource;
|
|
32083
|
+
const seed = typeof options.seed === "number" ? options.seed : 0;
|
|
32084
|
+
const totalPairs = tasks.length * BOUNDED_MEMORY_CONDITIONS.length;
|
|
32085
|
+
const byCondition = /* @__PURE__ */ new Map();
|
|
32086
|
+
for (const condition of BOUNDED_MEMORY_CONDITIONS) {
|
|
32087
|
+
const results = [];
|
|
32088
|
+
for (const task of tasks) {
|
|
32089
|
+
const resolvedSkills = condition === "typed-plus-skills" ? resolveInjectedSkills(task) : [];
|
|
32090
|
+
const started = performance.now();
|
|
32091
|
+
const pack = assemblePack(task, condition, BOUNDED_MEMORY_CONTRACT, resolvedSkills);
|
|
32092
|
+
const packedSkillIds = new Set(
|
|
32093
|
+
pack.slots.find((slot) => slot.id === "triggered_skills")?.items.map((it) => it.itemId) ?? []
|
|
32094
|
+
);
|
|
32095
|
+
const effectiveSkills = resolvedSkills.filter((sk) => packedSkillIds.has(sk.id));
|
|
32096
|
+
const decision = simulateAgent(task, pack, effectiveSkills);
|
|
32097
|
+
const latencyMs = Math.round(performance.now() - started);
|
|
32098
|
+
results.push({ task, pack, decision, latencyMs });
|
|
32099
|
+
}
|
|
32100
|
+
byCondition.set(condition, results);
|
|
32101
|
+
}
|
|
32102
|
+
const taskResults = [];
|
|
32103
|
+
const conditionAggregates = {};
|
|
32104
|
+
const scoredByCondition = {};
|
|
32105
|
+
let c3SkillLog = [];
|
|
32106
|
+
for (const condition of BOUNDED_MEMORY_CONDITIONS) {
|
|
32107
|
+
const results = byCondition.get(condition);
|
|
32108
|
+
const scoredPairs = results.map(({ task, pack, decision }) => {
|
|
32109
|
+
const scores = scoreTaskPair(task, pack, decision);
|
|
32110
|
+
return { task, scores, pack, decision };
|
|
32111
|
+
});
|
|
32112
|
+
scoredByCondition[condition] = scoredPairs.map(({ task, scores }) => ({ task, scores }));
|
|
32113
|
+
let skillLog = [];
|
|
32114
|
+
if (condition === "typed-plus-skills") {
|
|
32115
|
+
skillLog = scoredPairs.flatMap(({ task, pack }) => {
|
|
32116
|
+
const packedSkillIds = new Set(
|
|
32117
|
+
pack.slots.find((slot) => slot.id === "triggered_skills")?.items.map((it) => it.itemId) ?? []
|
|
32118
|
+
);
|
|
32119
|
+
return task.skills.map((skill) => {
|
|
32120
|
+
const verdict = classifySkillTrigger(skill, task);
|
|
32121
|
+
const injected = packedSkillIds.has(skill.id);
|
|
32122
|
+
return {
|
|
32123
|
+
taskId: task.id,
|
|
32124
|
+
skillId: skill.id,
|
|
32125
|
+
considered: verdict.considered,
|
|
32126
|
+
injected,
|
|
32127
|
+
triggerReason: verdict.reason,
|
|
32128
|
+
confidence: skill.confidence,
|
|
32129
|
+
outcome: skillOutcome(task, skill, injected)
|
|
32130
|
+
};
|
|
32131
|
+
});
|
|
32132
|
+
});
|
|
32133
|
+
c3SkillLog = skillLog;
|
|
32134
|
+
}
|
|
32135
|
+
conditionAggregates[condition] = aggregateCondition(condition, scoredByCondition[condition], skillLog);
|
|
32136
|
+
for (const { task, pack, decision, scores } of scoredPairs) {
|
|
32137
|
+
const tr = {
|
|
32138
|
+
taskId: `${condition}:${task.id}`,
|
|
32139
|
+
question: task.prompt,
|
|
32140
|
+
expected: task.expectedAnswer,
|
|
32141
|
+
actual: decision.answer,
|
|
32142
|
+
scores: { ...scores },
|
|
32143
|
+
latencyMs: results.find((r) => r.task.id === task.id).latencyMs,
|
|
32144
|
+
tokens: { input: pack.totalTokens, output: 0 },
|
|
32145
|
+
details: {
|
|
32146
|
+
condition,
|
|
32147
|
+
family: task.family,
|
|
32148
|
+
scope: task.scope,
|
|
32149
|
+
packTokens: pack.totalTokens,
|
|
32150
|
+
compressionRatio: scores.compression_ratio_vs_raw_transcript,
|
|
32151
|
+
recalledItemIds: decision.recalledItemIds,
|
|
32152
|
+
askedClarification: decision.askedClarification,
|
|
32153
|
+
acted: decision.acted
|
|
32154
|
+
}
|
|
32155
|
+
};
|
|
32156
|
+
taskResults.push(tr);
|
|
32157
|
+
options.onTaskComplete?.(tr, taskResults.length, totalPairs);
|
|
32158
|
+
}
|
|
32159
|
+
}
|
|
32160
|
+
const aggregates = aggregateTaskScores(taskResults.map((t) => t.scores));
|
|
32161
|
+
const remnicVersion = await getRemnicVersion();
|
|
32162
|
+
const totalLatencyMs = taskResults.reduce((sum, t) => sum + t.latencyMs, 0);
|
|
32163
|
+
const totalMemoryTokens = taskResults.reduce((sum, t) => sum + t.tokens.input, 0);
|
|
32164
|
+
let artifactReport = null;
|
|
32165
|
+
if (options.outputDir) {
|
|
32166
|
+
artifactReport = await writeArtifacts(options.outputDir, byCondition, conditionAggregates, tasks);
|
|
32167
|
+
}
|
|
32168
|
+
const skillTriggerLog = c3SkillLog;
|
|
32169
|
+
return {
|
|
32170
|
+
meta: {
|
|
32171
|
+
id: randomUUID32(),
|
|
32172
|
+
benchmark: options.benchmark.id,
|
|
32173
|
+
benchmarkTier: options.benchmark.tier,
|
|
32174
|
+
version: options.benchmark.meta.version,
|
|
32175
|
+
remnicVersion,
|
|
32176
|
+
gitSha: getGitSha(),
|
|
32177
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
32178
|
+
mode: options.mode,
|
|
32179
|
+
runCount: 1,
|
|
32180
|
+
seeds: [seed],
|
|
32181
|
+
datasetHash: fixtureHash(tasks)
|
|
32182
|
+
},
|
|
32183
|
+
config: {
|
|
32184
|
+
runtimeProfile: options.runtimeProfile ?? null,
|
|
32185
|
+
systemProvider: options.systemProvider ?? null,
|
|
32186
|
+
judgeProvider: options.judgeProvider ?? null,
|
|
32187
|
+
adapterMode: options.adapterMode ?? "deterministic-offline",
|
|
32188
|
+
remnicConfig: options.remnicConfig ?? {},
|
|
32189
|
+
benchmarkOptions: {
|
|
32190
|
+
conditions: conditionAggregates,
|
|
32191
|
+
skillTriggerLog,
|
|
32192
|
+
contract: BOUNDED_MEMORY_CONTRACT,
|
|
32193
|
+
fixtureTaskCount: tasks.length,
|
|
32194
|
+
conditionCount: BOUNDED_MEMORY_CONDITIONS.length,
|
|
32195
|
+
artifactReport
|
|
32196
|
+
}
|
|
32197
|
+
},
|
|
32198
|
+
cost: {
|
|
32199
|
+
totalTokens: totalMemoryTokens,
|
|
32200
|
+
inputTokens: totalMemoryTokens,
|
|
32201
|
+
outputTokens: 0,
|
|
32202
|
+
estimatedCostUsd: 0,
|
|
32203
|
+
totalLatencyMs,
|
|
32204
|
+
meanQueryLatencyMs: taskResults.length > 0 ? totalLatencyMs / taskResults.length : 0,
|
|
32205
|
+
judgeModelCalls: 0
|
|
32206
|
+
},
|
|
32207
|
+
results: {
|
|
32208
|
+
tasks: taskResults,
|
|
32209
|
+
aggregates,
|
|
32210
|
+
statistics: {
|
|
32211
|
+
confidenceIntervals: {},
|
|
32212
|
+
bootstrapSamples: 0
|
|
32213
|
+
}
|
|
32214
|
+
},
|
|
32215
|
+
environment: {
|
|
32216
|
+
os: process.platform,
|
|
32217
|
+
nodeVersion: process.version,
|
|
32218
|
+
hardware: process.arch
|
|
32219
|
+
}
|
|
32220
|
+
};
|
|
32221
|
+
}
|
|
32222
|
+
async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks) {
|
|
32223
|
+
const root = path32.resolve(outputDir);
|
|
32224
|
+
await mkdir16(path32.join(root, "conditions"), { recursive: true });
|
|
32225
|
+
await mkdir16(path32.join(root, "prompts"), { recursive: true });
|
|
32226
|
+
await mkdir16(path32.join(root, "retrieval"), { recursive: true });
|
|
32227
|
+
await mkdir16(path32.join(root, "scores"), { recursive: true });
|
|
32228
|
+
const csvRows = [
|
|
32229
|
+
"task_id,condition,family,scope,task_success,should_ask_accuracy,relevant_memory_recall,stale_memory_harm_rate,wrong_scope_retrieval_rate,supersession_respected_rate,citation_coverage,memory_tokens_injected,retrieved_item_count,compression_ratio_vs_raw_transcript"
|
|
32230
|
+
];
|
|
32231
|
+
for (const condition of BOUNDED_MEMORY_CONDITIONS) {
|
|
32232
|
+
const results = byCondition.get(condition);
|
|
32233
|
+
const condDir = path32.join(root, "conditions", condition);
|
|
32234
|
+
await mkdir16(condDir, { recursive: true });
|
|
32235
|
+
for (const { task, pack, decision } of results) {
|
|
32236
|
+
const scores = scoreTaskPair(task, pack, decision);
|
|
32237
|
+
const promptMd = renderPromptPack(task, condition, pack);
|
|
32238
|
+
const promptPath = path32.join(root, "prompts", `${task.id}.${condition}.md`);
|
|
32239
|
+
await mkdir16(path32.dirname(promptPath), { recursive: true });
|
|
32240
|
+
await writeFile15(promptPath, promptMd, "utf8");
|
|
32241
|
+
const retrievalJson = `${JSON.stringify(
|
|
32242
|
+
{
|
|
32243
|
+
taskId: task.id,
|
|
32244
|
+
condition,
|
|
32245
|
+
contract: BOUNDED_MEMORY_CONTRACT.id,
|
|
32246
|
+
pack: {
|
|
32247
|
+
slots: pack.slots.map((slot) => ({
|
|
32248
|
+
id: slot.id,
|
|
32249
|
+
items: slot.items
|
|
32250
|
+
})),
|
|
32251
|
+
transcriptBlock: pack.transcriptBlock,
|
|
32252
|
+
boundaryItem: pack.boundaryItem,
|
|
32253
|
+
totalTokens: pack.totalTokens,
|
|
32254
|
+
fullTranscriptTokens: pack.fullTranscriptTokens
|
|
32255
|
+
},
|
|
32256
|
+
decision,
|
|
32257
|
+
scores
|
|
32258
|
+
},
|
|
32259
|
+
null,
|
|
32260
|
+
2
|
|
32261
|
+
)}
|
|
32262
|
+
`;
|
|
32263
|
+
const retrievalPath = path32.join(root, "retrieval", `${task.id}.${condition}.json`);
|
|
32264
|
+
await writeFile15(retrievalPath, retrievalJson, "utf8");
|
|
32265
|
+
csvRows.push(
|
|
32266
|
+
[
|
|
32267
|
+
task.id,
|
|
32268
|
+
condition,
|
|
32269
|
+
task.family,
|
|
32270
|
+
task.scope,
|
|
32271
|
+
scores.task_success,
|
|
32272
|
+
scores.should_ask_accuracy,
|
|
32273
|
+
scores.relevant_memory_recall,
|
|
32274
|
+
scores.stale_memory_harm_rate,
|
|
32275
|
+
scores.wrong_scope_retrieval_rate,
|
|
32276
|
+
scores.supersession_respected_rate,
|
|
32277
|
+
scores.citation_coverage.toFixed(4),
|
|
32278
|
+
scores.memory_tokens_injected,
|
|
32279
|
+
scores.retrieved_item_count,
|
|
32280
|
+
scores.compression_ratio_vs_raw_transcript.toFixed(4)
|
|
32281
|
+
].join(",")
|
|
32282
|
+
);
|
|
32283
|
+
}
|
|
32284
|
+
await writeFile15(
|
|
32285
|
+
path32.join(condDir, "summary.json"),
|
|
32286
|
+
`${JSON.stringify(conditionAggregates[condition], null, 2)}
|
|
32287
|
+
`,
|
|
32288
|
+
"utf8"
|
|
32289
|
+
);
|
|
32290
|
+
}
|
|
32291
|
+
await writeFile15(path32.join(root, "scores", "per-task.csv"), `${csvRows.join("\n")}
|
|
32292
|
+
`, "utf8");
|
|
32293
|
+
await writeFile15(
|
|
32294
|
+
path32.join(root, "scores", "aggregate.json"),
|
|
32295
|
+
`${JSON.stringify(conditionAggregates, null, 2)}
|
|
32296
|
+
`,
|
|
32297
|
+
"utf8"
|
|
32298
|
+
);
|
|
32299
|
+
const report = renderReportMarkdown(tasks, conditionAggregates);
|
|
32300
|
+
await writeFile15(path32.join(root, "report.md"), report, "utf8");
|
|
32301
|
+
return path32.join(root, "report.md");
|
|
32302
|
+
}
|
|
32303
|
+
function renderPromptPack(task, condition, pack) {
|
|
32304
|
+
const lines = [];
|
|
32305
|
+
lines.push(`# Prompt pack \u2014 ${task.id} [${BOUNDED_MEMORY_CONDITION_LABELS[condition]}]`);
|
|
32306
|
+
lines.push("");
|
|
32307
|
+
lines.push("## System");
|
|
32308
|
+
lines.push(
|
|
32309
|
+
condition === "no-memory" ? "You have no prior memory. Answer from the current task only." : condition === "raw-transcript" ? "Prior session transcript follows. Use it as context." : "A bounded memory contract follows. Each item is typed, scoped, and cited."
|
|
32310
|
+
);
|
|
32311
|
+
lines.push("");
|
|
32312
|
+
lines.push("## Current task");
|
|
32313
|
+
lines.push(task.prompt);
|
|
32314
|
+
lines.push("");
|
|
32315
|
+
lines.push(`## Active scope`);
|
|
32316
|
+
lines.push(task.scope);
|
|
32317
|
+
lines.push("");
|
|
32318
|
+
if (condition === "no-memory") {
|
|
32319
|
+
lines.push("_(no memory injected)_");
|
|
32320
|
+
lines.push("");
|
|
32321
|
+
} else if (condition === "raw-transcript") {
|
|
32322
|
+
lines.push("## Raw transcript (budget-normalized)");
|
|
32323
|
+
lines.push("```");
|
|
32324
|
+
lines.push(pack.transcriptBlock ?? "_(empty)_");
|
|
32325
|
+
lines.push("```");
|
|
32326
|
+
lines.push("");
|
|
32327
|
+
} else {
|
|
32328
|
+
if (pack.boundaryItem) {
|
|
32329
|
+
lines.push("## Boundaries");
|
|
32330
|
+
lines.push(`- [${pack.boundaryItem.citation}] (${pack.boundaryItem.scope}, ${pack.boundaryItem.status}) ${pack.boundaryItem.content}`);
|
|
32331
|
+
lines.push("");
|
|
32332
|
+
}
|
|
32333
|
+
for (const slot of pack.slots) {
|
|
32334
|
+
if (slot.items.length === 0) continue;
|
|
32335
|
+
lines.push(`## ${slot.id}`);
|
|
32336
|
+
for (const it of slot.items) {
|
|
32337
|
+
lines.push(`- [${it.citation}] (${it.scope}, ${it.status}) ${it.content}`);
|
|
32338
|
+
}
|
|
32339
|
+
lines.push("");
|
|
32340
|
+
}
|
|
32341
|
+
}
|
|
32342
|
+
return `${lines.join("\n")}
|
|
32343
|
+
`;
|
|
32344
|
+
}
|
|
32345
|
+
|
|
30978
32346
|
// src/registry.ts
|
|
30979
32347
|
var REGISTERED_BENCHMARKS = [
|
|
30980
32348
|
{
|
|
@@ -31112,6 +32480,10 @@ var REGISTERED_BENCHMARKS = [
|
|
|
31112
32480
|
{
|
|
31113
32481
|
...memcorrectDefinition,
|
|
31114
32482
|
run: runMemCorrectBenchmark
|
|
32483
|
+
},
|
|
32484
|
+
{
|
|
32485
|
+
...boundedMemoryContractsDefinition,
|
|
32486
|
+
run: runBoundedMemoryContractsBenchmark
|
|
31115
32487
|
}
|
|
31116
32488
|
];
|
|
31117
32489
|
function listBenchmarks() {
|
|
@@ -31151,8 +32523,8 @@ function finalizeBenchmarkResultConfig(result, options) {
|
|
|
31151
32523
|
}
|
|
31152
32524
|
|
|
31153
32525
|
// src/benchmark.ts
|
|
31154
|
-
var DEFAULT_BASELINE_PATH =
|
|
31155
|
-
var DEFAULT_REPORT_PATH =
|
|
32526
|
+
var DEFAULT_BASELINE_PATH = path33.join(process.cwd(), "benchmarks", "baseline.json");
|
|
32527
|
+
var DEFAULT_REPORT_PATH = path33.join(process.cwd(), "benchmarks", "report.json");
|
|
31156
32528
|
var BASELINE_VERSION = 1;
|
|
31157
32529
|
var DEFAULT_TOLERANCE = 10;
|
|
31158
32530
|
var DEFAULT_FULL_RUN_COUNT = 5;
|
|
@@ -31230,7 +32602,7 @@ async function runBenchmark(benchmarkId, options) {
|
|
|
31230
32602
|
if (!willWrapPrimary && !willWrapCross) {
|
|
31231
32603
|
return void 0;
|
|
31232
32604
|
}
|
|
31233
|
-
const cacheDir = options.judgeCacheDir ?
|
|
32605
|
+
const cacheDir = options.judgeCacheDir ? path33.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path33.join(path33.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
|
|
31234
32606
|
if (cacheDir === void 0) {
|
|
31235
32607
|
return void 0;
|
|
31236
32608
|
}
|
|
@@ -31352,13 +32724,13 @@ function wrapJudgeWithCache(args) {
|
|
|
31352
32724
|
// differentiator is part of the prompt hash. Bumping
|
|
31353
32725
|
// JUDGE_CACHE_PROTOCOL_VERSION invalidates verdicts when judge
|
|
31354
32726
|
// prompt/parse semantics change (PR #1591, High).
|
|
31355
|
-
judgePromptHash:
|
|
32727
|
+
judgePromptHash: createHash11("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("").update(args.amaBenchJudgeProtocol).update("").update(args.role).digest("hex"),
|
|
31356
32728
|
judgeModelId: args.provider?.model !== void 0 && args.provider.model.length > 0 ? `${args.provider.model}${crossJudgeIdSuffix}` : `unknown-${args.role}-judge`,
|
|
31357
32729
|
// Full judge configuration, deterministically serialized (sorted
|
|
31358
32730
|
// keys) so provider/base-url/retry changes produce fresh cache
|
|
31359
32731
|
// keys. `role` is included so primary and cross judges never
|
|
31360
32732
|
// share a paramsHash.
|
|
31361
|
-
judgeParamsHash:
|
|
32733
|
+
judgeParamsHash: createHash11("sha256").update(
|
|
31362
32734
|
stableStringify2({
|
|
31363
32735
|
role: args.role,
|
|
31364
32736
|
provider: args.provider
|
|
@@ -31419,7 +32791,7 @@ function loadBaseline(baselinePath) {
|
|
|
31419
32791
|
return raw;
|
|
31420
32792
|
}
|
|
31421
32793
|
function saveBaseline(baselinePath, baseline) {
|
|
31422
|
-
fs2.mkdirSync(
|
|
32794
|
+
fs2.mkdirSync(path33.dirname(baselinePath), { recursive: true });
|
|
31423
32795
|
fs2.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}
|
|
31424
32796
|
`);
|
|
31425
32797
|
}
|
|
@@ -31649,7 +33021,7 @@ function generateReport(results, reportPath) {
|
|
|
31649
33021
|
totalDurationMs: results.reduce((sum, result) => sum + result.totalDurationMs, 0)
|
|
31650
33022
|
};
|
|
31651
33023
|
if (reportPath) {
|
|
31652
|
-
fs2.mkdirSync(
|
|
33024
|
+
fs2.mkdirSync(path33.dirname(reportPath), { recursive: true });
|
|
31653
33025
|
fs2.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}
|
|
31654
33026
|
`);
|
|
31655
33027
|
}
|
|
@@ -31657,7 +33029,7 @@ function generateReport(results, reportPath) {
|
|
|
31657
33029
|
}
|
|
31658
33030
|
|
|
31659
33031
|
// src/stats/effect-size.ts
|
|
31660
|
-
function
|
|
33032
|
+
function mean3(values) {
|
|
31661
33033
|
if (values.length === 0) {
|
|
31662
33034
|
throw new Error("effect size requires at least one value");
|
|
31663
33035
|
}
|
|
@@ -31673,8 +33045,8 @@ function cohensD(candidateValues, baselineValues) {
|
|
|
31673
33045
|
if (candidateValues.length === 0 || baselineValues.length === 0) {
|
|
31674
33046
|
throw new Error("effect size requires non-empty candidate and baseline arrays");
|
|
31675
33047
|
}
|
|
31676
|
-
const candidateMean =
|
|
31677
|
-
const baselineMean =
|
|
33048
|
+
const candidateMean = mean3(candidateValues);
|
|
33049
|
+
const baselineMean = mean3(baselineValues);
|
|
31678
33050
|
const candidateVariance = sampleVariance(candidateValues, candidateMean);
|
|
31679
33051
|
const baselineVariance = sampleVariance(baselineValues, baselineMean);
|
|
31680
33052
|
const pooledDegreesOfFreedom = candidateValues.length + baselineValues.length - 2;
|
|
@@ -32114,8 +33486,8 @@ function formatError(error) {
|
|
|
32114
33486
|
}
|
|
32115
33487
|
|
|
32116
33488
|
// src/benchmarks/custom/runner.ts
|
|
32117
|
-
import { randomUUID as
|
|
32118
|
-
import
|
|
33489
|
+
import { randomUUID as randomUUID33 } from "crypto";
|
|
33490
|
+
import path34 from "path";
|
|
32119
33491
|
import { expandTildePath as expandTildePath4 } from "@remnic/core";
|
|
32120
33492
|
async function runCustomBenchmarkFile(filePath, options) {
|
|
32121
33493
|
const spec = await loadCustomBenchmarkFile(filePath);
|
|
@@ -32128,7 +33500,7 @@ async function runCustomBenchmarkFile(filePath, options) {
|
|
|
32128
33500
|
let cacheRestore;
|
|
32129
33501
|
let cacheCounters;
|
|
32130
33502
|
if (spec.scoring === "llm_judge" && runOptions.system.judge !== void 0 && !runOptions.noJudgeCache && (runOptions.judgeProvider ?? null) !== null) {
|
|
32131
|
-
const cacheDir = runOptions.judgeCacheDir ?
|
|
33503
|
+
const cacheDir = runOptions.judgeCacheDir ? path34.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path34.join(path34.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
|
|
32132
33504
|
if (cacheDir !== void 0) {
|
|
32133
33505
|
const originalJudge = runOptions.system.judge;
|
|
32134
33506
|
const wrapped = wrapJudgeWithCache({
|
|
@@ -32202,7 +33574,7 @@ async function runCustomBenchmark(spec, options) {
|
|
|
32202
33574
|
const totalOutputTokens = tasks.reduce((sum, task) => sum + task.tokens.output, 0);
|
|
32203
33575
|
return finalizeBenchmarkResultConfig({
|
|
32204
33576
|
meta: {
|
|
32205
|
-
id:
|
|
33577
|
+
id: randomUUID33(),
|
|
32206
33578
|
benchmark: options.benchmark.id,
|
|
32207
33579
|
benchmarkTier: options.benchmark.tier,
|
|
32208
33580
|
version: options.benchmark.meta.version,
|
|
@@ -32329,7 +33701,7 @@ async function scoreTask(scoring, options, question, actual, expected) {
|
|
|
32329
33701
|
}
|
|
32330
33702
|
}
|
|
32331
33703
|
function createCustomBenchmarkDefinition(benchmark, filePath) {
|
|
32332
|
-
const id = `custom:${slugify(
|
|
33704
|
+
const id = `custom:${slugify(path34.basename(filePath, path34.extname(filePath)) || benchmark.name)}`;
|
|
32333
33705
|
return {
|
|
32334
33706
|
id,
|
|
32335
33707
|
title: benchmark.name,
|
|
@@ -33199,9 +34571,9 @@ var chatFixture = {
|
|
|
33199
34571
|
};
|
|
33200
34572
|
|
|
33201
34573
|
// src/judges/calibration-slice.ts
|
|
33202
|
-
import { createHash as
|
|
33203
|
-
import { mkdir as
|
|
33204
|
-
import
|
|
34574
|
+
import { createHash as createHash12, randomBytes as randomBytes3 } from "crypto";
|
|
34575
|
+
import { mkdir as mkdir17, readFile as readFile21, rename as rename3, unlink as unlink3, writeFile as writeFile16 } from "fs/promises";
|
|
34576
|
+
import path35 from "path";
|
|
33205
34577
|
|
|
33206
34578
|
// src/judges/cohen-kappa.ts
|
|
33207
34579
|
function computeCohensKappa(raterA, raterB) {
|
|
@@ -33276,7 +34648,7 @@ function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
|
|
|
33276
34648
|
unique.push(id);
|
|
33277
34649
|
}
|
|
33278
34650
|
}
|
|
33279
|
-
return unique.map((id) => ({ id, digest:
|
|
34651
|
+
return unique.map((id) => ({ id, digest: createHash12("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
|
|
33280
34652
|
}
|
|
33281
34653
|
async function runJudgeCalibration(options) {
|
|
33282
34654
|
const binScore = options.binScore ?? ((score) => binarizeJudgeScore(score));
|
|
@@ -33330,7 +34702,7 @@ async function runJudgeCalibration(options) {
|
|
|
33330
34702
|
};
|
|
33331
34703
|
}
|
|
33332
34704
|
async function writeJudgeCalibrationState(result, calibrationDir, identities) {
|
|
33333
|
-
await
|
|
34705
|
+
await mkdir17(calibrationDir, { recursive: true });
|
|
33334
34706
|
const state = {
|
|
33335
34707
|
kappa: result.kappa,
|
|
33336
34708
|
sampleSize: result.sampleSize,
|
|
@@ -33338,9 +34710,9 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
|
|
|
33338
34710
|
warning: result.warning,
|
|
33339
34711
|
...identities ? identities : {}
|
|
33340
34712
|
};
|
|
33341
|
-
const filePath =
|
|
34713
|
+
const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
|
|
33342
34714
|
const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
|
|
33343
|
-
await
|
|
34715
|
+
await writeFile16(tempPath, `${JSON.stringify(state, null, 2)}
|
|
33344
34716
|
`, "utf8");
|
|
33345
34717
|
try {
|
|
33346
34718
|
await rename3(tempPath, filePath);
|
|
@@ -33351,7 +34723,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
|
|
|
33351
34723
|
return filePath;
|
|
33352
34724
|
}
|
|
33353
34725
|
async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
|
|
33354
|
-
const filePath =
|
|
34726
|
+
const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
|
|
33355
34727
|
let raw;
|
|
33356
34728
|
try {
|
|
33357
34729
|
raw = await readFile21(filePath, "utf8");
|
|
@@ -33425,9 +34797,9 @@ function sanitizeCalibrationSegment(value) {
|
|
|
33425
34797
|
}
|
|
33426
34798
|
|
|
33427
34799
|
// src/benchmarks/remnic/procedural-recall/ablation.ts
|
|
33428
|
-
import { mkdir as
|
|
34800
|
+
import { mkdir as mkdir18, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile17, readFile as readFile22 } from "fs/promises";
|
|
33429
34801
|
import os9 from "os";
|
|
33430
|
-
import
|
|
34802
|
+
import path36 from "path";
|
|
33431
34803
|
import {
|
|
33432
34804
|
StorageManager as StorageManager3,
|
|
33433
34805
|
parseConfig as parseConfig5,
|
|
@@ -33458,7 +34830,7 @@ async function runSide(scenarios, proceduralEnabled) {
|
|
|
33458
34830
|
const observed = [];
|
|
33459
34831
|
for (const scenario of scenarios) {
|
|
33460
34832
|
const dir = await mkdtemp11(
|
|
33461
|
-
|
|
34833
|
+
path36.join(os9.tmpdir(), "remnic-bench-proc-ablation-")
|
|
33462
34834
|
);
|
|
33463
34835
|
try {
|
|
33464
34836
|
const storage = new StorageManager3(dir);
|
|
@@ -33473,7 +34845,7 @@ ${body}`,
|
|
|
33473
34845
|
);
|
|
33474
34846
|
const config = parseConfig5({
|
|
33475
34847
|
memoryDir: dir,
|
|
33476
|
-
workspaceDir:
|
|
34848
|
+
workspaceDir: path36.join(dir, "ws"),
|
|
33477
34849
|
openaiApiKey: "bench-key",
|
|
33478
34850
|
procedural: {
|
|
33479
34851
|
enabled: proceduralEnabled,
|
|
@@ -33644,9 +35016,9 @@ async function runProceduralAblationCli(args) {
|
|
|
33644
35016
|
random: args.random,
|
|
33645
35017
|
seed: args.seed
|
|
33646
35018
|
});
|
|
33647
|
-
const outDir =
|
|
33648
|
-
await
|
|
33649
|
-
await
|
|
35019
|
+
const outDir = path36.dirname(path36.resolve(args.outPath));
|
|
35020
|
+
await mkdir18(outDir, { recursive: true });
|
|
35021
|
+
await writeFile17(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
|
|
33650
35022
|
return artifact;
|
|
33651
35023
|
}
|
|
33652
35024
|
|
|
@@ -33942,7 +35314,7 @@ var PROCEDURAL_REAL_SCENARIOS_SMOKE = [
|
|
|
33942
35314
|
];
|
|
33943
35315
|
|
|
33944
35316
|
// src/security/extraction-attack/tokenize.ts
|
|
33945
|
-
function
|
|
35317
|
+
function tokenize5(text) {
|
|
33946
35318
|
return text.toLowerCase().split(/[^a-z0-9]+/u).filter((t) => t.length > 2);
|
|
33947
35319
|
}
|
|
33948
35320
|
|
|
@@ -33984,7 +35356,7 @@ function createSeededRng2(seed) {
|
|
|
33984
35356
|
}
|
|
33985
35357
|
};
|
|
33986
35358
|
}
|
|
33987
|
-
var tokenizeContent =
|
|
35359
|
+
var tokenizeContent = tokenize5;
|
|
33988
35360
|
function recoveryTokensFor(memory) {
|
|
33989
35361
|
if (memory.tokens && memory.tokens.length > 0) {
|
|
33990
35362
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -34463,11 +35835,11 @@ function createSyntheticTarget(options) {
|
|
|
34463
35835
|
}
|
|
34464
35836
|
const normalized = memories.map((m) => ({
|
|
34465
35837
|
memory: m,
|
|
34466
|
-
tokens: new Set((m.tokens ??
|
|
35838
|
+
tokens: new Set((m.tokens ?? tokenize5(m.content)).map((t) => t.toLowerCase()))
|
|
34467
35839
|
}));
|
|
34468
35840
|
return {
|
|
34469
35841
|
async recall(query, recallOptions) {
|
|
34470
|
-
const qTokens =
|
|
35842
|
+
const qTokens = tokenize5(query);
|
|
34471
35843
|
if (qTokens.length === 0) return [];
|
|
34472
35844
|
const requestedNs = recallOptions?.namespace;
|
|
34473
35845
|
if (enforceNamespaceAcl && requestedNs !== void 0 && requestedNs !== allowedNamespace) {
|
|
@@ -34711,7 +36083,7 @@ function createMitigatedTarget(config) {
|
|
|
34711
36083
|
}
|
|
34712
36084
|
|
|
34713
36085
|
// src/coding-graph/generator.ts
|
|
34714
|
-
import { createHash as
|
|
36086
|
+
import { createHash as createHash13 } from "crypto";
|
|
34715
36087
|
function createSeededRng3(seed) {
|
|
34716
36088
|
let state = seed >>> 0;
|
|
34717
36089
|
return function rng() {
|
|
@@ -34740,7 +36112,7 @@ var EDGE_TYPE_WEIGHTS = [
|
|
|
34740
36112
|
var PROVENANCE_VALUES = ["heuristic", "heuristic", "heuristic", "trace"];
|
|
34741
36113
|
var AVG_BYTES_PER_LINE = 40;
|
|
34742
36114
|
function hashContent(input) {
|
|
34743
|
-
return
|
|
36115
|
+
return createHash13("sha256").update(input).digest("hex").slice(0, 16);
|
|
34744
36116
|
}
|
|
34745
36117
|
function generateSyntheticRepo(config) {
|
|
34746
36118
|
const rng = createSeededRng3(config.seed);
|
|
@@ -34838,7 +36210,7 @@ import { performance as performance2 } from "perf_hooks";
|
|
|
34838
36210
|
import { mkdtemp as mkdtemp12, rm as rm14 } from "fs/promises";
|
|
34839
36211
|
import { statSync } from "fs";
|
|
34840
36212
|
import { tmpdir as tmpdir7 } from "os";
|
|
34841
|
-
import
|
|
36213
|
+
import path37 from "path";
|
|
34842
36214
|
import os10 from "os";
|
|
34843
36215
|
import {
|
|
34844
36216
|
GraphStore
|
|
@@ -34935,8 +36307,8 @@ async function runCodingGraphBenchmark(config = {}) {
|
|
|
34935
36307
|
const sampleRss = () => {
|
|
34936
36308
|
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
|
34937
36309
|
};
|
|
34938
|
-
const dir = await mkdtemp12(
|
|
34939
|
-
const dbPath =
|
|
36310
|
+
const dir = await mkdtemp12(path37.join(tmpdir7(), "coding-graph-bench-"));
|
|
36311
|
+
const dbPath = path37.join(dir, "bench.sqlite");
|
|
34940
36312
|
try {
|
|
34941
36313
|
const store = await GraphStore.open({ dbPath });
|
|
34942
36314
|
try {
|