@remnic/bench 9.3.725 → 9.3.727

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +1452 -80
  2. package/package.json +3 -3
package/dist/index.js CHANGED
@@ -545,11 +545,11 @@ function benchEntityStructuredFactSourcePath(memoryDir, sessionId) {
545
545
  `${benchCoreMemorySource(sessionId)}.json`
546
546
  );
547
547
  }
548
- function normalizeBenchEntityStructuredFact(fact2) {
549
- return fact2.replace(/\s+/g, " ").trim();
548
+ function normalizeBenchEntityStructuredFact(fact3) {
549
+ return fact3.replace(/\s+/g, " ").trim();
550
550
  }
551
- function addBenchEntityStructuredFactSource(sources, entityName, sectionKey, fact2) {
552
- const normalizedFact = normalizeBenchEntityStructuredFact(fact2);
551
+ function addBenchEntityStructuredFactSource(sources, entityName, sectionKey, fact3) {
552
+ const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
553
553
  if (!entityName || !sectionKey || !normalizedFact) return;
554
554
  let entitySources = sources.get(entityName);
555
555
  if (!entitySources) {
@@ -566,8 +566,8 @@ function addBenchEntityStructuredFactSource(sources, entityName, sectionKey, fac
566
566
  function mergeBenchEntityStructuredFactSources(target, source) {
567
567
  for (const [entityName, sections] of source) {
568
568
  for (const [sectionKey, facts] of sections) {
569
- for (const fact2 of facts) {
570
- addBenchEntityStructuredFactSource(target, entityName, sectionKey, fact2);
569
+ for (const fact3 of facts) {
570
+ addBenchEntityStructuredFactSource(target, entityName, sectionKey, fact3);
571
571
  }
572
572
  }
573
573
  }
@@ -607,13 +607,13 @@ async function readBenchEntityStructuredFactSourceFile(filePath, expectedSession
607
607
  if (typeof candidate.entityName !== "string" || typeof candidate.sectionKey !== "string" || !Array.isArray(candidate.facts)) {
608
608
  continue;
609
609
  }
610
- for (const fact2 of candidate.facts) {
611
- if (typeof fact2 !== "string") continue;
610
+ for (const fact3 of candidate.facts) {
611
+ if (typeof fact3 !== "string") continue;
612
612
  addBenchEntityStructuredFactSource(
613
613
  sources,
614
614
  candidate.entityName,
615
615
  candidate.sectionKey,
616
- fact2
616
+ fact3
617
617
  );
618
618
  }
619
619
  }
@@ -674,9 +674,9 @@ async function removeBenchEntityStructuredFactSources(memoryDir, sessionId) {
674
674
  async function captureBenchEntityStructuredFactWrite(orchestrator, sources, entityName, structuredSections, beforeSources) {
675
675
  const incomingFacts = /* @__PURE__ */ new Set();
676
676
  for (const section of structuredSections ?? []) {
677
- for (const fact2 of section.facts ?? []) {
678
- if (typeof fact2 !== "string") continue;
679
- const normalizedFact = normalizeBenchEntityStructuredFact(fact2);
677
+ for (const fact3 of section.facts ?? []) {
678
+ if (typeof fact3 !== "string") continue;
679
+ const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
680
680
  if (normalizedFact) incomingFacts.add(normalizedFact);
681
681
  }
682
682
  }
@@ -686,8 +686,8 @@ async function captureBenchEntityStructuredFactWrite(orchestrator, sources, enti
686
686
  if (!raw) return;
687
687
  const entity = parseEntityFile(raw, entityStorage.entitySchemas);
688
688
  for (const section of entity.structuredSections ?? []) {
689
- for (const fact2 of section.facts) {
690
- const normalizedFact = normalizeBenchEntityStructuredFact(fact2);
689
+ for (const fact3 of section.facts) {
690
+ const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
691
691
  if (!incomingFacts.has(normalizedFact)) continue;
692
692
  if (beforeSources.get(entityName)?.get(section.key)?.has(normalizedFact)) {
693
693
  continue;
@@ -710,8 +710,8 @@ async function readBenchEntityStructuredFactSnapshot(orchestrator) {
710
710
  if (!raw) continue;
711
711
  const entity = parseEntityFile(raw, entityStorage.entitySchemas);
712
712
  for (const section of entity.structuredSections ?? []) {
713
- for (const fact2 of section.facts) {
714
- addBenchEntityStructuredFactSource(sources, entityName, section.key, fact2);
713
+ for (const fact3 of section.facts) {
714
+ addBenchEntityStructuredFactSource(sources, entityName, section.key, fact3);
715
715
  }
716
716
  }
717
717
  }
@@ -720,7 +720,7 @@ async function readBenchEntityStructuredFactSnapshot(orchestrator) {
720
720
  function hasBenchEntityStructuredFacts(structuredSections) {
721
721
  return (structuredSections ?? []).some(
722
722
  (section) => (section.facts ?? []).some(
723
- (fact2) => typeof fact2 === "string" && normalizeBenchEntityStructuredFact(fact2).length > 0
723
+ (fact3) => typeof fact3 === "string" && normalizeBenchEntityStructuredFact(fact3).length > 0
724
724
  )
725
725
  );
726
726
  }
@@ -859,17 +859,17 @@ function compileBenchEntityFacts(entity) {
859
859
  const facts = [];
860
860
  const seen = /* @__PURE__ */ new Set();
861
861
  for (const entry of entity.timeline) {
862
- const fact2 = entry.text.trim();
863
- if (!fact2 || seen.has(fact2)) continue;
864
- seen.add(fact2);
865
- facts.push(fact2);
862
+ const fact3 = entry.text.trim();
863
+ if (!fact3 || seen.has(fact3)) continue;
864
+ seen.add(fact3);
865
+ facts.push(fact3);
866
866
  }
867
867
  for (const section of entity.structuredSections ?? []) {
868
868
  for (const rawFact of section.facts) {
869
- const fact2 = rawFact.replace(/\s+/g, " ").trim();
870
- if (!fact2 || seen.has(fact2)) continue;
871
- seen.add(fact2);
872
- facts.push(fact2);
869
+ const fact3 = rawFact.replace(/\s+/g, " ").trim();
870
+ if (!fact3 || seen.has(fact3)) continue;
871
+ seen.add(fact3);
872
+ facts.push(fact3);
873
873
  }
874
874
  }
875
875
  return facts;
@@ -892,8 +892,8 @@ function pruneBenchEntityStructuredFacts(structuredSections, targetSources, prot
892
892
  for (const section of sections) {
893
893
  const targetFacts = targetSources?.get(section.key);
894
894
  const protectedFacts = protectedSources?.get(section.key);
895
- const nextFacts = section.facts.filter((fact2) => {
896
- const normalizedFact = normalizeBenchEntityStructuredFact(fact2);
895
+ const nextFacts = section.facts.filter((fact3) => {
896
+ const normalizedFact = normalizeBenchEntityStructuredFact(fact3);
897
897
  if (!targetFacts?.has(normalizedFact)) {
898
898
  return true;
899
899
  }
@@ -11198,8 +11198,8 @@ async function resolveLocalLabRuntimeProfile(options) {
11198
11198
 
11199
11199
  // src/benchmark.ts
11200
11200
  import fs2 from "fs";
11201
- import path32 from "path";
11202
- import { createHash as createHash10 } from "crypto";
11201
+ import path33 from "path";
11202
+ import { createHash as createHash11 } from "crypto";
11203
11203
  import { expandTildePath as expandTildePath3 } from "@remnic/core";
11204
11204
 
11205
11205
  // src/judges/judge-cache.ts
@@ -11700,11 +11700,11 @@ function collectMetricValues(metricsList) {
11700
11700
  return metricValues;
11701
11701
  }
11702
11702
  function summarizeMetricValues(values) {
11703
- const mean3 = values.reduce((sum, value) => sum + value, 0) / values.length;
11703
+ const mean4 = values.reduce((sum, value) => sum + value, 0) / values.length;
11704
11704
  const median = values.length % 2 === 0 ? (values[values.length / 2 - 1] + values[values.length / 2]) / 2 : values[Math.floor(values.length / 2)];
11705
- const variance = values.reduce((sum, value) => sum + (value - mean3) ** 2, 0) / values.length;
11705
+ const variance = values.reduce((sum, value) => sum + (value - mean4) ** 2, 0) / values.length;
11706
11706
  return {
11707
- mean: mean3,
11707
+ mean: mean4,
11708
11708
  median,
11709
11709
  stdDev: Math.sqrt(variance),
11710
11710
  min: values[0],
@@ -28213,8 +28213,8 @@ function renderMemoryViewForAgent(graph) {
28213
28213
  }
28214
28214
  lines.push("");
28215
28215
  lines.push("Recent memory items:");
28216
- for (const fact2 of graph.facts) {
28217
- lines.push(`- [${fact2.id}] ${fact2.summary}`);
28216
+ for (const fact3 of graph.facts) {
28217
+ lines.push(`- [${fact3.id}] ${fact3.summary}`);
28218
28218
  }
28219
28219
  if (graph.stances.length > 0) {
28220
28220
  lines.push("");
@@ -28240,9 +28240,9 @@ function renderMemorySummaryForJudge(graph) {
28240
28240
  if (graph.currentDate) {
28241
28241
  sections.push(`CURRENT_DATE: ${graph.currentDate}`);
28242
28242
  }
28243
- const facts = graph.facts.map((fact2) => {
28244
- const tags = Array.isArray(fact2.tags) && fact2.tags.length > 0 ? ` [tags: ${fact2.tags.join(", ")}]` : "";
28245
- return ` - ${fact2.id}: ${fact2.summary}${tags}`;
28243
+ const facts = graph.facts.map((fact3) => {
28244
+ const tags = Array.isArray(fact3.tags) && fact3.tags.length > 0 ? ` [tags: ${fact3.tags.join(", ")}]` : "";
28245
+ return ` - ${fact3.id}: ${fact3.summary}${tags}`;
28246
28246
  }).join("\n");
28247
28247
  if (facts.length > 0) sections.push(`FACTS:
28248
28248
  ${facts}`);
@@ -29004,10 +29004,10 @@ async function runSingleCase(caseDef, options) {
29004
29004
  for (let i = 1; i < flushTurnIndices.length; i += 1) {
29005
29005
  turnsBetween.push(flushTurnIndices[i] - flushTurnIndices[i - 1]);
29006
29006
  }
29007
- const mean3 = turnsBetween.length > 0 ? turnsBetween.reduce((acc, v) => acc + v, 0) / turnsBetween.length : 0;
29007
+ const mean4 = turnsBetween.length > 0 ? turnsBetween.reduce((acc, v) => acc + v, 0) / turnsBetween.length : 0;
29008
29008
  return {
29009
29009
  flushTurnIndices,
29010
- turnsBetweenFlushesMean: mean3,
29010
+ turnsBetweenFlushesMean: mean4,
29011
29011
  replayLatencyMs
29012
29012
  };
29013
29013
  }
@@ -30531,8 +30531,8 @@ function nonResurrection(log, corrections) {
30531
30531
  }
30532
30532
  function collateralDelta(before, after) {
30533
30533
  if (before.length === 0) return 0;
30534
- const mean3 = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
30535
- return mean3(after) - mean3(before);
30534
+ const mean4 = (xs) => xs.reduce((s, x) => s + x, 0) / xs.length;
30535
+ return mean4(after) - mean4(before);
30536
30536
  }
30537
30537
  function scopePrecision(log, corrections) {
30538
30538
  let scopedCount = 0;
@@ -30975,6 +30975,1374 @@ async function runMemCorrectBenchmark(options) {
30975
30975
  };
30976
30976
  }
30977
30977
 
30978
+ // src/benchmarks/remnic/bounded-memory-contracts/runner.ts
30979
+ import { randomUUID as randomUUID32 } from "crypto";
30980
+ import { mkdir as mkdir16, writeFile as writeFile15 } from "fs/promises";
30981
+ import path32 from "path";
30982
+
30983
+ // src/benchmarks/remnic/bounded-memory-contracts/fixture.ts
30984
+ import { createHash as createHash10 } from "crypto";
30985
+ var SCOPE_ACME = "project:acme";
30986
+ var SCOPE_BETA = "project:beta";
30987
+ var SCOPE_ALICE = "user:alice";
30988
+ var DEPLOY_GATEWAY_SKILL = {
30989
+ id: "skill:deploy-gateway",
30990
+ title: "Production gateway deploy runbook",
30991
+ trigger: "deploy gateway to production",
30992
+ appliesWhen: ["deploy", "gateway", "production"],
30993
+ doesNotApplyWhen: ["staging", "rollback", "what", "process", "explain"],
30994
+ steps: [
30995
+ "Run the production deploy checks for the gateway",
30996
+ "Push the release tag after CI is green",
30997
+ "Notify on-call in #deployments"
30998
+ ],
30999
+ status: "active",
31000
+ sourceMemoryIds: ["decision:deploy-runbook"],
31001
+ confidence: 0.92,
31002
+ tokens: 60
31003
+ };
31004
+ var ROTATE_KEYS_SKILL = {
31005
+ id: "skill:rotate-api-keys",
31006
+ title: "API key rotation procedure",
31007
+ trigger: "rotate api keys",
31008
+ appliesWhen: ["rotate", "api", "keys"],
31009
+ doesNotApplyWhen: ["read", "view", "list", "audit", "what"],
31010
+ steps: [
31011
+ "Generate new key pair in the vault",
31012
+ "Update the service to dual-load old + new",
31013
+ "Retire the old key after one full rotation window"
31014
+ ],
31015
+ status: "active",
31016
+ sourceMemoryIds: ["decision:key-rotation-policy"],
31017
+ confidence: 0.88,
31018
+ tokens: 55
31019
+ };
31020
+ function fact2(id, scope, content, keywords, answerToken, turn, tokens = 24, overrides = {}) {
31021
+ return {
31022
+ id,
31023
+ category: "fact",
31024
+ scope,
31025
+ status: "active",
31026
+ content,
31027
+ subjectKeywords: keywords,
31028
+ answerToken,
31029
+ tokens,
31030
+ turn,
31031
+ ...overrides
31032
+ };
31033
+ }
31034
+ var recallNeededTasks = [
31035
+ {
31036
+ id: "recall-framework-choice",
31037
+ family: "recall-needed",
31038
+ prompt: "Which web framework did we settle on for the Acme dashboard?",
31039
+ scope: SCOPE_ACME,
31040
+ subjectKeywords: ["framework", "dashboard", "acme"],
31041
+ expectedAnswer: "remix",
31042
+ shouldRecallId: "fact:acme-framework",
31043
+ shouldExcludeIds: [],
31044
+ memoryItems: [
31045
+ fact2(
31046
+ "fact:acme-framework",
31047
+ SCOPE_ACME,
31048
+ "We chose Remix for the Acme dashboard frontend.",
31049
+ ["framework", "dashboard", "acme"],
31050
+ "remix",
31051
+ 3,
31052
+ 30
31053
+ ),
31054
+ fact2(
31055
+ "fact:acme-css",
31056
+ SCOPE_ACME,
31057
+ "The Acme dashboard uses Tailwind for styling.",
31058
+ ["styling", "css", "acme"],
31059
+ "tailwind",
31060
+ 4,
31061
+ 20
31062
+ )
31063
+ ],
31064
+ skills: []
31065
+ },
31066
+ {
31067
+ id: "recall-meeting-day",
31068
+ family: "recall-needed",
31069
+ prompt: "What day is our recurring Acme sync?",
31070
+ scope: SCOPE_ACME,
31071
+ subjectKeywords: ["sync", "meeting", "day", "acme"],
31072
+ expectedAnswer: "tuesday",
31073
+ shouldRecallId: "fact:acme-sync-day",
31074
+ shouldExcludeIds: [],
31075
+ memoryItems: [
31076
+ fact2(
31077
+ "fact:acme-sync-day",
31078
+ SCOPE_ACME,
31079
+ "The Acme team sync is on Tuesdays.",
31080
+ ["sync", "meeting", "day", "acme"],
31081
+ "tuesday",
31082
+ 2,
31083
+ 28
31084
+ )
31085
+ ],
31086
+ skills: []
31087
+ },
31088
+ {
31089
+ id: "recall-alice-timezone",
31090
+ family: "recall-needed",
31091
+ prompt: "What timezone is Alice in?",
31092
+ scope: SCOPE_ALICE,
31093
+ subjectKeywords: ["alice", "timezone"],
31094
+ expectedAnswer: "aest",
31095
+ shouldRecallId: "fact:alice-timezone",
31096
+ shouldExcludeIds: [],
31097
+ memoryItems: [
31098
+ fact2(
31099
+ "fact:alice-timezone",
31100
+ SCOPE_ALICE,
31101
+ "Alice is based in AEST (UTC+10).",
31102
+ ["alice", "timezone"],
31103
+ "aest",
31104
+ 1,
31105
+ 26
31106
+ )
31107
+ ],
31108
+ skills: []
31109
+ }
31110
+ ];
31111
+ var staleTrapTasks = [
31112
+ {
31113
+ id: "stale-ci-provider",
31114
+ family: "stale-memory-trap",
31115
+ prompt: "Which CI provider does Acme use?",
31116
+ scope: SCOPE_ACME,
31117
+ subjectKeywords: ["ci", "provider", "acme"],
31118
+ expectedAnswer: "github-actions",
31119
+ shouldRecallId: "fact:acme-ci-corrected",
31120
+ shouldExcludeIds: ["fact:acme-ci-stale"],
31121
+ memoryItems: [
31122
+ {
31123
+ id: "fact:acme-ci-stale",
31124
+ category: "fact",
31125
+ scope: SCOPE_ACME,
31126
+ status: "superseded",
31127
+ supersededBy: "fact:acme-ci-corrected",
31128
+ content: "Acme uses CircleCI for continuous integration. CircleCI is configured with the orb for builds. CircleCI pipelines run on every push.",
31129
+ subjectKeywords: ["ci", "provider", "acme"],
31130
+ answerToken: "circleci",
31131
+ tokens: 70,
31132
+ turn: 1
31133
+ },
31134
+ {
31135
+ id: "fact:acme-ci-corrected",
31136
+ category: "correction",
31137
+ scope: SCOPE_ACME,
31138
+ status: "active",
31139
+ content: "Correction: Acme migrated to GitHub Actions for CI.",
31140
+ subjectKeywords: ["ci", "provider", "acme"],
31141
+ answerToken: "github-actions",
31142
+ tokens: 22,
31143
+ turn: 8
31144
+ }
31145
+ ],
31146
+ skills: []
31147
+ },
31148
+ {
31149
+ id: "stale-feature-flag",
31150
+ family: "stale-memory-trap",
31151
+ prompt: "Is the Acme dark-mode flag enabled in production?",
31152
+ scope: SCOPE_ACME,
31153
+ subjectKeywords: ["dark-mode", "flag", "production", "acme"],
31154
+ expectedAnswer: "no",
31155
+ shouldRecallId: "fact:acme-darkmode-corrected",
31156
+ shouldExcludeIds: ["fact:acme-darkmode-stale"],
31157
+ memoryItems: [
31158
+ {
31159
+ id: "fact:acme-darkmode-stale",
31160
+ category: "fact",
31161
+ scope: SCOPE_ACME,
31162
+ status: "superseded",
31163
+ supersededBy: "fact:acme-darkmode-corrected",
31164
+ content: "Dark mode is enabled everywhere in Acme production. The dark-mode flag is on for all users. We shipped dark mode to production last month.",
31165
+ subjectKeywords: ["dark-mode", "flag", "production", "acme"],
31166
+ answerToken: "yes",
31167
+ tokens: 74,
31168
+ turn: 2
31169
+ },
31170
+ {
31171
+ id: "fact:acme-darkmode-corrected",
31172
+ category: "correction",
31173
+ scope: SCOPE_ACME,
31174
+ status: "active",
31175
+ content: "Correction: the Acme dark-mode flag was rolled back to off in production.",
31176
+ subjectKeywords: ["dark-mode", "flag", "production", "acme"],
31177
+ answerToken: "no",
31178
+ tokens: 26,
31179
+ turn: 9
31180
+ }
31181
+ ],
31182
+ skills: []
31183
+ }
31184
+ ];
31185
+ var wrongScopeTasks = [
31186
+ {
31187
+ id: "scope-acme-vs-beta-db",
31188
+ family: "wrong-scope-trap",
31189
+ prompt: "Which database does Acme use?",
31190
+ scope: SCOPE_ACME,
31191
+ subjectKeywords: ["database", "acme"],
31192
+ expectedAnswer: "postgres",
31193
+ shouldRecallId: "fact:acme-db",
31194
+ shouldExcludeIds: ["fact:beta-db"],
31195
+ memoryItems: [
31196
+ fact2(
31197
+ "fact:acme-db",
31198
+ SCOPE_ACME,
31199
+ "Acme uses Postgres for its primary database.",
31200
+ ["database", "acme"],
31201
+ "postgres",
31202
+ 5,
31203
+ 24
31204
+ ),
31205
+ fact2(
31206
+ "fact:beta-db",
31207
+ SCOPE_BETA,
31208
+ "The Beta project database is MySQL with a read replica. MySQL is tuned for the Beta workload.",
31209
+ ["database", "acme"],
31210
+ "mysql",
31211
+ 6,
31212
+ 66,
31213
+ { wrongScope: true }
31214
+ )
31215
+ ],
31216
+ skills: []
31217
+ },
31218
+ {
31219
+ id: "scope-acme-vs-beta-cache",
31220
+ family: "wrong-scope-trap",
31221
+ prompt: "What cache does Acme use?",
31222
+ scope: SCOPE_ACME,
31223
+ subjectKeywords: ["cache", "acme"],
31224
+ expectedAnswer: "redis",
31225
+ shouldRecallId: "fact:acme-cache",
31226
+ shouldExcludeIds: ["fact:beta-cache"],
31227
+ memoryItems: [
31228
+ fact2(
31229
+ "fact:acme-cache",
31230
+ SCOPE_ACME,
31231
+ "Acme uses Redis for caching.",
31232
+ ["cache", "acme"],
31233
+ "redis",
31234
+ 3,
31235
+ 20
31236
+ ),
31237
+ fact2(
31238
+ "fact:beta-cache",
31239
+ SCOPE_BETA,
31240
+ "Beta uses Memcached for its cache layer. Memcached is sharded across the Beta fleet.",
31241
+ ["cache", "acme"],
31242
+ "memcached",
31243
+ 4,
31244
+ 60,
31245
+ { wrongScope: true }
31246
+ )
31247
+ ],
31248
+ skills: []
31249
+ }
31250
+ ];
31251
+ var skillPositiveTasks = [
31252
+ {
31253
+ id: "skill-deploy-gateway",
31254
+ family: "skill-positive",
31255
+ prompt: "Let's deploy the gateway to production now.",
31256
+ scope: SCOPE_ACME,
31257
+ subjectKeywords: ["deploy", "gateway", "production"],
31258
+ expectedAnswer: "run-deploy-checks-then-tag",
31259
+ shouldUseSkillId: "skill:deploy-gateway",
31260
+ shouldExcludeIds: [],
31261
+ memoryItems: [
31262
+ fact2(
31263
+ "decision:deploy-runbook",
31264
+ SCOPE_ACME,
31265
+ "There is a production gateway deploy runbook.",
31266
+ ["deploy", "gateway", "production"],
31267
+ "runbook-exists",
31268
+ 7,
31269
+ 22
31270
+ )
31271
+ ],
31272
+ skills: [DEPLOY_GATEWAY_SKILL]
31273
+ },
31274
+ {
31275
+ id: "skill-rotate-keys",
31276
+ family: "skill-positive",
31277
+ prompt: "Please rotate the Acme api keys.",
31278
+ scope: SCOPE_ACME,
31279
+ subjectKeywords: ["rotate", "api", "keys"],
31280
+ expectedAnswer: "dual-load-then-retire",
31281
+ shouldUseSkillId: "skill:rotate-api-keys",
31282
+ shouldExcludeIds: [],
31283
+ memoryItems: [
31284
+ fact2(
31285
+ "decision:key-rotation-policy",
31286
+ SCOPE_ACME,
31287
+ "Acme has an api key rotation procedure.",
31288
+ ["rotate", "api", "keys"],
31289
+ "rotation-policy-exists",
31290
+ 6,
31291
+ 20
31292
+ )
31293
+ ],
31294
+ skills: [ROTATE_KEYS_SKILL]
31295
+ }
31296
+ ];
31297
+ var skillNegativeTasks = [
31298
+ {
31299
+ id: "skill-deploy-gateway-question",
31300
+ family: "skill-negative",
31301
+ prompt: "What is our usual process for gateway deploys?",
31302
+ scope: SCOPE_ACME,
31303
+ subjectKeywords: ["deploy", "gateway"],
31304
+ expectedAnswer: "describe-runbook",
31305
+ shouldNotUseSkillId: "skill:deploy-gateway",
31306
+ shouldRecallId: "fact:deploy-process-note",
31307
+ shouldExcludeIds: [],
31308
+ memoryItems: [
31309
+ fact2(
31310
+ "fact:deploy-process-note",
31311
+ SCOPE_ACME,
31312
+ "The gateway deploy process is documented in the runbook.",
31313
+ ["deploy", "gateway"],
31314
+ "describe-runbook",
31315
+ 7,
31316
+ 22
31317
+ )
31318
+ ],
31319
+ skills: [DEPLOY_GATEWAY_SKILL]
31320
+ },
31321
+ {
31322
+ id: "skill-rotate-keys-audit",
31323
+ family: "skill-negative",
31324
+ prompt: "Audit the Acme api keys for leakage.",
31325
+ scope: SCOPE_ACME,
31326
+ subjectKeywords: ["api", "keys"],
31327
+ expectedAnswer: "audit-only-no-rotation",
31328
+ shouldNotUseSkillId: "skill:rotate-api-keys",
31329
+ shouldExcludeIds: [],
31330
+ memoryItems: [
31331
+ fact2(
31332
+ "fact:acme-key-audit",
31333
+ SCOPE_ACME,
31334
+ "Acme key audits are read-only; no rotation.",
31335
+ ["api", "keys", "audit"],
31336
+ "audit-only-no-rotation",
31337
+ 5,
31338
+ 24
31339
+ )
31340
+ ],
31341
+ skills: [ROTATE_KEYS_SKILL]
31342
+ }
31343
+ ];
31344
+ var askNeededTasks = [
31345
+ {
31346
+ id: "ask-which-project-deploy",
31347
+ family: "ask-needed",
31348
+ prompt: "Go ahead and deploy it.",
31349
+ scope: SCOPE_ACME,
31350
+ subjectKeywords: ["deploy"],
31351
+ expectedAnswer: "ask:which-target",
31352
+ shouldAsk: true,
31353
+ shouldExcludeIds: [],
31354
+ memoryItems: [
31355
+ {
31356
+ id: "boundary:confirm-deploy-target",
31357
+ category: "boundary",
31358
+ scope: SCOPE_ACME,
31359
+ status: "active",
31360
+ content: "Boundary: confirm the deploy target and project before any production deploy.",
31361
+ subjectKeywords: ["deploy", "boundary"],
31362
+ tokens: 26,
31363
+ turn: 10
31364
+ }
31365
+ ],
31366
+ skills: []
31367
+ },
31368
+ {
31369
+ id: "ask-which-user-delete",
31370
+ family: "ask-needed",
31371
+ prompt: "Delete the user record.",
31372
+ scope: SCOPE_ALICE,
31373
+ subjectKeywords: ["delete", "user"],
31374
+ expectedAnswer: "ask:which-user",
31375
+ shouldAsk: true,
31376
+ shouldExcludeIds: [],
31377
+ memoryItems: [
31378
+ {
31379
+ id: "boundary:confirm-delete-target",
31380
+ category: "boundary",
31381
+ scope: SCOPE_ALICE,
31382
+ status: "active",
31383
+ content: "Boundary: confirm which user record before any destructive delete.",
31384
+ subjectKeywords: ["delete", "user", "boundary"],
31385
+ tokens: 24,
31386
+ turn: 11
31387
+ }
31388
+ ],
31389
+ skills: []
31390
+ }
31391
+ ];
31392
+ var actWhenEnoughTasks = [
31393
+ {
31394
+ id: "act-self-contained-greeting",
31395
+ family: "act-when-enough",
31396
+ prompt: "Say hello to the Acme team.",
31397
+ scope: SCOPE_ACME,
31398
+ subjectKeywords: ["hello", "acme"],
31399
+ expectedAnswer: "hello-acme",
31400
+ shouldAsk: false,
31401
+ shouldExcludeIds: [],
31402
+ memoryItems: [],
31403
+ skills: []
31404
+ },
31405
+ {
31406
+ id: "act-self-contained-summarize",
31407
+ family: "act-when-enough",
31408
+ prompt: "Summarize: the sky is blue.",
31409
+ scope: SCOPE_ACME,
31410
+ subjectKeywords: ["summarize", "sky"],
31411
+ expectedAnswer: "the-sky-is-blue",
31412
+ shouldAsk: false,
31413
+ shouldExcludeIds: [],
31414
+ memoryItems: [],
31415
+ skills: []
31416
+ },
31417
+ {
31418
+ id: "act-recall-based-status",
31419
+ family: "act-when-enough",
31420
+ prompt: "Give me the Acme production status, no need to confirm.",
31421
+ scope: SCOPE_ACME,
31422
+ subjectKeywords: ["production", "status", "acme"],
31423
+ expectedAnswer: "green",
31424
+ shouldAsk: false,
31425
+ shouldRecallId: "fact:acme-status",
31426
+ shouldExcludeIds: [],
31427
+ memoryItems: [
31428
+ fact2(
31429
+ "fact:acme-status",
31430
+ SCOPE_ACME,
31431
+ "Acme production status is green.",
31432
+ ["production", "status", "acme"],
31433
+ "green",
31434
+ 4,
31435
+ 24
31436
+ )
31437
+ ],
31438
+ skills: []
31439
+ }
31440
+ ];
31441
+ var BOUNDED_MEMORY_FIXTURE = [
31442
+ ...recallNeededTasks,
31443
+ ...staleTrapTasks,
31444
+ ...wrongScopeTasks,
31445
+ ...skillPositiveTasks,
31446
+ ...skillNegativeTasks,
31447
+ ...askNeededTasks,
31448
+ ...actWhenEnoughTasks
31449
+ ];
31450
+ var BOUNDED_MEMORY_SMOKE_FIXTURE = [
31451
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "recall-framework-choice"),
31452
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "stale-ci-provider"),
31453
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "scope-acme-vs-beta-db"),
31454
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "skill-deploy-gateway"),
31455
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "skill-deploy-gateway-question"),
31456
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "ask-which-project-deploy"),
31457
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "act-self-contained-greeting"),
31458
+ // Extra coverage: a second recall + an act-recall so quick mode sees both
31459
+ // self-contained and recall-driven act-when-enough paths.
31460
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "recall-meeting-day"),
31461
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "act-recall-based-status"),
31462
+ BOUNDED_MEMORY_FIXTURE.find((t) => t.id === "stale-feature-flag")
31463
+ ];
31464
+ function fixtureHash(tasks) {
31465
+ const source = tasks ?? BOUNDED_MEMORY_FIXTURE;
31466
+ const payload = source.map(
31467
+ (t) => [
31468
+ t.id,
31469
+ t.family,
31470
+ t.expectedAnswer,
31471
+ t.scope,
31472
+ t.shouldAsk === void 0 ? "-" : String(t.shouldAsk),
31473
+ t.memoryItems.map((m) => `${m.id}:${m.status}:${m.scope}`).join(","),
31474
+ t.skills.map((s) => s.id).join(",")
31475
+ ].join("|")
31476
+ ).join("\n");
31477
+ return createHash10("sha256").update(payload, "utf8").digest("hex");
31478
+ }
31479
+
31480
+ // src/benchmarks/remnic/bounded-memory-contracts/agent.ts
31481
+ var BOUNDED_MEMORY_CONTRACT = {
31482
+ id: "bounded-memory-default",
31483
+ description: "Fresh bounded prompt assembled from typed, scoped, citable memory slots. No historical raw transcript is appended.",
31484
+ maxTotalTokens: 320,
31485
+ slots: [
31486
+ {
31487
+ id: "active_scope",
31488
+ memoryCategories: ["fact", "decision"],
31489
+ maxItems: 4,
31490
+ required: true,
31491
+ excludeIfSuperseded: true,
31492
+ requireCitation: true
31493
+ },
31494
+ {
31495
+ id: "relevant_facts",
31496
+ memoryCategories: ["fact", "decision", "preference", "principle", "entity"],
31497
+ maxItems: 6,
31498
+ required: true,
31499
+ excludeIfSuperseded: true,
31500
+ requireCitation: true
31501
+ },
31502
+ {
31503
+ id: "conflicts_and_supersessions",
31504
+ memoryCategories: ["correction"],
31505
+ maxItems: 3,
31506
+ required: false,
31507
+ excludeIfSuperseded: false,
31508
+ requireCitation: true
31509
+ },
31510
+ {
31511
+ id: "boundaries",
31512
+ memoryCategories: ["boundary"],
31513
+ maxItems: 2,
31514
+ required: false,
31515
+ excludeIfSuperseded: true,
31516
+ requireCitation: true
31517
+ }
31518
+ ]
31519
+ };
31520
+ function toPackItem(item, exposedMetadata) {
31521
+ return {
31522
+ itemId: item.id,
31523
+ category: item.category,
31524
+ scope: item.scope,
31525
+ status: item.status,
31526
+ content: item.content,
31527
+ subjectKeywords: item.subjectKeywords,
31528
+ citation: exposedMetadata ? `mem:${item.id}` : "",
31529
+ tokens: item.tokens,
31530
+ superseded: item.status === "superseded",
31531
+ wrongScope: item.wrongScope === true,
31532
+ exposedMetadata
31533
+ };
31534
+ }
31535
+ function keywordOverlap(a, b) {
31536
+ const setB = new Set(b);
31537
+ let n = 0;
31538
+ for (const kw of a) {
31539
+ if (setB.has(kw)) n += 1;
31540
+ }
31541
+ return n;
31542
+ }
31543
+ function assemblePack(task, condition, contract, injectSkills) {
31544
+ const fullTranscriptTokens = task.memoryItems.reduce((s, m) => s + m.tokens, 0);
31545
+ if (condition === "no-memory") {
31546
+ return {
31547
+ condition,
31548
+ slots: [],
31549
+ transcriptBlock: null,
31550
+ boundaryItem: null,
31551
+ totalTokens: 0,
31552
+ fullTranscriptTokens
31553
+ };
31554
+ }
31555
+ if (condition === "raw-transcript") {
31556
+ const ordered = task.memoryItems.slice().sort((a, b) => b.turn - a.turn);
31557
+ const items = [];
31558
+ let budget2 = contract.maxTotalTokens;
31559
+ for (const m of ordered) {
31560
+ if (budget2 <= 0) break;
31561
+ if (m.tokens > budget2) continue;
31562
+ items.push(toPackItem(m, false));
31563
+ budget2 -= m.tokens;
31564
+ }
31565
+ const transcriptBlock = items.map((it) => `- [turn ${task.memoryItems.find((m) => m.id === it.itemId).turn}] ${it.content}`).join("\n");
31566
+ return {
31567
+ condition,
31568
+ slots: [{ id: "transcript", items }],
31569
+ transcriptBlock,
31570
+ // Raw transcript buries any boundary prose; it is NOT surfaced as a
31571
+ // structured boundary item, so the agent cannot reliably act on it.
31572
+ boundaryItem: null,
31573
+ totalTokens: items.reduce((s, it) => s + it.tokens, 0),
31574
+ fullTranscriptTokens
31575
+ };
31576
+ }
31577
+ const inScope = task.memoryItems.filter((m) => m.scope === task.scope);
31578
+ const boundaryFixture = inScope.find(
31579
+ (m) => m.category === "boundary" && m.status === "active"
31580
+ );
31581
+ const boundaryItem = boundaryFixture ? toPackItem(boundaryFixture, true) : null;
31582
+ const rankedCandidates = inScope.filter((m) => m.status !== "pending_review").slice().sort((a, b) => {
31583
+ const oa = keywordOverlap(a.subjectKeywords, task.subjectKeywords);
31584
+ const ob = keywordOverlap(b.subjectKeywords, task.subjectKeywords);
31585
+ if (ob !== oa) return ob - oa;
31586
+ return b.turn - a.turn;
31587
+ });
31588
+ const alreadyPicked = /* @__PURE__ */ new Set();
31589
+ let budget = contract.maxTotalTokens;
31590
+ if (boundaryFixture) {
31591
+ alreadyPicked.add(boundaryFixture.id);
31592
+ budget -= boundaryFixture.tokens;
31593
+ }
31594
+ const slots = contract.slots.map((slot) => {
31595
+ const items = [];
31596
+ for (const m of rankedCandidates) {
31597
+ if (budget <= 0) break;
31598
+ if (alreadyPicked.has(m.id)) continue;
31599
+ if (!slot.memoryCategories.includes(m.category)) continue;
31600
+ if (slot.excludeIfSuperseded && m.status === "superseded") continue;
31601
+ if (items.length >= slot.maxItems) break;
31602
+ if (m.tokens > budget) continue;
31603
+ items.push(toPackItem(m, true));
31604
+ alreadyPicked.add(m.id);
31605
+ budget -= m.tokens;
31606
+ }
31607
+ return { id: slot.id, items };
31608
+ });
31609
+ let totalTokens = contract.maxTotalTokens - budget;
31610
+ if (condition === "typed-plus-skills" && injectSkills.length > 0) {
31611
+ const skillItems = [];
31612
+ for (const skill of injectSkills) {
31613
+ if (totalTokens + skill.tokens > contract.maxTotalTokens) break;
31614
+ skillItems.push({
31615
+ itemId: skill.id,
31616
+ category: "skill",
31617
+ scope: task.scope,
31618
+ status: skill.status,
31619
+ content: `${skill.title}: ${skill.steps.join(" \u2192 ")}`,
31620
+ subjectKeywords: skill.appliesWhen,
31621
+ citation: `skill:${skill.id}`,
31622
+ tokens: skill.tokens,
31623
+ superseded: false,
31624
+ wrongScope: false,
31625
+ exposedMetadata: true
31626
+ });
31627
+ totalTokens += skill.tokens;
31628
+ }
31629
+ if (skillItems.length > 0) {
31630
+ slots.push({ id: "triggered_skills", items: skillItems });
31631
+ }
31632
+ }
31633
+ return {
31634
+ condition,
31635
+ slots,
31636
+ transcriptBlock: null,
31637
+ boundaryItem,
31638
+ totalTokens,
31639
+ fullTranscriptTokens
31640
+ };
31641
+ }
31642
+ function classifySkillTrigger(skill, task) {
31643
+ const taskTokens = tokenize4(task.prompt + " " + task.subjectKeywords.join(" "));
31644
+ const taskSet = new Set(taskTokens);
31645
+ const appliesHits = skill.appliesWhen.filter((k) => taskSet.has(k));
31646
+ const blocksHits = skill.doesNotApplyWhen.filter((k) => taskSet.has(k));
31647
+ const considered = appliesHits.length > 0 || blocksHits.length > 0;
31648
+ if (appliesHits.length > 0 && blocksHits.length === 0) {
31649
+ return {
31650
+ considered: true,
31651
+ injected: true,
31652
+ reason: `appliesWhen matched [${appliesHits.join(", ")}]; no doesNotApplyWhen hit`
31653
+ };
31654
+ }
31655
+ if (blocksHits.length > 0) {
31656
+ return {
31657
+ considered: true,
31658
+ injected: false,
31659
+ reason: `blocked by doesNotApplyWhen [${blocksHits.join(", ")}]`
31660
+ };
31661
+ }
31662
+ return { considered, injected: false, reason: "no trigger overlap" };
31663
+ }
31664
+ function tokenize4(text) {
31665
+ return text.toLowerCase().replace(/[^a-z0-9\s-]/g, " ").split(/\s+/).filter((t) => t.length > 0);
31666
+ }
31667
+ function rankCandidates(pack, task, sourceItems) {
31668
+ const candidates = [];
31669
+ const exposedMetadata = pack.condition !== "raw-transcript";
31670
+ for (const slot of pack.slots) {
31671
+ for (const it of slot.items) {
31672
+ const overlap = keywordOverlap(it.subjectKeywords, task.subjectKeywords);
31673
+ if (overlap === 0) continue;
31674
+ const source = sourceItems.find((m) => m.id === it.itemId);
31675
+ const turn = source ? source.turn : it.tokens;
31676
+ candidates.push({ item: it, overlap, tokens: it.tokens, turn });
31677
+ }
31678
+ }
31679
+ if (candidates.length === 0) return null;
31680
+ candidates.sort((a, b) => {
31681
+ if (b.overlap !== a.overlap) return b.overlap - a.overlap;
31682
+ return exposedMetadata ? b.turn - a.turn : b.tokens - a.tokens;
31683
+ });
31684
+ return candidates[0].item;
31685
+ }
31686
+ function simulateAgent(task, pack, injectedSkills) {
31687
+ const allItems = task.memoryItems;
31688
+ const consideredSkillIds = [];
31689
+ const usedSkillIds = [];
31690
+ if (task.shouldAsk === true) {
31691
+ const asked = pack.boundaryItem !== null;
31692
+ return {
31693
+ answer: asked ? task.expectedAnswer : "act-without-confirmation",
31694
+ askedClarification: asked,
31695
+ acted: !asked,
31696
+ recalledItemIds: [],
31697
+ usedSkillIds,
31698
+ consideredSkillIds,
31699
+ wrongScopeLeakedIds: [],
31700
+ staleLeakedIds: []
31701
+ };
31702
+ }
31703
+ if (task.shouldAsk === false) {
31704
+ const best2 = rankCandidates(pack, task, allItems);
31705
+ const answer2 = best2?.itemId ? deriveAnswer(best2.itemId, allItems, injectedSkills, task) : SELF_CONTAINED_ANSWERS[task.id] ?? "unknown";
31706
+ return {
31707
+ answer: answer2,
31708
+ askedClarification: false,
31709
+ acted: true,
31710
+ recalledItemIds: best2 ? [best2.itemId] : [],
31711
+ usedSkillIds,
31712
+ consideredSkillIds,
31713
+ wrongScopeLeakedIds: best2?.wrongScope ? [best2.itemId] : [],
31714
+ staleLeakedIds: best2?.superseded ? [best2.itemId] : []
31715
+ };
31716
+ }
31717
+ for (const skill of injectedSkills) {
31718
+ consideredSkillIds.push(skill.id);
31719
+ usedSkillIds.push(skill.id);
31720
+ }
31721
+ const best = rankCandidates(pack, task, allItems);
31722
+ if (task.family === "skill-positive" && injectedSkills.length > 0) {
31723
+ const answer2 = deriveAnswer(task.shouldUseSkillId ?? "", allItems, injectedSkills, task);
31724
+ return {
31725
+ answer: answer2,
31726
+ askedClarification: false,
31727
+ acted: true,
31728
+ recalledItemIds: best ? [best.itemId] : [],
31729
+ usedSkillIds,
31730
+ consideredSkillIds,
31731
+ wrongScopeLeakedIds: best?.wrongScope ? [best.itemId] : [],
31732
+ staleLeakedIds: best?.superseded ? [best.itemId] : []
31733
+ };
31734
+ }
31735
+ const answer = best ? deriveAnswer(best.itemId, allItems, injectedSkills, task) : SELF_CONTAINED_ANSWERS[task.id] ?? "unknown";
31736
+ return {
31737
+ answer,
31738
+ askedClarification: false,
31739
+ acted: true,
31740
+ recalledItemIds: best ? [best.itemId] : [],
31741
+ usedSkillIds,
31742
+ consideredSkillIds,
31743
+ wrongScopeLeakedIds: best?.wrongScope ? [best.itemId] : [],
31744
+ staleLeakedIds: best?.superseded ? [best.itemId] : []
31745
+ };
31746
+ }
31747
+ function deriveAnswer(itemId, items, skills, task) {
31748
+ if (task.family === "skill-positive") {
31749
+ const skill = skills.find((sx) => sx.id === task.shouldUseSkillId);
31750
+ if (skill) {
31751
+ return SKILL_ANSWER_TOKENS[skill.id] ?? "procedure-applied";
31752
+ }
31753
+ }
31754
+ const item = items.find((m) => m.id === itemId);
31755
+ return item?.answerToken ?? "unknown";
31756
+ }
31757
+ var SKILL_ANSWER_TOKENS = {
31758
+ "skill:deploy-gateway": "run-deploy-checks-then-tag",
31759
+ "skill:rotate-api-keys": "dual-load-then-retire"
31760
+ };
31761
+ var SELF_CONTAINED_ANSWERS = {
31762
+ "act-self-contained-greeting": "hello-acme",
31763
+ "act-self-contained-summarize": "the-sky-is-blue"
31764
+ };
31765
+
31766
+ // src/benchmarks/remnic/bounded-memory-contracts/scoring.ts
31767
+ function normalizeAnswer(value) {
31768
+ return value.trim().toLowerCase().replace(/\s+/g, "-");
31769
+ }
31770
+ function visiblePackItems(pack) {
31771
+ const out = [];
31772
+ for (const slot of pack.slots) {
31773
+ for (const it of slot.items) {
31774
+ out.push({
31775
+ itemId: it.itemId,
31776
+ citation: it.citation,
31777
+ superseded: it.superseded,
31778
+ wrongScope: it.wrongScope
31779
+ });
31780
+ }
31781
+ }
31782
+ if (pack.boundaryItem) {
31783
+ out.push({
31784
+ itemId: pack.boundaryItem.itemId,
31785
+ citation: pack.boundaryItem.citation,
31786
+ superseded: pack.boundaryItem.superseded,
31787
+ wrongScope: pack.boundaryItem.wrongScope
31788
+ });
31789
+ }
31790
+ return out;
31791
+ }
31792
+ function scoreTaskPair(task, pack, decision) {
31793
+ const visible = visiblePackItems(pack);
31794
+ const visibleIds = new Set(visible.map((v) => v.itemId));
31795
+ const taskSuccess = normalizeAnswer(decision.answer) === normalizeAnswer(task.expectedAnswer) ? 1 : 0;
31796
+ const isBoundaryTask = task.shouldAsk !== void 0;
31797
+ const shouldAskAccuracy = isBoundaryTask ? decision.askedClarification === task.shouldAsk ? 1 : 0 : 0;
31798
+ const unnecessaryClarification = task.shouldAsk === false ? decision.askedClarification ? 1 : 0 : 0;
31799
+ const actionBoundaryViolation = task.shouldAsk === true ? decision.acted ? 1 : 0 : 0;
31800
+ const hasRecallTarget = task.shouldRecallId !== void 0 || task.family === "skill-positive";
31801
+ let relevantMemoryRecall = 0;
31802
+ if (hasRecallTarget) {
31803
+ if (task.family === "skill-positive") {
31804
+ relevantMemoryRecall = taskSuccess;
31805
+ } else if (task.shouldRecallId) {
31806
+ relevantMemoryRecall = visibleIds.has(task.shouldRecallId) && decision.recalledItemIds.includes(task.shouldRecallId) ? 1 : 0;
31807
+ }
31808
+ }
31809
+ const stalePresent = visible.some((v) => v.superseded);
31810
+ const wrongScopePresent = visible.some((v) => v.wrongScope);
31811
+ const staleMemoryHarm = stalePresent ? 1 : 0;
31812
+ const wrongScopeRetrieval = wrongScopePresent ? 1 : 0;
31813
+ const hasSupersededInTrace = task.memoryItems.some((m) => m.status === "superseded");
31814
+ const supersessionRespected = hasSupersededInTrace ? stalePresent ? 0 : 1 : 1;
31815
+ const citationCoverage = visible.length > 0 ? visible.filter((v) => v.citation.length > 0).length / visible.length : 1;
31816
+ const memoryTokensInjected = pack.totalTokens;
31817
+ const retrievedItemCount = visible.length;
31818
+ const compressionRatio = pack.fullTranscriptTokens > 0 ? pack.fullTranscriptTokens / Math.max(pack.totalTokens, 1) : 1;
31819
+ return {
31820
+ task_success: taskSuccess,
31821
+ should_ask_accuracy: shouldAskAccuracy,
31822
+ unnecessary_clarification_rate: unnecessaryClarification,
31823
+ action_boundary_violation_rate: actionBoundaryViolation,
31824
+ relevant_memory_recall: relevantMemoryRecall,
31825
+ stale_memory_harm_rate: staleMemoryHarm,
31826
+ wrong_scope_retrieval_rate: wrongScopeRetrieval,
31827
+ supersession_respected_rate: supersessionRespected,
31828
+ citation_coverage: citationCoverage,
31829
+ memory_tokens_injected: memoryTokensInjected,
31830
+ retrieved_item_count: retrievedItemCount,
31831
+ compression_ratio_vs_raw_transcript: compressionRatio
31832
+ };
31833
+ }
31834
+ function mean2(values) {
31835
+ if (values.length === 0) return 0;
31836
+ return values.reduce((s, v) => s + v, 0) / values.length;
31837
+ }
31838
+ function aggregateCondition(condition, scored, skillLog) {
31839
+ const taskCount = scored.length;
31840
+ const all = scored.map((s) => s.scores);
31841
+ const taskSuccessRate = mean2(all.map((s) => s.task_success));
31842
+ const boundaryTasks = scored.filter((s) => s.task.shouldAsk !== void 0);
31843
+ const askNeeded = scored.filter((s) => s.task.shouldAsk === true);
31844
+ const actWhenEnough = scored.filter((s) => s.task.shouldAsk === false);
31845
+ const shouldAskAccuracy = mean2(boundaryTasks.map((s) => s.scores.should_ask_accuracy));
31846
+ const unnecessaryClarificationRate = mean2(
31847
+ actWhenEnough.map((s) => s.scores.unnecessary_clarification_rate)
31848
+ );
31849
+ const actionBoundaryViolationRate = mean2(
31850
+ askNeeded.map((s) => s.scores.action_boundary_violation_rate)
31851
+ );
31852
+ const recallTasks = scored.filter(
31853
+ (s) => (s.task.shouldRecallId !== void 0 || s.task.family === "skill-positive") && s.task.family !== "stale-memory-trap" && s.task.family !== "wrong-scope-trap"
31854
+ );
31855
+ const relevantMemoryRecall = mean2(recallTasks.map((s) => s.scores.relevant_memory_recall));
31856
+ const staleTasks = scored.filter((s) => s.task.family === "stale-memory-trap");
31857
+ const scopeTasks = scored.filter((s) => s.task.family === "wrong-scope-trap");
31858
+ const staleMemoryHarmRate = mean2(staleTasks.map((s) => s.scores.stale_memory_harm_rate));
31859
+ const wrongScopeRetrievalRate = mean2(scopeTasks.map((s) => s.scores.wrong_scope_retrieval_rate));
31860
+ const supersessionRespectedRate = mean2(staleTasks.map((s) => s.scores.supersession_respected_rate));
31861
+ const citedTasks = scored.filter((s) => s.scores.retrieved_item_count > 0);
31862
+ const citationCoverage = mean2(citedTasks.map((s) => s.scores.citation_coverage));
31863
+ const meanMemoryTokensInjected = mean2(all.map((s) => s.memory_tokens_injected));
31864
+ const meanRetrievedItemCount = mean2(all.map((s) => s.retrieved_item_count));
31865
+ const compressible = scored.filter((s) => s.task.memoryItems.length > 0);
31866
+ const meanCompressionRatio = mean2(compressible.map((s) => s.scores.compression_ratio_vs_raw_transcript));
31867
+ const considered = skillLog.filter((e) => e.considered);
31868
+ const injected = considered.filter((e) => e.injected);
31869
+ const tp = injected.filter((e) => e.outcome === "helped").length;
31870
+ const fp = injected.filter((e) => e.outcome === "harmed").length;
31871
+ const notInjected = considered.filter((e) => !e.injected);
31872
+ const fn = notInjected.filter((e) => e.outcome === "harmed").length;
31873
+ const tn = notInjected.filter((e) => e.outcome === "irrelevant").length;
31874
+ return {
31875
+ condition,
31876
+ taskCount,
31877
+ taskSuccessRate,
31878
+ shouldAskAccuracy,
31879
+ unnecessaryClarificationRate,
31880
+ actionBoundaryViolationRate,
31881
+ relevantMemoryRecall,
31882
+ staleMemoryHarmRate,
31883
+ wrongScopeRetrievalRate,
31884
+ supersessionRespectedRate,
31885
+ citationCoverage,
31886
+ meanMemoryTokensInjected,
31887
+ meanRetrievedItemCount,
31888
+ meanCompressionRatio,
31889
+ skillTriggerPrecision: tp + fp > 0 ? tp / (tp + fp) : 0,
31890
+ skillTriggerRecall: tp + fn > 0 ? tp / (tp + fn) : 0,
31891
+ skillFalsePositiveRate: fp + tn > 0 ? fp / (fp + tn) : 0,
31892
+ skillFalseNegativeRate: tp + fn > 0 ? fn / (tp + fn) : 0,
31893
+ skillHelpedCount: tp,
31894
+ skillHarmedCount: fp,
31895
+ skillIrrelevantCount: considered.filter((e) => e.outcome === "irrelevant").length
31896
+ };
31897
+ }
31898
+
31899
+ // src/benchmarks/remnic/bounded-memory-contracts/types.ts
31900
+ var BOUNDED_MEMORY_CONDITIONS = [
31901
+ "no-memory",
31902
+ "raw-transcript",
31903
+ "typed-contract",
31904
+ "typed-plus-skills"
31905
+ ];
31906
+ var BOUNDED_MEMORY_CONDITION_LABELS = {
31907
+ "no-memory": "C0 no-memory",
31908
+ "raw-transcript": "C1 raw-transcript",
31909
+ "typed-contract": "C2 typed-contract",
31910
+ "typed-plus-skills": "C3 typed-plus-skills"
31911
+ };
31912
+
31913
+ // src/benchmarks/remnic/bounded-memory-contracts/report.ts
31914
+ var CONDITION_ORDER = [
31915
+ "no-memory",
31916
+ "raw-transcript",
31917
+ "typed-contract",
31918
+ "typed-plus-skills"
31919
+ ];
31920
+ function pct(value) {
31921
+ return `${(value * 100).toFixed(1)}%`;
31922
+ }
31923
+ function num(value) {
31924
+ return value.toFixed(2);
31925
+ }
31926
+ function renderReportMarkdown(tasks, aggregates) {
31927
+ const lines = [];
31928
+ lines.push("# Bounded Memory Contracts \u2014 Benchmark Report");
31929
+ lines.push("");
31930
+ lines.push("> Generated by `@remnic/bench` (issue #1708). Fully synthetic, offline, deterministic. See the *Safe vs unsupported claims* section before citing any number.");
31931
+ lines.push("");
31932
+ lines.push("## Configuration");
31933
+ lines.push("");
31934
+ lines.push("- **Benchmark**: `bounded-memory-contracts` v1.0.0");
31935
+ lines.push("- **Model / provider**: none (deterministic offline simulation; no LLM called)");
31936
+ lines.push("- **Dataset source**: synthetic smoke fixture (no real user/client data)");
31937
+ lines.push(`- **Task count**: ${tasks.length} across 7 families`);
31938
+ lines.push("- **Conditions**: C0 no-memory \xB7 C1 raw-transcript \xB7 C2 typed-contract \xB7 C3 typed-plus-skills");
31939
+ lines.push("- **Budget**: shared token budget (budget-normalized comparison per the issue's mitigation for an unfair baseline)");
31940
+ lines.push("");
31941
+ lines.push("## Condition summary");
31942
+ lines.push("");
31943
+ lines.push("| Condition | Task success | Recall | Stale harm | Wrong-scope | Supersession respected | Citations | Mean tokens | Compression |");
31944
+ lines.push("|---|---|---|---|---|---|---|---|---|");
31945
+ for (const cond of CONDITION_ORDER) {
31946
+ const a = aggregates[cond];
31947
+ if (!a) continue;
31948
+ lines.push(
31949
+ `| ${BOUNDED_MEMORY_CONDITION_LABELS[cond]} | ${pct(a.taskSuccessRate)} | ${pct(a.relevantMemoryRecall)} | ${pct(a.staleMemoryHarmRate)} | ${pct(a.wrongScopeRetrievalRate)} | ${pct(a.supersessionRespectedRate)} | ${pct(a.citationCoverage)} | ${num(a.meanMemoryTokensInjected)} | ${num(a.meanCompressionRatio)}\xD7 |`
31950
+ );
31951
+ }
31952
+ lines.push("");
31953
+ lines.push("## Boundary behavior (ask vs act)");
31954
+ lines.push("");
31955
+ lines.push("| Condition | Should-ask accuracy | Unnecessary clarification | Boundary violation |");
31956
+ lines.push("|---|---|---|---|");
31957
+ for (const cond of CONDITION_ORDER) {
31958
+ const a = aggregates[cond];
31959
+ if (!a) continue;
31960
+ lines.push(
31961
+ `| ${BOUNDED_MEMORY_CONDITION_LABELS[cond]} | ${pct(a.shouldAskAccuracy)} | ${pct(a.unnecessaryClarificationRate)} | ${pct(a.actionBoundaryViolationRate)} |`
31962
+ );
31963
+ }
31964
+ lines.push("");
31965
+ const c3 = aggregates["typed-plus-skills"];
31966
+ lines.push("## Skill-trigger behavior (C3 only)");
31967
+ lines.push("");
31968
+ if (c3) {
31969
+ lines.push(`- **Trigger precision**: ${pct(c3.skillTriggerPrecision)}`);
31970
+ lines.push(`- **Trigger recall**: ${pct(c3.skillTriggerRecall)}`);
31971
+ lines.push(`- **False-positive rate**: ${pct(c3.skillFalsePositiveRate)}`);
31972
+ lines.push(`- **False-negative rate**: ${pct(c3.skillFalseNegativeRate)}`);
31973
+ lines.push(`- **Helped / harmed / irrelevant**: ${c3.skillHelpedCount} / ${c3.skillHarmedCount} / ${c3.skillIrrelevantCount}`);
31974
+ } else {
31975
+ lines.push("_C3 not run._");
31976
+ }
31977
+ lines.push("");
31978
+ lines.push("## Biggest wins (typed retrieval vs raw transcript)");
31979
+ lines.push("");
31980
+ const c1 = aggregates["raw-transcript"];
31981
+ const c2 = aggregates["typed-contract"];
31982
+ if (c1 && c2) {
31983
+ lines.push(`- **Stale-memory harm**: C1 ${pct(c1.staleMemoryHarmRate)} \u2192 C2 ${pct(c2.staleMemoryHarmRate)}. The typed contract excludes superseded facts; raw transcript cannot.`);
31984
+ lines.push(`- **Wrong-scope leakage**: C1 ${pct(c1.wrongScopeRetrievalRate)} \u2192 C2 ${pct(c2.wrongScopeRetrievalRate)}. Scope filtering removes cross-project decoys that raw transcript surfaces.`);
31985
+ lines.push(`- **Boundary violations**: C1 ${pct(c1.actionBoundaryViolationRate)} \u2192 C2 ${pct(c2.actionBoundaryViolationRate)}. Structured boundary notes are reliably surfaced; buried transcript prose is not.`);
31986
+ lines.push(`- **Citation coverage**: C1 ${pct(c1.citationCoverage)} \u2192 C2 ${pct(c2.citationCoverage)}. Typed slots carry citations; raw text does not.`);
31987
+ lines.push(`- **Compression**: C1 ${num(c1.meanCompressionRatio)}\xD7 \u2192 C2 ${num(c2.meanCompressionRatio)}\xD7 of the raw transcript. Typed packs are smaller for the same budget.`);
31988
+ }
31989
+ lines.push("");
31990
+ lines.push("## Biggest failures / honest weaknesses");
31991
+ lines.push("");
31992
+ lines.push("- **Recall parity on pure-recall tasks**: raw transcript (C1) is a *fair* baseline \u2014 when the needed fact is present and unambiguous, C1 recalls it as well as C2. Typed retrieval's advantage is governance, not brute recall.");
31993
+ lines.push("- **C0 no-memory**: fails every recall-dependent task by construction; it is the floor, not a competitor.");
31994
+ lines.push("- **Skill overfitting risk**: the smoke fixture is small and hand-authored around two skills; positive/negative trigger balance is preliminary.");
31995
+ lines.push("");
31996
+ lines.push("## Worked examples");
31997
+ lines.push("");
31998
+ const staleExample = tasks.find((t) => t.family === "stale-memory-trap");
31999
+ if (staleExample) {
32000
+ lines.push(`- **Stale-memory trap** (\`${staleExample.id}\`): a superseded fact (more repeated in the transcript) crowds out the one-line correction under C1, while C2 excludes it via \`excludeIfSuperseded\`.`);
32001
+ }
32002
+ const scopeExample = tasks.find((t) => t.family === "wrong-scope-trap");
32003
+ if (scopeExample) {
32004
+ lines.push(`- **Wrong-scope trap** (\`${scopeExample.id}\`): a same-subject fact from a different project leaks under C1 (no scope metadata) but is filtered out under C2/C3.`);
32005
+ }
32006
+ lines.push("");
32007
+ lines.push("## Token / cost / latency");
32008
+ lines.push("");
32009
+ lines.push("- **Estimated cost**: $0 \u2014 no model was called. Token counts reflect injected memory only.");
32010
+ lines.push("| Condition | Mean injected tokens | Mean retrieved items |");
32011
+ lines.push("|---|---|---|");
32012
+ for (const cond of CONDITION_ORDER) {
32013
+ const a = aggregates[cond];
32014
+ if (!a) continue;
32015
+ lines.push(`| ${BOUNDED_MEMORY_CONDITION_LABELS[cond]} | ${num(a.meanMemoryTokensInjected)} | ${num(a.meanRetrievedItemCount)} |`);
32016
+ }
32017
+ lines.push("");
32018
+ lines.push("## Safe vs unsupported claims");
32019
+ lines.push("");
32020
+ lines.push("**Safe to claim from this run:**");
32021
+ lines.push("");
32022
+ lines.push("- Typed retrieval contracts exclude superseded and wrong-scope memories that raw transcript stuffing surfaces, *under this deterministic simulation*.");
32023
+ lines.push("- Structured boundary notes are reliably surfaced by the typed contract and not by raw transcript, *under this simulation*.");
32024
+ lines.push("- Typed packs are smaller than the raw transcript for the same task, under the shared budget.");
32025
+ lines.push("");
32026
+ lines.push("**NOT supported by this run (do not claim):**");
32027
+ lines.push("");
32028
+ lines.push("- That Remnic beats Mem0 / Zep / Letta or any other system. No cross-system comparison was performed.");
32029
+ lines.push("- That these numbers generalize to real users. The fixture is small, synthetic, and hand-authored.");
32030
+ lines.push("- Any frontier-model quality figures. No LLM was called; the agent is a deterministic decision procedure.");
32031
+ lines.push("- That skill-triggered memory helps on real procedural workloads. Two hand-authored skills are not evidence.");
32032
+ lines.push("");
32033
+ lines.push("## Recommended next experiment");
32034
+ lines.push("");
32035
+ lines.push("1. Scale to the full fixture (100+ tasks) with de-identified or synthetic traces.");
32036
+ lines.push("2. Swap the deterministic agent for a real responder under a blinded judge (condition names hidden).");
32037
+ lines.push("3. Add a `model-context-max` raw-transcript variant as a *separate, non-primary* condition.");
32038
+ lines.push("4. Expand the skill library and report false-positive/negative rates over a larger trigger set.");
32039
+ lines.push("");
32040
+ return lines.join("\n");
32041
+ }
32042
+
32043
+ // src/benchmarks/remnic/bounded-memory-contracts/runner.ts
32044
+ var boundedMemoryContractsDefinition = {
32045
+ id: "bounded-memory-contracts",
32046
+ title: "Bounded Memory Contracts",
32047
+ tier: "remnic",
32048
+ status: "ready",
32049
+ runnerAvailable: true,
32050
+ meta: {
32051
+ name: "bounded-memory-contracts",
32052
+ version: "1.0.0",
32053
+ description: "Ablates raw transcript stuffing vs typed retrieval contracts vs skill-triggered memory under a shared token budget (issue #1708).",
32054
+ category: "agentic",
32055
+ citation: "Remnic internal synthetic benchmark for issue #1708"
32056
+ }
32057
+ };
32058
+ function resolveInjectedSkills(task) {
32059
+ const injected = [];
32060
+ for (const skill of task.skills) {
32061
+ const verdict = classifySkillTrigger(skill, task);
32062
+ if (verdict.injected) {
32063
+ injected.push(skill);
32064
+ }
32065
+ }
32066
+ return injected;
32067
+ }
32068
+ function skillOutcome(task, skill, injected) {
32069
+ if (task.family === "skill-positive") {
32070
+ const expected = task.shouldUseSkillId === skill.id;
32071
+ if (injected && expected) return "helped";
32072
+ if (injected !== expected) return "harmed";
32073
+ return "irrelevant";
32074
+ }
32075
+ if (task.family === "skill-negative") {
32076
+ return injected ? "harmed" : "irrelevant";
32077
+ }
32078
+ return "irrelevant";
32079
+ }
32080
+ async function runBoundedMemoryContractsBenchmark(options) {
32081
+ const fixtureSource = options.mode === "quick" ? BOUNDED_MEMORY_SMOKE_FIXTURE : BOUNDED_MEMORY_FIXTURE;
32082
+ const tasks = options.mode === "full" && typeof options.limit === "number" && options.limit > 0 && Number.isFinite(options.limit) ? fixtureSource.slice(0, Math.floor(options.limit)) : fixtureSource;
32083
+ const seed = typeof options.seed === "number" ? options.seed : 0;
32084
+ const totalPairs = tasks.length * BOUNDED_MEMORY_CONDITIONS.length;
32085
+ const byCondition = /* @__PURE__ */ new Map();
32086
+ for (const condition of BOUNDED_MEMORY_CONDITIONS) {
32087
+ const results = [];
32088
+ for (const task of tasks) {
32089
+ const resolvedSkills = condition === "typed-plus-skills" ? resolveInjectedSkills(task) : [];
32090
+ const started = performance.now();
32091
+ const pack = assemblePack(task, condition, BOUNDED_MEMORY_CONTRACT, resolvedSkills);
32092
+ const packedSkillIds = new Set(
32093
+ pack.slots.find((slot) => slot.id === "triggered_skills")?.items.map((it) => it.itemId) ?? []
32094
+ );
32095
+ const effectiveSkills = resolvedSkills.filter((sk) => packedSkillIds.has(sk.id));
32096
+ const decision = simulateAgent(task, pack, effectiveSkills);
32097
+ const latencyMs = Math.round(performance.now() - started);
32098
+ results.push({ task, pack, decision, latencyMs });
32099
+ }
32100
+ byCondition.set(condition, results);
32101
+ }
32102
+ const taskResults = [];
32103
+ const conditionAggregates = {};
32104
+ const scoredByCondition = {};
32105
+ let c3SkillLog = [];
32106
+ for (const condition of BOUNDED_MEMORY_CONDITIONS) {
32107
+ const results = byCondition.get(condition);
32108
+ const scoredPairs = results.map(({ task, pack, decision }) => {
32109
+ const scores = scoreTaskPair(task, pack, decision);
32110
+ return { task, scores, pack, decision };
32111
+ });
32112
+ scoredByCondition[condition] = scoredPairs.map(({ task, scores }) => ({ task, scores }));
32113
+ let skillLog = [];
32114
+ if (condition === "typed-plus-skills") {
32115
+ skillLog = scoredPairs.flatMap(({ task, pack }) => {
32116
+ const packedSkillIds = new Set(
32117
+ pack.slots.find((slot) => slot.id === "triggered_skills")?.items.map((it) => it.itemId) ?? []
32118
+ );
32119
+ return task.skills.map((skill) => {
32120
+ const verdict = classifySkillTrigger(skill, task);
32121
+ const injected = packedSkillIds.has(skill.id);
32122
+ return {
32123
+ taskId: task.id,
32124
+ skillId: skill.id,
32125
+ considered: verdict.considered,
32126
+ injected,
32127
+ triggerReason: verdict.reason,
32128
+ confidence: skill.confidence,
32129
+ outcome: skillOutcome(task, skill, injected)
32130
+ };
32131
+ });
32132
+ });
32133
+ c3SkillLog = skillLog;
32134
+ }
32135
+ conditionAggregates[condition] = aggregateCondition(condition, scoredByCondition[condition], skillLog);
32136
+ for (const { task, pack, decision, scores } of scoredPairs) {
32137
+ const tr = {
32138
+ taskId: `${condition}:${task.id}`,
32139
+ question: task.prompt,
32140
+ expected: task.expectedAnswer,
32141
+ actual: decision.answer,
32142
+ scores: { ...scores },
32143
+ latencyMs: results.find((r) => r.task.id === task.id).latencyMs,
32144
+ tokens: { input: pack.totalTokens, output: 0 },
32145
+ details: {
32146
+ condition,
32147
+ family: task.family,
32148
+ scope: task.scope,
32149
+ packTokens: pack.totalTokens,
32150
+ compressionRatio: scores.compression_ratio_vs_raw_transcript,
32151
+ recalledItemIds: decision.recalledItemIds,
32152
+ askedClarification: decision.askedClarification,
32153
+ acted: decision.acted
32154
+ }
32155
+ };
32156
+ taskResults.push(tr);
32157
+ options.onTaskComplete?.(tr, taskResults.length, totalPairs);
32158
+ }
32159
+ }
32160
+ const aggregates = aggregateTaskScores(taskResults.map((t) => t.scores));
32161
+ const remnicVersion = await getRemnicVersion();
32162
+ const totalLatencyMs = taskResults.reduce((sum, t) => sum + t.latencyMs, 0);
32163
+ const totalMemoryTokens = taskResults.reduce((sum, t) => sum + t.tokens.input, 0);
32164
+ let artifactReport = null;
32165
+ if (options.outputDir) {
32166
+ artifactReport = await writeArtifacts(options.outputDir, byCondition, conditionAggregates, tasks);
32167
+ }
32168
+ const skillTriggerLog = c3SkillLog;
32169
+ return {
32170
+ meta: {
32171
+ id: randomUUID32(),
32172
+ benchmark: options.benchmark.id,
32173
+ benchmarkTier: options.benchmark.tier,
32174
+ version: options.benchmark.meta.version,
32175
+ remnicVersion,
32176
+ gitSha: getGitSha(),
32177
+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),
32178
+ mode: options.mode,
32179
+ runCount: 1,
32180
+ seeds: [seed],
32181
+ datasetHash: fixtureHash(tasks)
32182
+ },
32183
+ config: {
32184
+ runtimeProfile: options.runtimeProfile ?? null,
32185
+ systemProvider: options.systemProvider ?? null,
32186
+ judgeProvider: options.judgeProvider ?? null,
32187
+ adapterMode: options.adapterMode ?? "deterministic-offline",
32188
+ remnicConfig: options.remnicConfig ?? {},
32189
+ benchmarkOptions: {
32190
+ conditions: conditionAggregates,
32191
+ skillTriggerLog,
32192
+ contract: BOUNDED_MEMORY_CONTRACT,
32193
+ fixtureTaskCount: tasks.length,
32194
+ conditionCount: BOUNDED_MEMORY_CONDITIONS.length,
32195
+ artifactReport
32196
+ }
32197
+ },
32198
+ cost: {
32199
+ totalTokens: totalMemoryTokens,
32200
+ inputTokens: totalMemoryTokens,
32201
+ outputTokens: 0,
32202
+ estimatedCostUsd: 0,
32203
+ totalLatencyMs,
32204
+ meanQueryLatencyMs: taskResults.length > 0 ? totalLatencyMs / taskResults.length : 0,
32205
+ judgeModelCalls: 0
32206
+ },
32207
+ results: {
32208
+ tasks: taskResults,
32209
+ aggregates,
32210
+ statistics: {
32211
+ confidenceIntervals: {},
32212
+ bootstrapSamples: 0
32213
+ }
32214
+ },
32215
+ environment: {
32216
+ os: process.platform,
32217
+ nodeVersion: process.version,
32218
+ hardware: process.arch
32219
+ }
32220
+ };
32221
+ }
32222
+ async function writeArtifacts(outputDir, byCondition, conditionAggregates, tasks) {
32223
+ const root = path32.resolve(outputDir);
32224
+ await mkdir16(path32.join(root, "conditions"), { recursive: true });
32225
+ await mkdir16(path32.join(root, "prompts"), { recursive: true });
32226
+ await mkdir16(path32.join(root, "retrieval"), { recursive: true });
32227
+ await mkdir16(path32.join(root, "scores"), { recursive: true });
32228
+ const csvRows = [
32229
+ "task_id,condition,family,scope,task_success,should_ask_accuracy,relevant_memory_recall,stale_memory_harm_rate,wrong_scope_retrieval_rate,supersession_respected_rate,citation_coverage,memory_tokens_injected,retrieved_item_count,compression_ratio_vs_raw_transcript"
32230
+ ];
32231
+ for (const condition of BOUNDED_MEMORY_CONDITIONS) {
32232
+ const results = byCondition.get(condition);
32233
+ const condDir = path32.join(root, "conditions", condition);
32234
+ await mkdir16(condDir, { recursive: true });
32235
+ for (const { task, pack, decision } of results) {
32236
+ const scores = scoreTaskPair(task, pack, decision);
32237
+ const promptMd = renderPromptPack(task, condition, pack);
32238
+ const promptPath = path32.join(root, "prompts", `${task.id}.${condition}.md`);
32239
+ await mkdir16(path32.dirname(promptPath), { recursive: true });
32240
+ await writeFile15(promptPath, promptMd, "utf8");
32241
+ const retrievalJson = `${JSON.stringify(
32242
+ {
32243
+ taskId: task.id,
32244
+ condition,
32245
+ contract: BOUNDED_MEMORY_CONTRACT.id,
32246
+ pack: {
32247
+ slots: pack.slots.map((slot) => ({
32248
+ id: slot.id,
32249
+ items: slot.items
32250
+ })),
32251
+ transcriptBlock: pack.transcriptBlock,
32252
+ boundaryItem: pack.boundaryItem,
32253
+ totalTokens: pack.totalTokens,
32254
+ fullTranscriptTokens: pack.fullTranscriptTokens
32255
+ },
32256
+ decision,
32257
+ scores
32258
+ },
32259
+ null,
32260
+ 2
32261
+ )}
32262
+ `;
32263
+ const retrievalPath = path32.join(root, "retrieval", `${task.id}.${condition}.json`);
32264
+ await writeFile15(retrievalPath, retrievalJson, "utf8");
32265
+ csvRows.push(
32266
+ [
32267
+ task.id,
32268
+ condition,
32269
+ task.family,
32270
+ task.scope,
32271
+ scores.task_success,
32272
+ scores.should_ask_accuracy,
32273
+ scores.relevant_memory_recall,
32274
+ scores.stale_memory_harm_rate,
32275
+ scores.wrong_scope_retrieval_rate,
32276
+ scores.supersession_respected_rate,
32277
+ scores.citation_coverage.toFixed(4),
32278
+ scores.memory_tokens_injected,
32279
+ scores.retrieved_item_count,
32280
+ scores.compression_ratio_vs_raw_transcript.toFixed(4)
32281
+ ].join(",")
32282
+ );
32283
+ }
32284
+ await writeFile15(
32285
+ path32.join(condDir, "summary.json"),
32286
+ `${JSON.stringify(conditionAggregates[condition], null, 2)}
32287
+ `,
32288
+ "utf8"
32289
+ );
32290
+ }
32291
+ await writeFile15(path32.join(root, "scores", "per-task.csv"), `${csvRows.join("\n")}
32292
+ `, "utf8");
32293
+ await writeFile15(
32294
+ path32.join(root, "scores", "aggregate.json"),
32295
+ `${JSON.stringify(conditionAggregates, null, 2)}
32296
+ `,
32297
+ "utf8"
32298
+ );
32299
+ const report = renderReportMarkdown(tasks, conditionAggregates);
32300
+ await writeFile15(path32.join(root, "report.md"), report, "utf8");
32301
+ return path32.join(root, "report.md");
32302
+ }
32303
+ function renderPromptPack(task, condition, pack) {
32304
+ const lines = [];
32305
+ lines.push(`# Prompt pack \u2014 ${task.id} [${BOUNDED_MEMORY_CONDITION_LABELS[condition]}]`);
32306
+ lines.push("");
32307
+ lines.push("## System");
32308
+ lines.push(
32309
+ condition === "no-memory" ? "You have no prior memory. Answer from the current task only." : condition === "raw-transcript" ? "Prior session transcript follows. Use it as context." : "A bounded memory contract follows. Each item is typed, scoped, and cited."
32310
+ );
32311
+ lines.push("");
32312
+ lines.push("## Current task");
32313
+ lines.push(task.prompt);
32314
+ lines.push("");
32315
+ lines.push(`## Active scope`);
32316
+ lines.push(task.scope);
32317
+ lines.push("");
32318
+ if (condition === "no-memory") {
32319
+ lines.push("_(no memory injected)_");
32320
+ lines.push("");
32321
+ } else if (condition === "raw-transcript") {
32322
+ lines.push("## Raw transcript (budget-normalized)");
32323
+ lines.push("```");
32324
+ lines.push(pack.transcriptBlock ?? "_(empty)_");
32325
+ lines.push("```");
32326
+ lines.push("");
32327
+ } else {
32328
+ if (pack.boundaryItem) {
32329
+ lines.push("## Boundaries");
32330
+ lines.push(`- [${pack.boundaryItem.citation}] (${pack.boundaryItem.scope}, ${pack.boundaryItem.status}) ${pack.boundaryItem.content}`);
32331
+ lines.push("");
32332
+ }
32333
+ for (const slot of pack.slots) {
32334
+ if (slot.items.length === 0) continue;
32335
+ lines.push(`## ${slot.id}`);
32336
+ for (const it of slot.items) {
32337
+ lines.push(`- [${it.citation}] (${it.scope}, ${it.status}) ${it.content}`);
32338
+ }
32339
+ lines.push("");
32340
+ }
32341
+ }
32342
+ return `${lines.join("\n")}
32343
+ `;
32344
+ }
32345
+
30978
32346
  // src/registry.ts
30979
32347
  var REGISTERED_BENCHMARKS = [
30980
32348
  {
@@ -31112,6 +32480,10 @@ var REGISTERED_BENCHMARKS = [
31112
32480
  {
31113
32481
  ...memcorrectDefinition,
31114
32482
  run: runMemCorrectBenchmark
32483
+ },
32484
+ {
32485
+ ...boundedMemoryContractsDefinition,
32486
+ run: runBoundedMemoryContractsBenchmark
31115
32487
  }
31116
32488
  ];
31117
32489
  function listBenchmarks() {
@@ -31151,8 +32523,8 @@ function finalizeBenchmarkResultConfig(result, options) {
31151
32523
  }
31152
32524
 
31153
32525
  // src/benchmark.ts
31154
- var DEFAULT_BASELINE_PATH = path32.join(process.cwd(), "benchmarks", "baseline.json");
31155
- var DEFAULT_REPORT_PATH = path32.join(process.cwd(), "benchmarks", "report.json");
32526
+ var DEFAULT_BASELINE_PATH = path33.join(process.cwd(), "benchmarks", "baseline.json");
32527
+ var DEFAULT_REPORT_PATH = path33.join(process.cwd(), "benchmarks", "report.json");
31156
32528
  var BASELINE_VERSION = 1;
31157
32529
  var DEFAULT_TOLERANCE = 10;
31158
32530
  var DEFAULT_FULL_RUN_COUNT = 5;
@@ -31230,7 +32602,7 @@ async function runBenchmark(benchmarkId, options) {
31230
32602
  if (!willWrapPrimary && !willWrapCross) {
31231
32603
  return void 0;
31232
32604
  }
31233
- const cacheDir = options.judgeCacheDir ? path32.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path32.join(path32.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
32605
+ const cacheDir = options.judgeCacheDir ? path33.resolve(expandTildePath3(options.judgeCacheDir)) : options.outputDir ? path33.join(path33.resolve(expandTildePath3(options.outputDir)), "judge-cache") : void 0;
31234
32606
  if (cacheDir === void 0) {
31235
32607
  return void 0;
31236
32608
  }
@@ -31352,13 +32724,13 @@ function wrapJudgeWithCache(args) {
31352
32724
  // differentiator is part of the prompt hash. Bumping
31353
32725
  // JUDGE_CACHE_PROTOCOL_VERSION invalidates verdicts when judge
31354
32726
  // prompt/parse semantics change (PR #1591, High).
31355
- judgePromptHash: createHash10("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("").update(args.amaBenchJudgeProtocol).update("").update(args.role).digest("hex"),
32727
+ judgePromptHash: createHash11("sha256").update(JUDGE_CACHE_PROTOCOL_VERSION).update("").update(args.amaBenchJudgeProtocol).update("").update(args.role).digest("hex"),
31356
32728
  judgeModelId: args.provider?.model !== void 0 && args.provider.model.length > 0 ? `${args.provider.model}${crossJudgeIdSuffix}` : `unknown-${args.role}-judge`,
31357
32729
  // Full judge configuration, deterministically serialized (sorted
31358
32730
  // keys) so provider/base-url/retry changes produce fresh cache
31359
32731
  // keys. `role` is included so primary and cross judges never
31360
32732
  // share a paramsHash.
31361
- judgeParamsHash: createHash10("sha256").update(
32733
+ judgeParamsHash: createHash11("sha256").update(
31362
32734
  stableStringify2({
31363
32735
  role: args.role,
31364
32736
  provider: args.provider
@@ -31419,7 +32791,7 @@ function loadBaseline(baselinePath) {
31419
32791
  return raw;
31420
32792
  }
31421
32793
  function saveBaseline(baselinePath, baseline) {
31422
- fs2.mkdirSync(path32.dirname(baselinePath), { recursive: true });
32794
+ fs2.mkdirSync(path33.dirname(baselinePath), { recursive: true });
31423
32795
  fs2.writeFileSync(baselinePath, `${JSON.stringify(baseline, null, 2)}
31424
32796
  `);
31425
32797
  }
@@ -31649,7 +33021,7 @@ function generateReport(results, reportPath) {
31649
33021
  totalDurationMs: results.reduce((sum, result) => sum + result.totalDurationMs, 0)
31650
33022
  };
31651
33023
  if (reportPath) {
31652
- fs2.mkdirSync(path32.dirname(reportPath), { recursive: true });
33024
+ fs2.mkdirSync(path33.dirname(reportPath), { recursive: true });
31653
33025
  fs2.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}
31654
33026
  `);
31655
33027
  }
@@ -31657,7 +33029,7 @@ function generateReport(results, reportPath) {
31657
33029
  }
31658
33030
 
31659
33031
  // src/stats/effect-size.ts
31660
- function mean2(values) {
33032
+ function mean3(values) {
31661
33033
  if (values.length === 0) {
31662
33034
  throw new Error("effect size requires at least one value");
31663
33035
  }
@@ -31673,8 +33045,8 @@ function cohensD(candidateValues, baselineValues) {
31673
33045
  if (candidateValues.length === 0 || baselineValues.length === 0) {
31674
33046
  throw new Error("effect size requires non-empty candidate and baseline arrays");
31675
33047
  }
31676
- const candidateMean = mean2(candidateValues);
31677
- const baselineMean = mean2(baselineValues);
33048
+ const candidateMean = mean3(candidateValues);
33049
+ const baselineMean = mean3(baselineValues);
31678
33050
  const candidateVariance = sampleVariance(candidateValues, candidateMean);
31679
33051
  const baselineVariance = sampleVariance(baselineValues, baselineMean);
31680
33052
  const pooledDegreesOfFreedom = candidateValues.length + baselineValues.length - 2;
@@ -32114,8 +33486,8 @@ function formatError(error) {
32114
33486
  }
32115
33487
 
32116
33488
  // src/benchmarks/custom/runner.ts
32117
- import { randomUUID as randomUUID32 } from "crypto";
32118
- import path33 from "path";
33489
+ import { randomUUID as randomUUID33 } from "crypto";
33490
+ import path34 from "path";
32119
33491
  import { expandTildePath as expandTildePath4 } from "@remnic/core";
32120
33492
  async function runCustomBenchmarkFile(filePath, options) {
32121
33493
  const spec = await loadCustomBenchmarkFile(filePath);
@@ -32128,7 +33500,7 @@ async function runCustomBenchmarkFile(filePath, options) {
32128
33500
  let cacheRestore;
32129
33501
  let cacheCounters;
32130
33502
  if (spec.scoring === "llm_judge" && runOptions.system.judge !== void 0 && !runOptions.noJudgeCache && (runOptions.judgeProvider ?? null) !== null) {
32131
- const cacheDir = runOptions.judgeCacheDir ? path33.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path33.join(path33.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
33503
+ const cacheDir = runOptions.judgeCacheDir ? path34.resolve(expandTildePath4(runOptions.judgeCacheDir)) : runOptions.outputDir ? path34.join(path34.resolve(expandTildePath4(runOptions.outputDir)), "judge-cache") : void 0;
32132
33504
  if (cacheDir !== void 0) {
32133
33505
  const originalJudge = runOptions.system.judge;
32134
33506
  const wrapped = wrapJudgeWithCache({
@@ -32202,7 +33574,7 @@ async function runCustomBenchmark(spec, options) {
32202
33574
  const totalOutputTokens = tasks.reduce((sum, task) => sum + task.tokens.output, 0);
32203
33575
  return finalizeBenchmarkResultConfig({
32204
33576
  meta: {
32205
- id: randomUUID32(),
33577
+ id: randomUUID33(),
32206
33578
  benchmark: options.benchmark.id,
32207
33579
  benchmarkTier: options.benchmark.tier,
32208
33580
  version: options.benchmark.meta.version,
@@ -32329,7 +33701,7 @@ async function scoreTask(scoring, options, question, actual, expected) {
32329
33701
  }
32330
33702
  }
32331
33703
  function createCustomBenchmarkDefinition(benchmark, filePath) {
32332
- const id = `custom:${slugify(path33.basename(filePath, path33.extname(filePath)) || benchmark.name)}`;
33704
+ const id = `custom:${slugify(path34.basename(filePath, path34.extname(filePath)) || benchmark.name)}`;
32333
33705
  return {
32334
33706
  id,
32335
33707
  title: benchmark.name,
@@ -33199,9 +34571,9 @@ var chatFixture = {
33199
34571
  };
33200
34572
 
33201
34573
  // src/judges/calibration-slice.ts
33202
- import { createHash as createHash11, randomBytes as randomBytes3 } from "crypto";
33203
- import { mkdir as mkdir16, readFile as readFile21, rename as rename3, unlink as unlink3, writeFile as writeFile15 } from "fs/promises";
33204
- import path34 from "path";
34574
+ import { createHash as createHash12, randomBytes as randomBytes3 } from "crypto";
34575
+ import { mkdir as mkdir17, readFile as readFile21, rename as rename3, unlink as unlink3, writeFile as writeFile16 } from "fs/promises";
34576
+ import path35 from "path";
33205
34577
 
33206
34578
  // src/judges/cohen-kappa.ts
33207
34579
  function computeCohensKappa(raterA, raterB) {
@@ -33276,7 +34648,7 @@ function selectCalibrationSlice(questionIds, size = CALIBRATION_SLICE_SIZE) {
33276
34648
  unique.push(id);
33277
34649
  }
33278
34650
  }
33279
- return unique.map((id) => ({ id, digest: createHash11("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
34651
+ return unique.map((id) => ({ id, digest: createHash12("sha256").update(id, "utf8").digest("hex") })).sort((a, b) => a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0).slice(0, Math.min(size, unique.length)).map((entry) => entry.id);
33280
34652
  }
33281
34653
  async function runJudgeCalibration(options) {
33282
34654
  const binScore = options.binScore ?? ((score) => binarizeJudgeScore(score));
@@ -33330,7 +34702,7 @@ async function runJudgeCalibration(options) {
33330
34702
  };
33331
34703
  }
33332
34704
  async function writeJudgeCalibrationState(result, calibrationDir, identities) {
33333
- await mkdir16(calibrationDir, { recursive: true });
34705
+ await mkdir17(calibrationDir, { recursive: true });
33334
34706
  const state = {
33335
34707
  kappa: result.kappa,
33336
34708
  sampleSize: result.sampleSize,
@@ -33338,9 +34710,9 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
33338
34710
  warning: result.warning,
33339
34711
  ...identities ? identities : {}
33340
34712
  };
33341
- const filePath = path34.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
34713
+ const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(result.benchmarkId)}.json`);
33342
34714
  const tempPath = `${filePath}.${randomBytes3(6).toString("hex")}.tmp`;
33343
- await writeFile15(tempPath, `${JSON.stringify(state, null, 2)}
34715
+ await writeFile16(tempPath, `${JSON.stringify(state, null, 2)}
33344
34716
  `, "utf8");
33345
34717
  try {
33346
34718
  await rename3(tempPath, filePath);
@@ -33351,7 +34723,7 @@ async function writeJudgeCalibrationState(result, calibrationDir, identities) {
33351
34723
  return filePath;
33352
34724
  }
33353
34725
  async function loadJudgeCalibrationState(benchmarkId, calibrationDir) {
33354
- const filePath = path34.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
34726
+ const filePath = path35.join(calibrationDir, `${sanitizeCalibrationSegment(benchmarkId)}.json`);
33355
34727
  let raw;
33356
34728
  try {
33357
34729
  raw = await readFile21(filePath, "utf8");
@@ -33425,9 +34797,9 @@ function sanitizeCalibrationSegment(value) {
33425
34797
  }
33426
34798
 
33427
34799
  // src/benchmarks/remnic/procedural-recall/ablation.ts
33428
- import { mkdir as mkdir17, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile16, readFile as readFile22 } from "fs/promises";
34800
+ import { mkdir as mkdir18, mkdtemp as mkdtemp11, rm as rm13, writeFile as writeFile17, readFile as readFile22 } from "fs/promises";
33429
34801
  import os9 from "os";
33430
- import path35 from "path";
34802
+ import path36 from "path";
33431
34803
  import {
33432
34804
  StorageManager as StorageManager3,
33433
34805
  parseConfig as parseConfig5,
@@ -33458,7 +34830,7 @@ async function runSide(scenarios, proceduralEnabled) {
33458
34830
  const observed = [];
33459
34831
  for (const scenario of scenarios) {
33460
34832
  const dir = await mkdtemp11(
33461
- path35.join(os9.tmpdir(), "remnic-bench-proc-ablation-")
34833
+ path36.join(os9.tmpdir(), "remnic-bench-proc-ablation-")
33462
34834
  );
33463
34835
  try {
33464
34836
  const storage = new StorageManager3(dir);
@@ -33473,7 +34845,7 @@ ${body}`,
33473
34845
  );
33474
34846
  const config = parseConfig5({
33475
34847
  memoryDir: dir,
33476
- workspaceDir: path35.join(dir, "ws"),
34848
+ workspaceDir: path36.join(dir, "ws"),
33477
34849
  openaiApiKey: "bench-key",
33478
34850
  procedural: {
33479
34851
  enabled: proceduralEnabled,
@@ -33644,9 +35016,9 @@ async function runProceduralAblationCli(args) {
33644
35016
  random: args.random,
33645
35017
  seed: args.seed
33646
35018
  });
33647
- const outDir = path35.dirname(path35.resolve(args.outPath));
33648
- await mkdir17(outDir, { recursive: true });
33649
- await writeFile16(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
35019
+ const outDir = path36.dirname(path36.resolve(args.outPath));
35020
+ await mkdir18(outDir, { recursive: true });
35021
+ await writeFile17(args.outPath, JSON.stringify(artifact, null, 2) + "\n", "utf8");
33650
35022
  return artifact;
33651
35023
  }
33652
35024
 
@@ -33942,7 +35314,7 @@ var PROCEDURAL_REAL_SCENARIOS_SMOKE = [
33942
35314
  ];
33943
35315
 
33944
35316
  // src/security/extraction-attack/tokenize.ts
33945
- function tokenize4(text) {
35317
+ function tokenize5(text) {
33946
35318
  return text.toLowerCase().split(/[^a-z0-9]+/u).filter((t) => t.length > 2);
33947
35319
  }
33948
35320
 
@@ -33984,7 +35356,7 @@ function createSeededRng2(seed) {
33984
35356
  }
33985
35357
  };
33986
35358
  }
33987
- var tokenizeContent = tokenize4;
35359
+ var tokenizeContent = tokenize5;
33988
35360
  function recoveryTokensFor(memory) {
33989
35361
  if (memory.tokens && memory.tokens.length > 0) {
33990
35362
  const seen = /* @__PURE__ */ new Set();
@@ -34463,11 +35835,11 @@ function createSyntheticTarget(options) {
34463
35835
  }
34464
35836
  const normalized = memories.map((m) => ({
34465
35837
  memory: m,
34466
- tokens: new Set((m.tokens ?? tokenize4(m.content)).map((t) => t.toLowerCase()))
35838
+ tokens: new Set((m.tokens ?? tokenize5(m.content)).map((t) => t.toLowerCase()))
34467
35839
  }));
34468
35840
  return {
34469
35841
  async recall(query, recallOptions) {
34470
- const qTokens = tokenize4(query);
35842
+ const qTokens = tokenize5(query);
34471
35843
  if (qTokens.length === 0) return [];
34472
35844
  const requestedNs = recallOptions?.namespace;
34473
35845
  if (enforceNamespaceAcl && requestedNs !== void 0 && requestedNs !== allowedNamespace) {
@@ -34711,7 +36083,7 @@ function createMitigatedTarget(config) {
34711
36083
  }
34712
36084
 
34713
36085
  // src/coding-graph/generator.ts
34714
- import { createHash as createHash12 } from "crypto";
36086
+ import { createHash as createHash13 } from "crypto";
34715
36087
  function createSeededRng3(seed) {
34716
36088
  let state = seed >>> 0;
34717
36089
  return function rng() {
@@ -34740,7 +36112,7 @@ var EDGE_TYPE_WEIGHTS = [
34740
36112
  var PROVENANCE_VALUES = ["heuristic", "heuristic", "heuristic", "trace"];
34741
36113
  var AVG_BYTES_PER_LINE = 40;
34742
36114
  function hashContent(input) {
34743
- return createHash12("sha256").update(input).digest("hex").slice(0, 16);
36115
+ return createHash13("sha256").update(input).digest("hex").slice(0, 16);
34744
36116
  }
34745
36117
  function generateSyntheticRepo(config) {
34746
36118
  const rng = createSeededRng3(config.seed);
@@ -34838,7 +36210,7 @@ import { performance as performance2 } from "perf_hooks";
34838
36210
  import { mkdtemp as mkdtemp12, rm as rm14 } from "fs/promises";
34839
36211
  import { statSync } from "fs";
34840
36212
  import { tmpdir as tmpdir7 } from "os";
34841
- import path36 from "path";
36213
+ import path37 from "path";
34842
36214
  import os10 from "os";
34843
36215
  import {
34844
36216
  GraphStore
@@ -34935,8 +36307,8 @@ async function runCodingGraphBenchmark(config = {}) {
34935
36307
  const sampleRss = () => {
34936
36308
  peakRss = Math.max(peakRss, process.memoryUsage().rss);
34937
36309
  };
34938
- const dir = await mkdtemp12(path36.join(tmpdir7(), "coding-graph-bench-"));
34939
- const dbPath = path36.join(dir, "bench.sqlite");
36310
+ const dir = await mkdtemp12(path37.join(tmpdir7(), "coding-graph-bench-"));
36311
+ const dbPath = path37.join(dir, "bench.sqlite");
34940
36312
  try {
34941
36313
  const store = await GraphStore.open({ dbPath });
34942
36314
  try {