@spinabot/brigade 0.1.1 → 1.0.0

package/dist/agents/memory/eval/gold-rich.js

@@ -0,0 +1,82 @@
+/**
+ * RICH gold set (Tideline build Step 2/3 — the methodology-completeness tier).
+ *
+ * The synthetic + hard sets are SINGLE-RELEVANT: every case has exactly one
+ * answer, so recall@k ≡ hit-rate and nDCG is a monotone transform of MRR — the
+ * four reported metrics carry ONE bit of ranking signal per case (noted in
+ * gold-hard.test.ts). This set adds the cases those can't express:
+ *
+ *   • MULTI-RELEVANT — a query whose answer is a SET of facts that all share a
+ *     query term (lexically findable, so it's a fair ranking test, NOT a synonymy
+ *     test — synonymy is the deferred learned-embedder lane). Now recall@k ("did
+ *     we get ALL of them in the top-k?") and nDCG ("are they ranked above the
+ *     distractors?") carry real, independent signal. Two of the five sets
+ *     (ALLERGIES + TEAM) also carry a same-term DISTRACTOR (a generic/untrusted
+ *     fact that shares the term but isn't the user's) so precision + trust matter;
+ *     the remaining three (LANGUAGES, PROJECTS, MORNING ROUTINE) are pure
+ *     multi-relevant without a distractor, isolating recall completeness.
+ *   • TRANSITION CHAINS — a subject superseded twice (Lisbon→Berlin→Tokyo); only
+ *     the CURRENT fact is relevant + the two stale ones are archived, so this
+ *     verifies bi-temporal recall surfaces the live value and never a stale one.
+ *   • MULTI-SESSION — multi-relevant facts that accumulated across sessions (the
+ *     persistence the "model-switch" north-star reduces to for a RECALL metric;
+ *     true model-switch CONTINUITY is a conversation-layer concern, not recall).
+ *
+ * Honesty: queries share LEXICAL terms with their relevant facts (BM25 can find
+ * them); this measures RANKING of a known relevant set under competition, which
+ * is exactly where BM25-primary + trust/importance modulation should beat a
+ * weighted-sum fusion at the same (model-free) embedder. Pure-synonymy recall
+ * (no shared term) is intentionally NOT here — that needs the learned embedder.
+ */
+export const RICH_GOLD = {
+    facts: [
+        // ── ALLERGIES (multi-relevant; all share "allergic") + an untrusted same-term distractor.
+        { key: "a-shellfish", content: "I am allergic to shellfish.", segment: "identity" },
+        { key: "a-penicillin", content: "I am allergic to penicillin.", segment: "identity" },
+        { key: "a-pollen", content: "I am allergic to pollen in the spring.", segment: "identity" },
+        { key: "a-distract", content: "A medical leaflet explains what being allergic means.", segment: "knowledge", sourceType: "retrieved_document" },
+        // ── TEAM (multi-relevant; all share "team") + a same-term distractor.
+        { key: "t-priya", content: "Priya is my manager on the platform team.", segment: "relationship" },
+        { key: "t-sam", content: "Sam is a backend engineer on my team.", segment: "relationship" },
+        { key: "t-lee", content: "Lee is the designer on my team.", segment: "relationship" },
+        { key: "t-distract", content: "The sales team sits in a different building.", segment: "knowledge", sourceType: "tool_output" },
+        // ── LANGUAGES (multi-relevant; all share "speak").
+        { key: "l-fr", content: "I speak French at a native level.", segment: "identity" },
+        { key: "l-ja", content: "I speak Japanese conversationally.", segment: "identity" },
+        { key: "l-pt", content: "I speak Portuguese fluently.", segment: "identity" },
+        // ── PROJECTS (multi-relevant; all share "project").
+        { key: "pr-tide", content: "I am building the Tideline memory project.", segment: "project" },
+        { key: "pr-dash", content: "I am building the metahuman dashboard project.", segment: "project" },
+        { key: "pr-mobile", content: "I am building the mobile client project.", segment: "project" },
+        // ── MORNING ROUTINE (multi-relevant, multi-session; all share "morning").
+        { key: "rt-coffee", content: "Every morning I drink a flat white before anything else.", segment: "preference" },
+        { key: "rt-gym", content: "I go to the gym every morning before work.", segment: "preference" },
+        { key: "rt-commute", content: "I commute by train every morning.", segment: "preference" },
+        // ── TRANSITION CHAIN: city, superseded twice (all share "live"); only Tokyo is current.
+        { key: "c-lisbon", content: "I live in Lisbon.", segment: "identity" },
+        { key: "c-berlin", content: "I live in Berlin.", segment: "identity", supersedesKeys: ["c-lisbon"] },
+        { key: "c-tokyo", content: "I live in Tokyo.", segment: "identity", supersedesKeys: ["c-berlin"] },
+        // ── TRANSITION CHAIN: role, superseded twice (all share "work as"); only principal is current.
+        { key: "r-intern", content: "I work as an intern.", segment: "project" },
+        { key: "r-eng", content: "I work as a software engineer.", segment: "project", supersedesKeys: ["r-intern"] },
+        { key: "r-principal", content: "I work as a principal engineer.", segment: "project", supersedesKeys: ["r-eng"] },
+        // ── corpus noise (share no query term with any answer).
+        { key: "noise1", content: "Mountain weather turns cold in winter.", segment: "knowledge" },
+        { key: "noise2", content: "A sourdough recipe needs flour and patience.", segment: "knowledge" },
+    ],
+    cases: [
+        // MULTI-RELEVANT — relevant is the SET; where a same-term distractor exists (allergy, team) it must NOT crowd them out.
+        { id: "rc-allergy", query: "what am I allergic to", relevantKeys: ["a-shellfish", "a-penicillin", "a-pollen"], category: "multi-session" },
+        { id: "rc-team", query: "who is on my team", relevantKeys: ["t-priya", "t-sam", "t-lee"], category: "multi-session" },
+        { id: "rc-lang", query: "what languages do I speak", relevantKeys: ["l-fr", "l-ja", "l-pt"], category: "multi-session" },
+        { id: "rc-proj", query: "what projects am I building", relevantKeys: ["pr-tide", "pr-dash", "pr-mobile"], category: "multi-session" },
+        { id: "rc-routine", query: "what do I do every morning", relevantKeys: ["rt-coffee", "rt-gym", "rt-commute"], category: "multi-session" },
+        // TRANSITION — only the CURRENT value is relevant; stale ones are archived.
+        { id: "rc-city", query: "where do I live", relevantKeys: ["c-tokyo"], category: "transition" },
+        { id: "rc-role", query: "what do I work as", relevantKeys: ["r-principal"], category: "transition" },
+        // ABSTENTION — nothing answers these.
+        { id: "rc-movie", query: "what is my favorite movie", relevantKeys: [], category: "abstention" },
+        { id: "rc-shoe", query: "what is my shoe size", relevantKeys: [], category: "abstention" },
+    ],
+};
+//# sourceMappingURL=gold-rich.js.map

package/dist/agents/memory/eval/gold-rich.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-rich.js","sourceRoot":"","sources":["../../../../src/agents/memory/eval/gold-rich.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAIH,MAAM,CAAC,MAAM,SAAS,GAAa;IAClC,KAAK,EAAE;QACN,2FAA2F;QAC3F,EAAE,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,6BAA6B,EAAE,OAAO,EAAE,UAAU,EAAE;QACnF,EAAE,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,8BAA8B,EAAE,OAAO,EAAE,UAAU,EAAE;QACrF,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,wCAAwC,EAAE,OAAO,EAAE,UAAU,EAAE;QAC3F,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,uDAAuD,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,oBAAoB,EAAE;QAE/I,uEAAuE;QACvE,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,2CAA2C,EAAE,OAAO,EAAE,cAAc,EAAE;QACjG,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,uCAAuC,EAAE,OAAO,EAAE,cAAc,EAAE;QAC3F,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,iCAAiC,EAAE,OAAO,EAAE,cAAc,EAAE;QACrF,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,8CAA8C,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE;QAE/H,oDAAoD;QACpD,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,mCAAmC,EAAE,OAAO,EAAE,UAAU,EAAE;QAClF,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,oCAAoC,EAAE,OAAO,EAAE,UAAU,EAAE;QACnF,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,8BAA8B,EAAE,OAAO,EAAE,UAAU,EAAE;QAE7E,qDAAqD;QACrD,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,OAAO,EAAE,SAAS,EAAE;QAC7F,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,gDAAgD,EAAE,OAAO,EAAE,SAAS,EAAE;QACjG,EAAE,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,0CAA0C,EAAE,OAAO,EAAE,SAAS,EAAE;QAE7F,2EAA2E;QAC3E,EAAE,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,0DAA0D,EAAE,OAAO,EAAE,YAAY,EAAE;QAChH,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,4CAA4C,EAAE,OAAO,EAAE,YAAY,EAAE;QAC/F,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,mCAAmC,EAAE,OAAO,EAAE,YAAY,EAAE;QAE1F,yFAAyF;QACzF,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,UAAU,EAAE;QACtE,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE;QACpG,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,kBAAkB,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE;QAElG,gGAAgG;QAChG,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,sBAAsB,EAAE,OAAO,EAAE,SAAS,EAAE;QACxE,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,UAAU,CAAC,EAAE;QAC7G,EAAE,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,iCAAiC,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,OAAO,CAAC,EAAE;QAEjH,yDAAyD;QACzD,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,wCAAwC,EAAE,OAAO,EAAE,WAAW,EAAE;QAC1F,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,8CAA8C,EAAE,OAAO,EAAE,WAAW,EAAE;KAChG;IACD,KAAK,EAAE;QACN,wHAAwH;QACxH,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,aAAa,EAAE,cAAc,EAAE,UAAU,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE;QAC1I,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE;QACrH,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,2BAA2B,EAAE,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE;QACxH,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,6BAA6B,EAAE,YAAY,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE;QACrI,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,4BAA4B,EAAE,YAAY,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE;QAEzI,4EAA4E;QAC5E,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE;QAC9F,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE;QAEpG,sCAAsC;QACtC,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,2BAA2B,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE;QAChG,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,sBAAsB,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE;KAC1F;CACD,CAAC"}

package/dist/agents/memory/eval/gold-synthetic.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Synthetic hard-case gold set (Tideline build Step 2) — a self-contained
+ * corpus + cases covering every taxonomy bucket, runnable today without the
+ * operator's real data. The REAL gold set (export + decrypt the operator's
+ * Convex facts + human-approve) is layered on top of this when we test
+ * together; the synthetic set keeps CI honest in the meantime.
+ *
+ * Construction notes:
+ *   - `job-old` is listed before `job-new`, which supersedes it — so the
+ *     knowledge-update / transition cases must return the NEW employer and the
+ *     archived old fact must NOT surface.
+ *   - Every case shares enough query terms with NON-relevant facts that a
+ *     careless lexical match would over-return — the gold answer is the one
+ *     that should rank, not merely "a fact that contains the word".
+ *   - Abstention cases have no matching fact at all.
+ */
+import type { GoldSpec } from "./gold.js";
+export declare const SYNTHETIC_GOLD: GoldSpec;
+//# sourceMappingURL=gold-synthetic.d.ts.map

package/dist/agents/memory/eval/gold-synthetic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-synthetic.d.ts","sourceRoot":"","sources":["../../../../src/agents/memory/eval/gold-synthetic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,eAAO,MAAM,cAAc,EAAE,QA+C5B,CAAC"}

package/dist/agents/memory/eval/gold-synthetic.js

@@ -0,0 +1,65 @@
+/**
+ * Synthetic hard-case gold set (Tideline build Step 2) — a self-contained
+ * corpus + cases covering every taxonomy bucket, runnable today without the
+ * operator's real data. The REAL gold set (export + decrypt the operator's
+ * Convex facts + human-approve) is layered on top of this when we test
+ * together; the synthetic set keeps CI honest in the meantime.
+ *
+ * Construction notes:
+ *   - `job-old` is listed before `job-new`, which supersedes it — so the
+ *     knowledge-update / transition cases must return the NEW employer and the
+ *     archived old fact must NOT surface.
+ *   - Every case shares enough query terms with NON-relevant facts that a
+ *     careless lexical match would over-return — the gold answer is the one
+ *     that should rank, not merely "a fact that contains the word".
+ *   - Abstention cases have no matching fact at all.
+ */
+export const SYNTHETIC_GOLD = {
+    facts: [
+        { key: "home", content: "I live in Hyderabad, India.", segment: "identity" },
+        { key: "lang", content: "My native language is Telugu.", segment: "identity" },
+        { key: "allergy", content: "I am allergic to peanuts.", segment: "identity", importance: 0.95 },
+        { key: "birthday", content: "My birthday is in May.", segment: "relationship" },
+        { key: "pet", content: "I have a dog named Biscuit.", segment: "relationship" },
+        { key: "pref-editor", content: "I prefer tabs over spaces when coding.", segment: "preference" },
+        { key: "pref-coffee", content: "I drink black coffee with no sugar.", segment: "preference" },
+        { key: "car", content: "I drive a blue Hyundai.", segment: "context" },
+        { key: "proj-tideline", content: "I am building Tideline, a memory framework for Brigade.", segment: "project" },
+        // job-old MUST precede job-new (which supersedes it).
+        { key: "job-old", content: "I work at Acme Corp as a backend engineer.", segment: "project" },
+        {
+            key: "job-new",
+            content: "I now work at Beta Labs as a staff engineer.",
+            segment: "project",
+            supersedesKeys: ["job-old"],
+        },
+    ],
+    cases: [
+        { id: "g-home", query: "where do I live", relevantKeys: ["home"], category: "single-session" },
+        { id: "g-coffee", query: "how do I take my coffee", relevantKeys: ["pref-coffee"], category: "preference" },
+        { id: "g-editor", query: "tabs or spaces", relevantKeys: ["pref-editor"], category: "preference" },
+        { id: "g-pet", query: "what is my dog's name", relevantKeys: ["pet"], category: "single-session" },
+        { id: "g-birthday", query: "when is my birthday", relevantKeys: ["birthday"], category: "temporal" },
+        { id: "g-build", query: "what am I building", relevantKeys: ["proj-tideline"], category: "multi-session" },
+        { id: "g-lang", query: "what language do I speak natively", relevantKeys: ["lang"], category: "single-session" },
+        // knowledge-update: must return the NEW employer; the superseded old one is archived.
+        { id: "g-job-now", query: "where do I work now", relevantKeys: ["job-new"], category: "knowledge-update" },
+        // transition: the move itself — current employer is the live answer. Query
+        // shares the term "work" with job-new so bundled lexical recall can match it.
+        { id: "g-job-change", query: "what is my current job and where do I work", relevantKeys: ["job-new"], category: "transition" },
+        // model-switch: a safety-critical fact that must remain recallable regardless of model.
+        { id: "g-allergy", query: "am I allergic to anything", relevantKeys: ["allergy"], category: "model-switch" },
+        // abstention: nothing in the corpus answers these.
+        { id: "g-movie", query: "what is my favorite movie", relevantKeys: [], category: "abstention" },
+        { id: "g-shoe", query: "what is my shoe size", relevantKeys: [], category: "abstention" },
+        // archived-fact guard: this query lexically matches job-old's content
+        // ("Acme Corp"), but job-old is superseded and must NOT surface — so the
+        // correct behavior is abstention. A job-old hit registers as a violation.
+        // Phrased to share NO term with any ACTIVE fact ("Acme"/"Corp"/"employer"
+        // live only in the archived job-old), honoring the no-matching-active-fact
+        // abstention invariant above — only a recall that wrongly surfaces an
+        // archived fact would hit here.
+        { id: "g-job-archived", query: "is Acme Corp my employer", relevantKeys: [], category: "abstention" },
+    ],
+};
+//# sourceMappingURL=gold-synthetic.js.map

package/dist/agents/memory/eval/gold-synthetic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold-synthetic.js","sourceRoot":"","sources":["../../../../src/agents/memory/eval/gold-synthetic.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,MAAM,CAAC,MAAM,cAAc,GAAa;IACvC,KAAK,EAAE;QACN,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,6BAA6B,EAAE,OAAO,EAAE,UAAU,EAAE;QAC5E,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,+BAA+B,EAAE,OAAO,EAAE,UAAU,EAAE;QAC9E,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE;QAC/F,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,wBAAwB,EAAE,OAAO,EAAE,cAAc,EAAE;QAC/E,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,6BAA6B,EAAE,OAAO,EAAE,cAAc,EAAE;QAC/E,EAAE,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,wCAAwC,EAAE,OAAO,EAAE,YAAY,EAAE;QAChG,EAAE,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,qCAAqC,EAAE,OAAO,EAAE,YAAY,EAAE;QAC7F,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,yBAAyB,EAAE,OAAO,EAAE,SAAS,EAAE;QACtE,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,EAAE,yDAAyD,EAAE,OAAO,EAAE,SAAS,EAAE;QAChH,sDAAsD;QACtD,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,4CAA4C,EAAE,OAAO,EAAE,SAAS,EAAE;QAC7F;YACC,GAAG,EAAE,SAAS;YACd,OAAO,EAAE,8CAA8C;YACvD,OAAO,EAAE,SAAS;YAClB,cAAc,EAAE,CAAC,SAAS,CAAC;SAC3B;KACD;IACD,KAAK,EAAE;QACN,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,gBAAgB,EAAE;QAC9F,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,yBAAyB,EAAE,YAAY,EAAE,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE;QAC3G,EAAE,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE;QAClG,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,gBAAgB,EAAE;QAClG,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;QACpG,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,oBAAoB,EAAE,YAAY,EAAE,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,eAAe,EAAE;QAC1G,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,mCAAmC,EAAE,YAAY,EAAE,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,gBAAgB,EAAE;QAChH,sFAAsF;QACtF,EAAE,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,qBAAqB,EAAE,YAAY,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,kBAAkB,EAAE;QAC1G,2EAA2E;QAC3E,8EAA8E;QAC9E,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,4CAA4C,EAAE,YAAY,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,YAAY,EAAE;QAC9H,wFAAwF;QACxF,EAAE,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,2BAA2B,EAAE,YAAY,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE;QAC5G,mDAAmD;QACnD,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,2BAA2B,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE;QAC/F,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,sBAAsB,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE;QACzF,sEAAsE;QACtE,yEAAyE;QACzE,0EAA0E;QAC1E,0EAA0E;QAC1E,2EAA2E;QAC3E,sEAAsE;QACtE,gCAAgC;QAChC,EAAE,EAAE,EAAE,gBAAgB,EAAE,KAAK,EAAE,0BAA0B,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE;KACrG;CACD,CAAC"}

package/dist/agents/memory/eval/gold.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Tideline gold set — the cases the eval harness scores (build Step 2).
+ *
+ * A gold set is a corpus of FACTS plus CASES (a query + which facts should
+ * surface). Facts are addressed by a stable logical `key` so a case can name
+ * its relevant facts BEFORE their `memoryId`s exist; `seedGold` writes the
+ * facts into a `FactStore`, captures key→memoryId, applies supersedes (so
+ * knowledge-update / transition cases archive the stale fact), and resolves
+ * each case's `relevantKeys` into the `relevantIds` the harness compares
+ * against. EMPTY `relevantKeys` ⇒ an abstention case.
+ *
+ * Two sources, per the plan: the SYNTHETIC hard cases in the sibling
+ * gold-synthetic / gold-hard / gold-rich files (self-contained, runnable
+ * today) and — exported + decrypted + human-approved — the operator's REAL
+ * Convex facts (`loadGoldSpec` reads that JSON; built jointly). This file is
+ * infrastructure only: types, `seedGold`, `loadGoldSpec`, and `GOLD_CATEGORIES`.
+ */
+import { FactStore, type MemoryRecordOrigin, type MemorySegment, type MemorySourceType } from "../records.js";
+import type { EvalCase } from "./harness.js";
+/** A fact in a gold corpus, addressed by a stable logical `key`. */
+export interface GoldFact {
+    key: string;
+    content: string;
+    segment: MemorySegment;
+    importance?: number;
+    createdBy?: MemoryRecordOrigin;
+    /** Provenance — drives trust modulation + the write-gate. Omit ⇒ owner/trusted.
+     *  `tool_output`/`retrieved_document`/`compaction` = untrusted (the poison
+     *  lane the hybrid is meant to down-weight). Untrusted facts may only occupy
+     *  DESCRIPTIVE segments (knowledge/context/project/relationship) — the gate
+     *  blocks them from identity/preference/correction. */
+    sourceType?: MemorySourceType;
+    /** keys this fact supersedes (knowledge-update / transition) — those facts
+     *  must already be listed EARLIER in `facts` (they're written first). */
+    supersedesKeys?: string[];
+}
+/** A gold case referencing relevant facts by their logical key. */
+export interface GoldCase {
+    id: string;
+    query: string;
+    /** logical keys of facts that SHOULD surface. EMPTY ⇒ abstention. */
+    relevantKeys: string[];
+    category: string;
+}
+export interface GoldSpec {
+    facts: GoldFact[];
+    cases: GoldCase[];
+    /** Real-data path ONLY (`loadGoldSpec`): an exported scaffold is `false`; the
+     *  operator MUST review + rewrite the trivial auto-queries into real paraphrases
+     *  and flip this to `true` before it can be scored. `loadGoldSpec` REFUSES a spec
+     *  that isn't explicitly `true` (an un-reviewed scaffold self-matches ⇒ inflated
+     *  recall). Committed hand-authored sets bypass `loadGoldSpec`, so they ignore it. */
+    approved?: boolean;
+}
+/** The full, deliberately-unique sentinel an export scaffold emits for a query it
+ *  could not auto-derive. `loadGoldSpec` rejects any spec whose queries still
+ *  contain it — a left-in placeholder means the operator hasn't finished the
+ *  human-approval rewrite. (The whole phrase, not the bare "TODO: rewrite" token,
+ *  so a legitimate query mentioning "TODO" or "rewrite" is never falsely rejected.) */
+export declare const GOLD_REVIEW_PLACEHOLDER = "TODO: rewrite (auto-extraction empty)";
+/**
+ * Seed `store` with the spec's facts (in order, so supersede targets exist)
+ * and resolve its cases into `EvalCase`s. Returns the cases ready for
+ * `runRecallEval`. Throws on a case referencing an unknown fact key — a typo
+ * in a gold set should fail loud, not silently score zero.
+ */
+export declare function seedGold(store: FactStore, spec: GoldSpec): EvalCase[];
+/** The taxonomy buckets a gold set should cover (the harness groups by these). */
+export declare const GOLD_CATEGORIES: readonly ["single-session", "multi-session", "temporal", "knowledge-update", "preference", "abstention", "model-switch", "transition"];
+/** Load a gold spec from a JSON file (the real-data path: export + decrypt the
+ *  operator's Convex facts → human-approved cases → this JSON). Validates shape. */
+export declare function loadGoldSpec(jsonPath: string): GoldSpec;
+//# sourceMappingURL=gold.d.ts.map

package/dist/agents/memory/eval/gold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold.d.ts","sourceRoot":"","sources":["../../../../src/agents/memory/eval/gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,EACN,SAAS,EAGT,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,gBAAgB,EAErB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAE7C,oEAAoE;AACpE,MAAM,WAAW,QAAQ;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,aAAa,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,kBAAkB,CAAC;IAC/B;;;;2DAIuD;IACvD,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;6EACyE;IACzE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAED,mEAAmE;AACnE,MAAM,WAAW,QAAQ;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,qEAAqE;IACrE,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB;;;;0FAIsF;IACtF,QAAQ,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;;uFAIuF;AACvF,eAAO,MAAM,uBAAuB,0CAA0C,CAAC;AAE/E;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,GAAG,QAAQ,EAAE,CA8ErE;AAED,kFAAkF;AAClF,eAAO,MAAM,eAAe,wIASlB,CAAC;AAEX;oFACoF;AACpF,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAyDvD"}

package/dist/agents/memory/eval/gold.js

@@ -0,0 +1,174 @@
+/**
+ * Tideline gold set — the cases the eval harness scores (build Step 2).
+ *
+ * A gold set is a corpus of FACTS plus CASES (a query + which facts should
+ * surface). Facts are addressed by a stable logical `key` so a case can name
+ * its relevant facts BEFORE their `memoryId`s exist; `seedGold` writes the
+ * facts into a `FactStore`, captures key→memoryId, applies supersedes (so
+ * knowledge-update / transition cases archive the stale fact), and resolves
+ * each case's `relevantKeys` into the `relevantIds` the harness compares
+ * against. EMPTY `relevantKeys` ⇒ an abstention case.
+ *
+ * Two sources, per the plan: the SYNTHETIC hard cases in the sibling
+ * gold-synthetic / gold-hard / gold-rich files (self-contained, runnable
+ * today) and — exported + decrypted + human-approved — the operator's REAL
+ * Convex facts (`loadGoldSpec` reads that JSON; built jointly). This file is
+ * infrastructure only: types, `seedGold`, `loadGoldSpec`, and `GOLD_CATEGORIES`.
+ */
+import * as fs from "node:fs";
+import { MEMORY_SEGMENTS, MEMORY_SOURCE_TYPES, } from "../records.js";
+/** The full, deliberately-unique sentinel an export scaffold emits for a query it
+ *  could not auto-derive. `loadGoldSpec` rejects any spec whose queries still
+ *  contain it — a left-in placeholder means the operator hasn't finished the
+ *  human-approval rewrite. (The whole phrase, not the bare "TODO: rewrite" token,
+ *  so a legitimate query mentioning "TODO" or "rewrite" is never falsely rejected.) */
+export const GOLD_REVIEW_PLACEHOLDER = "TODO: rewrite (auto-extraction empty)";
+/**
+ * Seed `store` with the spec's facts (in order, so supersede targets exist)
+ * and resolve its cases into `EvalCase`s. Returns the cases ready for
+ * `runRecallEval`. Throws on a case referencing an unknown fact key — a typo
+ * in a gold set should fail loud, not silently score zero.
+ */
+export function seedGold(store, spec) {
+    const keyToId = new Map();
+    const seenIds = new Set();
+    const explicitlySuperseded = new Set();
+    for (const f of spec.facts) {
+        // Keys must be unique — a duplicate would silently overwrite in keyToId
+        // and a case could then resolve to the wrong fact.
+        if (keyToId.has(f.key)) {
+            throw new Error(`gold fact "${f.key}": duplicate fact key — keys must be unique`);
+        }
+        const supersedes = f.supersedesKeys?.map((k) => {
+            const id = keyToId.get(k);
+            if (!id) {
+                throw new Error(`gold fact "${f.key}": supersedesKey "${k}" must be listed earlier in facts`);
+            }
+            return id;
+        });
+        for (const id of supersedes ?? [])
+            explicitlySuperseded.add(id);
+        const nf = {
+            content: f.content,
+            segment: f.segment,
+            ...(f.importance !== undefined ? { importance: f.importance } : {}),
+            ...(f.createdBy !== undefined ? { createdBy: f.createdBy } : {}),
+            ...(f.sourceType !== undefined ? { sourceType: f.sourceType } : {}),
+            ...(supersedes && supersedes.length > 0 ? { supersedes } : {}),
+        };
+        const rec = store.write(nf);
+        // Fail loud on a write-time dedup collision: FactStore.write merges a
+        // near-identical same-origin fact into an existing record and returns THAT
+        // id, so two distinct gold keys could silently map to one memoryId —
+        // corrupting relevance labels. Gold facts must stay distinct.
+        if (seenIds.has(rec.memoryId)) {
+            throw new Error(`gold fact "${f.key}": write-time dedup merged it into an earlier fact (id ${rec.memoryId}). ` +
+                `Make its content less similar to a sibling, or give it a distinct supersedesKey.`);
+        }
+        seenIds.add(rec.memoryId);
+        keyToId.set(f.key, rec.memoryId);
+    }
+    // CORPUS CONSERVATION: every written fact must remain ACTIVE unless it was an
+    // EXPLICIT supersedesKey target. A silent shrink — an auto-supersede (a
+    // content/slot collision) archiving an UNREFERENCED distractor — would quietly
+    // weaken the competition a case relies on (the corpus is smaller / easier than
+    // authored) without tripping the per-case guard below. Fail loud so the gold
+    // author makes the distractor distinct or supersedes it explicitly.
+    const active = store.list();
+    const expectedActive = spec.facts.length - explicitlySuperseded.size;
+    if (active.length !== expectedActive) {
+        const activeSet = new Set(active.map((r) => r.memoryId));
+        const silentlyArchived = [...seenIds].filter((id) => !activeSet.has(id) && !explicitlySuperseded.has(id)).length;
+        throw new Error(`gold seeding shrank the corpus: ${active.length} active vs ${expectedActive} expected ` +
+            `(${spec.facts.length} written − ${explicitlySuperseded.size} explicitly superseded); ` +
+            `${silentlyArchived} fact(s) auto-archived by a content/slot collision — make them distinct or supersede explicitly.`);
+    }
+    // Fail loud if a later correction/slot-supersede write ARCHIVED a fact a case
+    // still references: store.list() is active-only, so its label would silently
+    // score zero. (Extends the dedup-collision fail-loud above to the content/slot
+    // auto-supersede path.)
+    const activeIds = new Set(active.map((r) => r.memoryId));
+    return spec.cases.map((c) => ({
+        id: c.id,
+        query: c.query,
+        relevantIds: c.relevantKeys.map((k) => {
+            const id = keyToId.get(k);
+            if (!id)
+                throw new Error(`gold case "${c.id}": unknown fact key "${k}"`);
+            if (!activeIds.has(id)) {
+                throw new Error(`gold case "${c.id}": fact key "${k}" resolved to a record archived by a later ` +
+                    `supersede/correction write — its relevance label would silently score zero.`);
+            }
+            return id;
+        }),
+        category: c.category,
+    }));
+}
+/** The taxonomy buckets a gold set should cover (the harness groups by these). */
+export const GOLD_CATEGORIES = [
+    "single-session",
+    "multi-session",
+    "temporal",
+    "knowledge-update",
+    "preference",
+    "abstention",
+    "model-switch",
+    "transition",
+];
+/** Load a gold spec from a JSON file (the real-data path: export + decrypt the
+ *  operator's Convex facts → human-approved cases → this JSON). Validates shape. */
+export function loadGoldSpec(jsonPath) {
+    const raw = JSON.parse(fs.readFileSync(jsonPath, "utf8"));
+    if (!Array.isArray(raw.facts) || !Array.isArray(raw.cases)) {
+        throw new Error(`gold spec at ${jsonPath} must have { facts: [], cases: [] }`);
+    }
+    raw.facts.forEach((f, i) => {
+        if (typeof f?.key !== "string" || typeof f?.content !== "string" || typeof f?.segment !== "string") {
+            throw new Error(`gold spec at ${jsonPath}: facts[${i}] needs string key, content, and segment`);
+        }
+        // Validate segment against the known set — an unknown/typo'd segment would
+        // silently degrade to `context` defaults (skewing tier/importance/decay).
+        if (!MEMORY_SEGMENTS.includes(f.segment)) {
+            throw new Error(`gold spec at ${jsonPath}: facts[${i}] has unknown segment "${f.segment}"`);
+        }
+        // Validate sourceType the SAME way (when present) — anything outside the known
+        // set is treated as TRUSTED by the write-gate (isUntrustedSource → false), so a
+        // typo'd "tool-output" would silently turn a poison distractor into a trusted
+        // answer. Mirror the segment guard; fail loud rather than mislabel the trust lane.
+        if (f.sourceType !== undefined && !MEMORY_SOURCE_TYPES.includes(f.sourceType)) {
+            throw new Error(`gold spec at ${jsonPath}: facts[${i}] has unknown sourceType "${f.sourceType}"`);
+        }
+    });
+    raw.cases.forEach((c, i) => {
+        if (typeof c?.id !== "string" || typeof c?.query !== "string") {
+            throw new Error(`gold spec at ${jsonPath}: cases[${i}] needs string id and query`);
+        }
+        if (!Array.isArray(c.relevantKeys)) {
+            throw new Error(`gold spec at ${jsonPath}: cases[${i}] (${c.id}) needs an array relevantKeys`);
+        }
+        if (!c.relevantKeys.every((k) => typeof k === "string")) {
+            throw new Error(`gold spec at ${jsonPath}: cases[${i}] (${c.id}) relevantKeys must all be strings`);
+        }
+        if (typeof c.category !== "string") {
+            throw new Error(`gold spec at ${jsonPath}: cases[${i}] (${c.id}) needs a string category`);
+        }
+    });
+    // APPROVAL GATE — an exported scaffold's queries are lifted straight from its
+    // facts (trivially self-matching ⇒ inflated recall). It is NOT a measurement
+    // until the operator rewrites those queries into real paraphrases and marks the
+    // spec reviewed. Refuse to load anything not explicitly approved, and refuse a
+    // spec that still carries an auto-extraction placeholder (an un-rewritten case).
+    if (raw.approved !== true) {
+        throw new Error(`gold spec at ${jsonPath} is an un-approved scaffold (approved !== true). Its auto-generated ` +
+            `queries self-match their own facts and would inflate recall. Review each case — rewrite the ` +
+            `query into a realistic paraphrase, set the GOLD_CATEGORIES taxonomy label, drop noise/mark ` +
+            `abstentions — then set "approved": true to score it.`);
+    }
+    const placeheld = raw.cases.find((c) => c.query.includes(GOLD_REVIEW_PLACEHOLDER));
+    if (placeheld) {
+        throw new Error(`gold spec at ${jsonPath}: case "${placeheld.id}" still carries the "${GOLD_REVIEW_PLACEHOLDER}" ` +
+            `placeholder — finish the human-approval rewrite before scoring (a left-in placeholder is an un-reviewed case).`);
+    }
+    return raw;
+}
+//# sourceMappingURL=gold.js.map

package/dist/agents/memory/eval/gold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gold.js","sourceRoot":"","sources":["../../../../src/agents/memory/eval/gold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAE9B,OAAO,EAEN,eAAe,EACf,mBAAmB,GAKnB,MAAM,eAAe,CAAC;AAyCvB;;;;uFAIuF;AACvF,MAAM,CAAC,MAAM,uBAAuB,GAAG,uCAAuC,CAAC;AAE/E;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAgB,EAAE,IAAc;IACxD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC5B,wEAAwE;QACxE,mDAAmD;QACnD,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,6CAA6C,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,UAAU,GAAG,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,EAAE,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,qBAAqB,CAAC,mCAAmC,CAAC,CAAC;YAC/F,CAAC;YACD,OAAO,EAAE,CAAC;QACX,CAAC,CAAC,CAAC;QACH,KAAK,MAAM,EAAE,IAAI,UAAU,IAAI,EAAE;YAAE,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,EAAE,GAAY;YACnB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9D,CAAC;QACF,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5B,sEAAsE;QACtE,2EAA2E;QAC3E,qEAAqE;QACrE,8DAA8D;QAC9D,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACd,cAAc,CAAC,CAAC,GAAG,0DAA0D,GAAG,CAAC,QAAQ,KAAK;gBAC7F,kFAAkF,CACnF,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IACD,8EAA8E;IAC9E,wEAAwE;IACxE,+EAA+E;IAC/E,+EAA+E;IAC/E,6EAA6E;IAC7E,oEAAoE;IACpE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,oBAAoB,CAAC,IAAI,CAAC;IACrE,IAAI,MAAM,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzD,MAAM,gBAAgB,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACjH,MAAM,IAAI,KAAK,CACd,mCAAmC,MAAM,CAAC,MAAM,cAAc,cAAc,YAAY;YACvF,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,cAAc,oBAAoB,CAAC,IAAI,2BAA2B;YACvF,GAAG,gBAAgB,kGAAkG,CACtH,CAAC;IACH,CAAC;IACD,8EAA8E;IAC9E,6EAA6E;IAC7E,+EAA+E;IAC/E,wBAAwB;IACxB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,WAAW,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACrC,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,wBAAwB,CAAC,GAAG,CAAC,CAAC;YACzE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACd,cAAc,CAAC,CAAC,EAAE,gBAAgB,CAAC,6CAA6C;oBAC/E,6EAA6E,CAC9E,CAAC;YACH,CAAC;YACD,OAAO,EAAE,CAAC;QACX,CAAC,CAAC;QACF,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACpB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kFAAkF;AAClF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,gBAAgB;IAChB,eAAe;IACf,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,YAAY;CACH,CAAC;AAEX;oFACoF;AACpF,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAa,CAAC;IACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,qCAAqC,CAAC,CAAC;IAChF,CAAC;IACD,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,EAAE,GAAG,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,OAAO,KAAK,QAAQ,EAAE,CAAC;YACpG,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,0CAA0C,CAAC,CAAC;QACjG,CAAC;QACD,2EAA2E;QAC3E,0EAA0E;QAC1E,IAAI,CAAE,eAAqC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,0BAA0B,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;QAC7F,CAAC;QACD,+EAA+E;QAC/E,gFAAgF;QAChF,8EAA8E;QAC9E,mFAAmF;QACnF,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAE,mBAAyC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAoB,CAAC,EAAE,CAAC;YAChH,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,6BAA6B,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;QACnG,CAAC;IACF,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,EAAE,EAAE,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,6BAA6B,CAAC,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,+BAA+B,CAAC,CAAC;QAChG,CAAC;QACD,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;QACrG,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,gBAAgB,QAAQ,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;QAC5F,CAAC;IACF,CAAC,CAAC,CAAC;IACH,8EAA8E;IAC9E,6EAA6E;IAC7E,gFAAgF;IAChF,+EAA+E;IAC/E,iFAAiF;IACjF,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACd,gBAAgB,QAAQ,sEAAsE;YAC7F,8FAA8F;YAC9F,6FAA6F;YAC7F,sDAAsD,CACvD,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC,CAAC;IACnF,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACd,gBAAgB,QAAQ,WAAW,SAAS,CAAC,EAAE,wBAAwB,uBAAuB,IAAI;YACjG,gHAAgH,CACjH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC"}

package/dist/agents/memory/eval/harness.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Tideline eval harness — Step 1 of the build (the measuring stick).
+ *
+ * `runRecallEval(cap, cases)` drives a backend-agnostic search capability over
+ * a set of gold cases and reports recall@k / MRR / nDCG@k / hitRate + latency
+ * percentiles. It is what turns "best" from a claim into a NUMBER: every
+ * backend (the bundled file store, the convex store, a future hybrid index, a
+ * reproduced competitor baseline) is scored through the SAME function on the
+ * SAME cases, so the comparison is honest.
+ *
+ * Backend-agnostic by construction: the harness depends ONLY on the minimal
+ * `RecallCapability` seam below — a structural subset of the production
+ * `MemoryCapability` (src/agents/extensions/types.ts:677) — so a real backend,
+ * a baseline, or a fixture fake all plug in identically. This seam is what the
+ * SPI freeze (build step 5) lifts into the standalone Tideline package.
+ *
+ * Abstention cases (queries whose correct answer is "nothing relevant",
+ * `relevantIds: []`) are EXCLUDED from the accuracy means — a recall
+ * denominator of zero is meaningless. They're tallied separately so a backend
+ * that surfaces a hit on a no-answer query is still caught (abstentionViolations).
+ */
+/** One ranked hit. The only field the harness needs is a stable `id`. */
+export interface RecallHit {
+    id: string;
+    score?: number;
+}
+/**
+ * The minimal search seam the harness scores. The production
+ * `MemoryCapability.search` (returns `{id, content, score, source}[]`)
+ * satisfies this structurally; so does a baseline or a fixture fake.
+ */
+export interface RecallCapability {
+    search(query: string, opts?: {
+        limit?: number;
+    }): Promise<ReadonlyArray<RecallHit>>;
+}
+/** A gold case: a query + the ids that SHOULD surface. */
+export interface EvalCase {
+    /** Stable case id — for per-case reporting + debugging a regression. */
+    id: string;
+    query: string;
+    /** Gold ids that should appear. EMPTY ⇒ abstention (correct answer = nothing). */
+    relevantIds: readonly string[];
+    /**
+     * Taxonomy bucket — single-session | multi-session | temporal |
+     * knowledge-update | preference | abstention | model-switch | transition.
+     * Free-form so the gold set can add buckets without a code change; the
+     * harness just groups by it.
+     */
+    category?: string;
+}
+export interface PerCaseResult {
+    caseId: string;
+    category?: string;
+    abstention: boolean;
+    retrievedIds: string[];
+    /** undefined on abstention cases (excluded from accuracy). */
+    recallAtK?: number;
+    reciprocalRank?: number;
+    ndcgAtK?: number;
+    hitAtK?: number;
+    /** abstention only: true if the backend surfaced ANY hit (should have stayed quiet). */
+    abstentionViolation?: boolean;
+    latencyMs: number;
+}
+export interface CategoryRollup {
+    n: number;
+    recallAtK: number;
+    mrr: number;
+    ndcgAtK: number;
+    hitRate: number;
+}
+export interface RecallEvalResult {
+    k: number;
+    /** total cases run. */
+    n: number;
+    /** non-abstention cases — the accuracy denominator. */
+    nScored: number;
+    nAbstention: number;
+    /** abstention cases where the backend wrongly surfaced a hit. */
+    abstentionViolations: number;
+    recallAtK: number;
+    mrr: number;
+    ndcgAtK: number;
+    hitRate: number;
+    p50LatencyMs: number;
+    p95LatencyMs: number;
+    byCategory: Record<string, CategoryRollup>;
+    perCase: PerCaseResult[];
+}
+export interface RunRecallEvalOptions {
+    /** Top-k cutoff for the rank metrics. Default 10. */
+    k?: number;
+    /**
+     * `limit` passed to `search`. Default = max(k, 20) so the backend returns
+     * enough to fill the top-k window even when it over-fetches for reranking.
+     */
+    limit?: number;
+    /** Injectable monotonic clock (ms) for deterministic latency tests. */
+    clock?: () => number;
+}
+/**
+ * Score `cap` over `cases`. Drives `cap.search` once per case (timed),
+ * computes per-case metrics, and aggregates the accuracy means over the
+ * non-abstention cases only. Per-category rollups group by `case.category`.
+ */
+export declare function runRecallEval(cap: RecallCapability, cases: ReadonlyArray<EvalCase>, opts?: RunRecallEvalOptions): Promise<RecallEvalResult>;
+/** Compact human-readable summary — the "show me the numbers" line. */
+export declare function formatRecallEval(res: RecallEvalResult): string;
+//# sourceMappingURL=harness.d.ts.map

package/dist/agents/memory/eval/harness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../../../../src/agents/memory/eval/harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAMH,yEAAyE;AACzE,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAChC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC;CACpF;AAED,0DAA0D;AAC1D,MAAM,WAAW,QAAQ;IACxB,wEAAwE;IACxE,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IAC/B;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wFAAwF;IACxF,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC9B,CAAC,EAAE,MAAM,CAAC;IACV,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAChC,CAAC,EAAE,MAAM,CAAC;IACV,uBAAuB;IACvB,CAAC,EAAE,MAAM,CAAC;IACV,uDAAuD;IACvD,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,oBAAoB,EAAE,MAAM,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC3C,OAAO,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,oBAAoB;IACpC,qDAAqD;IACrD,CAAC,CAAC,EAAE,MAAM,CAAC;IACX;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;CACrB;AAID;;;;GAIG;AACH,wBAAsB,aAAa,CAClC,GAAG,EAAE,gBAAgB,EACrB,KAAK,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC9B,IAAI,GAAE,oBAAyB,GAC7B,OAAO,CAAC,gBAAgB,CAAC,CA2E3B;AAED,uEAAuE;AACvE,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,GAAG,MAAM,CAyB9D"}