cognitive-core 0.2.4 → 0.3.1

package/dist/eval/memory-ab.js

@@ -0,0 +1,161 @@
+import { DEFAULT_EVAL_CONFIG } from './types.js';
+import { SequentialEvalRunner } from './runner.js';
+import { ciDisjoint } from './metrics.js';
+import { evaluateLadder, formatLadder, } from './retrieval/ladder.js';
+/**
+ * Memory A/B harness — drives the proxy → outcome ladder from a real run.
+ *
+ * Two distinct A/B axes, with different backends:
+ *
+ *   1. memory on/off          — does carried memory help at all? The
+ *      `MemorySensitiveMockBackend` models knowledge *presence* (cold vs warm
+ *      success prob), so this runs deterministically on the mock right now
+ *      ({@link runMemoryOnOffMock}).
+ *
+ *   2. embedding vs lexical   — does the A-MEM retrieval upgrade specifically
+ *      help? This depends on retrieval *relevance*, which the mock does NOT
+ *      model (it only knows whether knowledge was surfaced, not how good it
+ *      was). So it needs the live agent: run the live harness twice (one arm
+ *      with `setEmbeddingProvider(provider)`, one with `setEmbeddingProvider(
+ *      provider, { lexicalRetrieval: true })`), then feed the two
+ *      `ConditionResult`s plus the offline `ABSignal` into
+ *      {@link compareMemoryArms}.
+ *
+ * Either way the comparison + ladder composition is the same backend-agnostic
+ * code path here; only the source of the arms differs.
+ */
+const NON_ERROR = (r) => r.status !== 'env_error';
+function flat(c) {
+    return c.perSeed.flat().filter(NON_ERROR);
+}
+/** Fraction of (non-error) tasks where relevant knowledge was surfaced. */
+function knowledgeSurfacedRate(c) {
+    const tasks = flat(c);
+    if (tasks.length === 0)
+        return 0;
+    return tasks.filter((r) => r.knowledgeSurfaced).length / tasks.length;
+}
+/** Success rate among tasks that surfaced knowledge (utilization → success). */
+function surfacedSuccessRate(c) {
+    const surfaced = flat(c).filter((r) => r.knowledgeSurfaced);
+    if (surfaced.length === 0)
+        return 0;
+    return surfaced.filter((r) => r.success).length / surfaced.length;
+}
+function summarize(c) {
+    return {
+        label: c.label,
+        successRate: c.successRate,
+        successRateCI: c.successRateCI,
+        tasks: flat(c).length,
+        knowledgeSurfacedRate: knowledgeSurfacedRate(c),
+    };
+}
+/**
+ * Frame two arms as the ladder's outcome rungs: `treatment` is "guided",
+ * `baseline` is "unguided", so `successRateDelta` is the **arm-vs-arm lift**
+ * (the A/B signal we care about), and `knowledgeApplicationRate` is the
+ * treatment arm's utilization. Pass the treatment's real
+ * `memory.getEffectivenessMetrics()` as `liveTreatment` to use true
+ * application inference instead of the surfaced-rate proxy.
+ */
+export function armsToEffectiveness(treatment, baseline, liveTreatment) {
+    const tTasks = flat(treatment);
+    const bTasks = flat(baseline);
+    const utilization = liveTreatment?.knowledgeApplicationRate ?? knowledgeSurfacedRate(treatment);
+    const tAvgSteps = tTasks.length ? tTasks.reduce((s, r) => s + r.stepCount, 0) / tTasks.length : 0;
+    const bAvgSteps = bTasks.length ? bTasks.reduce((s, r) => s + r.stepCount, 0) / bTasks.length : 0;
+    return {
+        totalTasks: tTasks.length + bTasks.length,
+        guidedTasks: tTasks.length,
+        unguidedTasks: bTasks.length,
+        guidedSuccessRate: treatment.successRate,
+        unguidedSuccessRate: baseline.successRate,
+        successRateDelta: treatment.successRate - baseline.successRate,
+        guidedAvgSteps: tAvgSteps,
+        unguidedAvgSteps: bAvgSteps,
+        stepCountDelta: tAvgSteps - bAvgSteps,
+        knowledgeApplicationRate: utilization,
+        appliedKnowledgeSuccessRate: liveTreatment?.appliedKnowledgeSuccessRate ?? surfacedSuccessRate(treatment),
+        knowledgeNoteGuidedTasks: liveTreatment?.knowledgeNoteGuidedTasks ?? 0,
+        knowledgeNoteGuidedSuccessRate: liveTreatment?.knowledgeNoteGuidedSuccessRate ?? 0,
+        noKnowledgeNoteSuccessRate: liveTreatment?.noKnowledgeNoteSuccessRate ?? 0,
+        knowledgeNoteSuccessDelta: liveTreatment?.knowledgeNoteSuccessDelta ?? 0,
+        guidedAvgErrorRecoveries: 0,
+        unguidedAvgErrorRecoveries: 0,
+        playbookEffectiveness: [],
+        windowStart: new Date(0),
+        windowEnd: new Date(0),
+    };
+}
+/**
+ * Compare a treatment arm against a baseline arm and (when an offline retrieval
+ * `ABSignal` is supplied) compose the full proxy → outcome ladder.
+ */
+export function compareMemoryArms(input) {
+    const byLabel = new Map(input.arms.map((c) => [c.label, c]));
+    const treatment = byLabel.get(input.treatment);
+    const baseline = byLabel.get(input.baseline);
+    if (!treatment || !baseline) {
+        throw new Error(`compareMemoryArms: missing arm(s) — have [${[...byLabel.keys()].join(', ')}], ` +
+            `need treatment="${input.treatment}" + baseline="${input.baseline}"`);
+    }
+    const ladder = input.retrieval
+        ? evaluateLadder({
+            retrieval: input.retrieval,
+            effectiveness: armsToEffectiveness(treatment, baseline, input.liveTreatmentEffectiveness),
+            thresholds: input.ladderThresholds,
+        })
+        : undefined;
+    return {
+        backend: input.backend ?? 'mock',
+        arms: input.arms.map(summarize),
+        comparison: {
+            treatment: input.treatment,
+            baseline: input.baseline,
+            successDelta: treatment.successRate - baseline.successRate,
+            ciDisjoint: ciDisjoint(treatment.successRateCI, baseline.successRateCI),
+        },
+        ladder,
+        note: input.note,
+    };
+}
+/**
+ * Axis 1 — memory on/off, on the deterministic mock. Runs the existing
+ * `SequentialEvalRunner` arm machinery (carryMemory true vs false) and returns
+ * the comparison. Validates the whole pipeline without a live agent; the
+ * embedding-vs-lexical ladder needs {@link compareMemoryArms} fed by live arms.
+ */
+export async function runMemoryOnOffMock(opts) {
+    const config = { ...DEFAULT_EVAL_CONFIG, ...opts?.config, backend: 'mock' };
+    const runner = new SequentialEvalRunner(config);
+    const withMemory = await runner.runArm('with-memory', { carryMemory: true });
+    const noMemory = await runner.runArm('no-memory', { carryMemory: false });
+    return compareMemoryArms({
+        arms: [withMemory, noMemory],
+        treatment: 'with-memory',
+        baseline: 'no-memory',
+        retrieval: opts?.retrieval,
+        backend: 'mock',
+        note: 'Mock A/B (knowledge presence). Embedding-vs-lexical retrieval quality needs the live backend.',
+    });
+}
+/** Human-readable report for logs / `docs/runs`. */
+export function formatMemoryAB(r) {
+    const pct = (x) => `${(x * 100).toFixed(0)}%`;
+    const lines = [`memory A/B (${r.backend}):`];
+    for (const a of r.arms) {
+        lines.push(`  ${a.label.padEnd(13)} success ${pct(a.successRate).padStart(4)} ` +
+            `CI[${pct(a.successRateCI[0])}, ${pct(a.successRateCI[1])}] | ` +
+            `surfaced ${pct(a.knowledgeSurfacedRate)} | n=${a.tasks}`);
+    }
+    lines.push(`  Δ ${r.comparison.treatment}−${r.comparison.baseline} = ` +
+        `${r.comparison.successDelta >= 0 ? '+' : ''}${pct(r.comparison.successDelta)} ` +
+        `${r.comparison.ciDisjoint ? '✓CIs disjoint' : '✗CIs overlap'}`);
+    if (r.note)
+        lines.push(`  note: ${r.note}`);
+    if (r.ladder)
+        lines.push('', formatLadder(r.ladder));
+    return lines.join('\n');
+}
+//# sourceMappingURL=memory-ab.js.map

package/dist/eval/memory-ab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-ab.js","sourceRoot":"","sources":["../../src/eval/memory-ab.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG1C,OAAO,EACL,cAAc,EACd,YAAY,GAGb,MAAM,uBAAuB,CAAC;AAE/B;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,MAAM,SAAS,GAAG,CAAC,CAAa,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;AAE9D,SAAS,IAAI,CAAC,CAAkB;IAC9B,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AAC5C,CAAC;AAED,2EAA2E;AAC3E,SAAS,qBAAqB,CAAC,CAAkB;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACtB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;AACxE,CAAC;AAED,gFAAgF;AAChF,SAAS,mBAAmB,CAAC,CAAkB;IAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAC5D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;AACpE,CAAC;AAUD,SAAS,SAAS,CAAC,CAAkB;IACnC,OAAO;QACL,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;QACrB,qBAAqB,EAAE,qBAAqB,CAAC,CAAC,CAAC;KAChD,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CACjC,SAA0B,EAC1B,QAAyB,EACzB,aAAoC;IAEpC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,aAAa,EAAE,wBAAwB,IAAI,qBAAqB,CAAC,SAAS,CAAC,CAAC;IAChG,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAElG,OAAO;QACL,UAAU,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QACzC,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,aAAa,EAAE,MAAM,CAAC,MAAM;QAC5B,iBAAiB,EAAE,SAAS,CAAC,WAAW;QACxC,mBAAmB,EAAE,QAAQ,CAAC,WAAW;QACzC,gBAAgB,EAAE,SAAS,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW;QAC9D,cAAc,EAAE,SAAS;QACzB,gBAAgB,EAAE,SAAS;QAC3B,cAAc,EAAE,SAAS,GAAG,SAAS;QACrC,wBAAwB,EAAE,WAAW;QACrC,2BAA2B,EACzB,aAAa,EAAE,2BAA2B,IAAI,mBAAmB,CAAC,SAAS,CAAC;QAC9E,wBAAwB,EAAE,aAAa,EAAE,wBAAwB,IAAI,CAAC;QACtE,8BAA8B,EAAE,aAAa,EAAE,8BAA8B,IAAI,CAAC;QAClF,0BAA0B,EAAE,aAAa,EAAE,0BAA0B,IAAI,CAAC;QAC1E,yBAAyB,EAAE,aAAa,EAAE,yBAAyB,IAAI,CAAC;QACxE,wBAAwB,EAAE,CAAC;QAC3B,0BAA0B,EAAE,CAAC;QAC7B,qBAAqB,EAAE,EAAE;QACzB,WAAW,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACxB,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;KACvB,CAAC;AACJ,CAAC;AAiBD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KASjC;IACC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;YAC9E,mBAAmB,KAAK,CAAC,SAAS,iBAAiB,KAAK,CAAC,QAAQ,GAAG,CACvE,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS;QAC5B,CAAC,CAAC,cAAc,CAAC;YACb,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,aAAa,EAAE,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,0BAA0B,CAAC;YACzF,UAAU,EAAE,KAAK,CAAC,gBAAgB;SACnC,CAAC;QACJ,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,MAAM;QAChC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;QAC/B,UAAU,EAAE;YACV,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,SAAS,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW;YAC1D,UAAU,EAAE,UAAU,CAAC,SAAS,CAAC,aAAa,EAAE,QAAQ,CAAC,aAAa,CAAC;SACxE;QACD,MAAM;QACN,IAAI,EAAE,KAAK,CAAC,IAAI;KACjB,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAGxC;IACC,MAAM,MAAM,GAAkB,EAAE,GAAG,mBAAmB,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3F,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IAE1E,OAAO,iBAAiB,CAAC;QACvB,IAAI,EAAE,CAAC,UAAU,EAAE,QAAQ,CAAC;QAC5B,SAAS,EAAE,aAAa;QACxB,QAAQ,EAAE,WAAW;QACrB,SAAS,EAAE,IAAI,EAAE,SAAS;QAC1B,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,+FAA+F;KACtG,CAAC,CAAC;AACL,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,cAAc,CAAC,CAAiB;IAC9C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACtD,MAAM,KAAK,GAAG,CAAC,eAAe,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;YAClE,MAAM,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM;YAC/D,YAAY,GAAG,CAAC,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAC5D,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,CAAC,UAAU,CAAC,QAAQ,KAAK;QACzD,GAAG,CAAC,CAAC,UAAU,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,GAAG;QAChF,GAAG,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,EAAE,CAClE,CAAC;IACF,IAAI,CAAC,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,IAAI,CAAC,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/eval/memory-mock.d.ts

@@ -0,0 +1,54 @@
+import type { AgentBackend, AgentSpawnConfig, AgentSession } from '../runtime/types.js';
+/** Per-task telemetry the runner reads back after a solve. */
+export interface MockTelemetry {
+    knowledgeSurfaced: boolean;
+    stepCount: number;
+}
+/**
+ * Memory-sensitive mock backend for the learning-curve eval.
+ *
+ * Unlike the generic test MockBackend (fixed behavior per task description),
+ * this backend's success probability depends on whether *relevant learned
+ * knowledge* was injected for the task's skill family:
+ *
+ *   - "warm" (a prior successful experience or a playbook for this skill was
+ *     surfaced) → high success probability, fewer steps.
+ *   - "cold" (nothing relevant surfaced) → low success probability, more steps.
+ *
+ * This lets the harness validate its own machinery: with learning enabled the
+ * system accumulates experiences and the success curve must bend upward; with
+ * learning disabled every task stays cold and the curve stays flat. The draw is
+ * deterministic per task id (seeded), so the two conditions form a paired
+ * comparison — the only thing that differs is warm vs. cold, not luck.
+ */
+export declare class MemorySensitiveMockBackend implements AgentBackend {
+    private readonly opts;
+    readonly name = "mock";
+    readonly supportedTypes: string[];
+    private sessions;
+    private telemetry;
+    constructor(opts: {
+        seed: number;
+        coldProb: number;
+        warmProb: number;
+    });
+    isAvailable(): Promise<boolean>;
+    /** Read back what happened for a given task id (set during spawn). */
+    getTelemetry(taskId: string): MockTelemetry | undefined;
+    spawn(config: AgentSpawnConfig): Promise<AgentSession>;
+    /**
+     * A task is "warm" if the injected knowledge contains either a prior
+     * successful experience for this skill, or a playbook mentioning it.
+     */
+    private hasRelevantKnowledge;
+    getSession(sessionId: string): Promise<AgentSession | undefined>;
+    listSessions(): Promise<AgentSession[]>;
+    terminate(sessionId: string): Promise<void>;
+    clear(): void;
+}
+export declare function createMemorySensitiveMock(opts: {
+    seed: number;
+    coldProb: number;
+    warmProb: number;
+}): MemorySensitiveMockBackend;
+//# sourceMappingURL=memory-mock.d.ts.map

package/dist/eval/memory-mock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-mock.d.ts","sourceRoot":"","sources":["../../src/eval/memory-mock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAY,MAAM,qBAAqB,CAAC;AAKlG,8DAA8D;AAC9D,MAAM,WAAW,aAAa;IAC5B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,0BAA2B,YAAW,YAAY;IAOjD,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,cAAc,WAAoB;IAE3C,OAAO,CAAC,QAAQ,CAAwC;IACxD,OAAO,CAAC,SAAS,CAAyC;gBAE7B,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IAEjF,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAIrC,sEAAsE;IACtE,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAIjD,KAAK,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,YAAY,CAAC;IAuD5D;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAuBtB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;IAIhE,YAAY,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAIvC,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQjD,KAAK,IAAI,IAAI;CAId;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,0BAA0B,CAE7B"}

package/dist/eval/memory-mock.js

@@ -0,0 +1,129 @@
+import { seededRng } from './rng.js';
+/**
+ * Memory-sensitive mock backend for the learning-curve eval.
+ *
+ * Unlike the generic test MockBackend (fixed behavior per task description),
+ * this backend's success probability depends on whether *relevant learned
+ * knowledge* was injected for the task's skill family:
+ *
+ *   - "warm" (a prior successful experience or a playbook for this skill was
+ *     surfaced) → high success probability, fewer steps.
+ *   - "cold" (nothing relevant surfaced) → low success probability, more steps.
+ *
+ * This lets the harness validate its own machinery: with learning enabled the
+ * system accumulates experiences and the success curve must bend upward; with
+ * learning disabled every task stays cold and the curve stays flat. The draw is
+ * deterministic per task id (seeded), so the two conditions form a paired
+ * comparison — the only thing that differs is warm vs. cold, not luck.
+ */
+export class MemorySensitiveMockBackend {
+    opts;
+    name = 'mock';
+    supportedTypes = ['mock', 'test'];
+    sessions = new Map();
+    telemetry = new Map();
+    constructor(opts) {
+        this.opts = opts;
+    }
+    async isAvailable() {
+        return true;
+    }
+    /** Read back what happened for a given task id (set during spawn). */
+    getTelemetry(taskId) {
+        return this.telemetry.get(taskId);
+    }
+    async spawn(config) {
+        const task = config.task;
+        const skillTag = String(task.metadata?.skillTag ?? '');
+        const knowledgeSurfaced = this.hasRelevantKnowledge(config, skillTag);
+        // Paired, deterministic draw: same task id → same uniform across conditions.
+        const u = seededRng('mock-outcome', this.opts.seed, task.id)();
+        const pSuccess = knowledgeSurfaced ? this.opts.warmProb : this.opts.coldProb;
+        const success = u < pSuccess;
+        // Warm runs are more efficient; failures cost an extra step.
+        const stepCount = (knowledgeSurfaced ? 2 : 5) + (success ? 0 : 1);
+        this.telemetry.set(task.id, { knowledgeSurfaced, stepCount });
+        const sessionId = crypto.randomUUID();
+        const toolCalls = [];
+        for (let i = 0; i < stepCount; i++) {
+            toolCalls.push({
+                id: crypto.randomUUID(),
+                name: i === 0 ? 'analyze' : i === stepCount - 1 ? 'finalize' : 'edit',
+                input: { step: i, skill: skillTag },
+                output: success && i === stepCount - 1 ? 'ok' : 'working',
+                startTime: new Date(),
+                endTime: new Date(),
+            });
+        }
+        const session = {
+            id: sessionId,
+            agentType: config.agentType,
+            task,
+            state: success ? 'completed' : 'failed',
+            messages: [
+                {
+                    role: 'assistant',
+                    content: success
+                        ? `Solved [skill:${skillTag}] using ${knowledgeSurfaced ? 'recalled' : 'fresh'} approach.`
+                        : `Failed to solve [skill:${skillTag}] (${knowledgeSurfaced ? 'warm' : 'cold'} attempt).`,
+                    timestamp: new Date(),
+                },
+            ],
+            toolCalls,
+            startTime: new Date(),
+            endTime: new Date(),
+            result: success ? `Solution for [skill:${skillTag}]` : undefined,
+            error: success ? undefined : 'mock cold failure',
+            metadata: { knowledgeSurfaced },
+        };
+        this.sessions.set(sessionId, session);
+        return session;
+    }
+    /**
+     * A task is "warm" if the injected knowledge contains either a prior
+     * successful experience for this skill, or a playbook mentioning it.
+     */
+    hasRelevantKnowledge(config, skillTag) {
+        if (!skillTag)
+            return false;
+        const token = `[skill:${skillTag}]`;
+        const knowledge = config.injectedKnowledge;
+        if (!knowledge)
+            return false;
+        const experiences = (knowledge.experiences ?? []);
+        for (const { experience } of experiences) {
+            if (experience.success && experience.taskInput.includes(token)) {
+                return true;
+            }
+        }
+        const playbooks = (knowledge.playbooks ?? []);
+        for (const match of playbooks) {
+            if (JSON.stringify(match.playbook).includes(skillTag)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    async getSession(sessionId) {
+        return this.sessions.get(sessionId);
+    }
+    async listSessions() {
+        return Array.from(this.sessions.values());
+    }
+    async terminate(sessionId) {
+        const session = this.sessions.get(sessionId);
+        if (!session)
+            return;
+        session.state = 'failed';
+        session.error = 'Terminated';
+        session.endTime = new Date();
+    }
+    clear() {
+        this.sessions.clear();
+        this.telemetry.clear();
+    }
+}
+export function createMemorySensitiveMock(opts) {
+    return new MemorySensitiveMockBackend(opts);
+}
+//# sourceMappingURL=memory-mock.js.map

package/dist/eval/memory-mock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-mock.js","sourceRoot":"","sources":["../../src/eval/memory-mock.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAQrC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,OAAO,0BAA0B;IAOR;IANpB,IAAI,GAAG,MAAM,CAAC;IACd,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnC,QAAQ,GAA8B,IAAI,GAAG,EAAE,CAAC;IAChD,SAAS,GAA+B,IAAI,GAAG,EAAE,CAAC;IAE1D,YAA6B,IAA0D;QAA1D,SAAI,GAAJ,IAAI,CAAsD;IAAG,CAAC;IAE3F,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sEAAsE;IACtE,YAAY,CAAC,MAAc;QACzB,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAwB;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC;QAEvD,MAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAEtE,6EAA6E;QAC7E,MAAM,CAAC,GAAG,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/D,MAAM,QAAQ,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;QAC7E,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC;QAE7B,6DAA6D;QAC7D,MAAM,SAAS,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAElE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC,CAAC;QAE9D,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACtC,MAAM,SAAS,GAAe,EAAE,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC;gBACb,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;gBACvB,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM;gBACrE,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACnC,MAAM,EAAE,OAAO,IAAI,CAAC,KAAK,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;gBACzD,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,IAAI,IAAI,EAAE;aACpB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAiB;YAC5B,EAAE,EAAE,SAAS;YACb,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,IAAI;YACJ,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ;YACvC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,WAAW;oBACjB,OAAO,EAAE,OAAO;wBACd,CAAC,CAAC,iBAAiB,QAAQ,WAAW,iBAAiB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,YAAY;wBAC1F,CAAC,CAAC,0BAA0B,QAAQ,MAAM,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,YAAY;oBAC3F,SAAS,EAAE,IAAI,IAAI,EAAE;iBACtB;aACF;YACD,SAAS;YACT,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,OAAO,EAAE,IAAI,IAAI,EAAE;YACnB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,uBAAuB,QAAQ,GAAG,CAAC,CAAC,CAAC,SAAS;YAChE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB;YAChD,QAAQ,EAAE,EAAE,iBAAiB,EAAE;SAChC,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,oBAAoB,CAAC,MAAwB,EAAE,QAAgB;QACrE,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5B,MAAM,KAAK,GAAG,UAAU,QAAQ,GAAG,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAC3C,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QAE7B,MAAM,WAAW,GAAG,CAAC,SAAS,CAAC,WAAW,IAAI,EAAE,CAAsC,CAAC;QACvF,KAAK,MAAM,EAAE,UAAU,EAAE,IAAI,WAAW,EAAE,CAAC;YACzC,IAAI,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/D,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,CAAoB,CAAC;QACjE,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACtD,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,SAAiB;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC;QACzB,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC;QAC7B,OAAO,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACF;AAED,MAAM,UAAU,yBAAyB,CAAC,IAIzC;IACC,OAAO,IAAI,0BAA0B,CAAC,IAAI,CAAC,CAAC;AAC9C,CAAC"}

package/dist/eval/memory-qa/index.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Memory-QA benchmark adapter (LongMemEval, LoCoMo) — ingest-then-QA memory
+ * benchmarks that test the serving (retrieval) layer with headroom.
+ * See docs/DESIGN-memory-qa-benchmarks.md.
+ */
+export * from './types.js';
+export * from './loaders.js';
+export * from './retrieval.js';
+export * from './qa.js';
+export * from './serving.js';
+export * from './iterative.js';
+export * from './metrics.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/memory-qa/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/eval/memory-qa/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC"}

package/dist/eval/memory-qa/index.js

@@ -0,0 +1,13 @@
+/**
+ * Memory-QA benchmark adapter (LongMemEval, LoCoMo) — ingest-then-QA memory
+ * benchmarks that test the serving (retrieval) layer with headroom.
+ * See docs/DESIGN-memory-qa-benchmarks.md.
+ */
+export * from './types.js';
+export * from './loaders.js';
+export * from './retrieval.js';
+export * from './qa.js';
+export * from './serving.js';
+export * from './iterative.js';
+export * from './metrics.js';
+//# sourceMappingURL=index.js.map

package/dist/eval/memory-qa/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/eval/memory-qa/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,SAAS,CAAC;AACxB,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC"}

package/dist/eval/memory-qa/iterative.d.ts

@@ -0,0 +1,80 @@
+import type { SearchProvider } from '../../memory/search-provider.js';
+import type { KnowledgeNote } from '../../types/knowledge.js';
+import type { MemQuestion } from './types.js';
+/** LLM completion callback. Canonical home (qa.ts re-exports it) — defined
+ * here so iterative.ts does not import qa.ts back (qa imports iterative). */
+export type CompleteFn = (prompt: string) => Promise<string>;
+/**
+ * MVP iterative reflective-retrieval controller — a minimal MemR3-style closed
+ * loop over cognitive-core's existing {@link SearchProvider} (hybrid retrieval).
+ *
+ *   Retrieve → Reflect (evidence-gap) → {Retrieve refined | Answer}
+ *
+ * The hypothesis (docs/DESIGN-memory-qa-benchmarks.md §4i): single-shot top-k
+ * misses evidence that isn't semantically near the *raw* question (e.g. temporal
+ * trailed in §4g because the date-bearing turn wasn't in top-k). Reflecting on
+ * what's still missing and refining the query — while MASKING already-seen notes
+ * so each round surfaces new evidence — should recover those misses more
+ * surgically than brute-forcing a larger k. Unlike lever-1's LLM re-rank (§4g,
+ * net-negative), reflection is ADDITIVE: it steers the next query, it never
+ * discards retrieved evidence.
+ *
+ * This is the MVP (mask + reflect + refine + early-stop, max_iter=3). The full
+ * MemR3 port (cross-encoder rerank, router, 5 iters) and MGRetrieval's
+ * structured-keyword guidance are a sequenced follow-up.
+ */
+export interface IterativeConfig {
+    /** Max retrieval rounds (reflection runs between rounds). Default 3. */
+    maxIter?: number;
+    /** New notes surfaced per round (already-seen notes are masked). Default 8. */
+    perIterK?: number;
+    /**
+     * Conservative sufficiency gate (default false). The reflection only requests
+     * another round when a SPECIFIC named fact is clearly absent — defaulting to
+     * "sufficient" otherwise. Curbs the distractor over-retrieval behind the n=200
+     * regressions (§4i): fewer refinement rounds → fewer chances to pull a
+     * conflicting date/entity that derails a correct first-round hit.
+     */
+    conservative?: boolean;
+    /**
+     * MGRetrieval-style keyword guidance (default false). On a gap, the reflector
+     * emits structured KEYWORDS for the missing slot (entity/attribute/date) and
+     * the next round retrieves those keywords — more targeted than re-querying with
+     * a free-form sentence (the free-form append is what surfaced distractors).
+     */
+    keywordGuided?: boolean;
+}
+export interface IterStep {
+    /** Query used for THIS round's retrieval (raw question, then refined). */
+    query: string;
+    /** Note ids newly surfaced this round (after masking). */
+    newNoteIds: string[];
+    /** Reflection verdict after this round; null on the final round (no reflect call). */
+    sufficient: boolean | null;
+    /** What the reflector said is still missing (diagnosis). */
+    missing: string;
+}
+export interface IterativeResult {
+    answer: string;
+    excerpts: string[];
+    /** Number of retrieval rounds actually run (≤ maxIter). */
+    iterations: number;
+    trace: IterStep[];
+}
+interface Reflection {
+    sufficient: boolean;
+    missing: string;
+    refinedQuery: string;
+    /** MGRetrieval keyword guidance: keywords pinpointing the missing slot. */
+    keywords: string[];
+}
+/** Parse the reflection JSON; tolerate prose around it and degrade safely. */
+export declare function parseReflection(raw: string): Reflection;
+/**
+ * Run the iterative retrieve→reflect→answer loop for one question. Retrieval and
+ * the final answer use the SAME provider/prompt as the single-shot path, so an
+ * A/B isolates the loop itself (see runMemoryQA `iterative` option).
+ */
+export declare function runIterativeRetrieval(question: MemQuestion, notes: KnowledgeNote[], noteById: Map<string, KnowledgeNote>, retriever: SearchProvider, complete: CompleteFn, buildAnswerPrompt: (q: MemQuestion, excerpts: string[]) => string, opts?: IterativeConfig): Promise<IterativeResult>;
+export {};
+//# sourceMappingURL=iterative.d.ts.map

package/dist/eval/memory-qa/iterative.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterative.d.ts","sourceRoot":"","sources":["../../../src/eval/memory-qa/iterative.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;6EAC6E;AAC7E,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAE7D;;;;;;;;;;;;;;;;;;GAkBG;AAEH,MAAM,WAAW,eAAe;IAC9B,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,sFAAsF;IACtF,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAED,UAAU,UAAU;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAkCD,8EAA8E;AAC9E,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAqBvD;AAED;;;;GAIG;AACH,wBAAsB,qBAAqB,CACzC,QAAQ,EAAE,WAAW,EACrB,KAAK,EAAE,aAAa,EAAE,EACtB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,EACpC,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,UAAU,EACpB,iBAAiB,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,MAAM,EACjE,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,eAAe,CAAC,CAuD1B"}

package/dist/eval/memory-qa/iterative.js

@@ -0,0 +1,107 @@
+/** Evidence-gap reflection: is what we have enough, and if not, what to search next? */
+function buildReflectPrompt(question, evidence, opts = {}) {
+    const numbered = evidence.map((e, i) => `${i + 1}. ${e}`).join('\n');
+    // Conservative gate: high bar to spend another round (prefer the evidence in hand).
+    const gateLine = opts.conservative
+        ? 'Decide whether the evidence is sufficient. Set sufficient=TRUE unless a SPECIFIC named fact required to answer is clearly ABSENT (do NOT request more just to be thorough — extra retrieval risks surfacing conflicting/distractor evidence).'
+        : 'Decide whether the evidence is sufficient to answer, and if not, what is still missing.';
+    // Keyword guidance: emit targeted keywords for the missing slot, not a sentence.
+    const refineLine = opts.keywordGuided
+        ? 'If NOT sufficient, list 1–4 KEYWORDS (entities, attributes, dates) that pinpoint the missing fact — not a sentence.'
+        : 'If NOT sufficient, give a short search query for the missing evidence.';
+    const jsonLine = opts.keywordGuided
+        ? '{"sufficient": true|false, "missing": "<what is still needed, or empty>", "keywords": ["<keyword>", ...]}'
+        : '{"sufficient": true|false, "missing": "<what is still needed, or empty>", "refined_query": "<short search query, or empty>"}';
+    return [
+        'You are gathering evidence from a past conversation to answer a question.',
+        gateLine,
+        refineLine,
+        '',
+        `Question: ${question}`,
+        '',
+        `Evidence so far:\n${numbered || '(none yet)'}`,
+        '',
+        'Reply with ONLY a compact JSON object, nothing else:',
+        jsonLine,
+    ].join('\n');
+}
+/** Parse the reflection JSON; tolerate prose around it and degrade safely. */
+export function parseReflection(raw) {
+    const m = raw.match(/\{[\s\S]*\}/);
+    if (m) {
+        try {
+            const o = JSON.parse(m[0]);
+            const kw = Array.isArray(o.keywords)
+                ? o.keywords.map((x) => String(x)).filter(Boolean)
+                : [];
+            return {
+                sufficient: o.sufficient === true || /^(true|yes)$/i.test(String(o.sufficient ?? '')),
+                missing: String(o.missing ?? ''),
+                refinedQuery: String(o.refined_query ?? o.refinedQuery ?? ''),
+                keywords: kw,
+            };
+        }
+        catch {
+            // fall through to the safe default
+        }
+    }
+    // Unparseable → "not sufficient, no refinement": the loop just takes the next
+    // top-k, degrading gracefully toward single-shot rather than erroring.
+    return { sufficient: false, missing: '', refinedQuery: '', keywords: [] };
+}
+/**
+ * Run the iterative retrieve→reflect→answer loop for one question. Retrieval and
+ * the final answer use the SAME provider/prompt as the single-shot path, so an
+ * A/B isolates the loop itself (see runMemoryQA `iterative` option).
+ */
+export async function runIterativeRetrieval(question, notes, noteById, retriever, complete, buildAnswerPrompt, opts = {}) {
+    const maxIter = opts.maxIter ?? 3;
+    const perIterK = opts.perIterK ?? 8;
+    const masked = new Set();
+    const excerpts = [];
+    const trace = [];
+    let query = question.question;
+    for (let round = 0; round < maxIter; round++) {
+        // Over-fetch by the masked count so that after filtering we still get ~perIterK NEW notes.
+        const hits = await retriever.search(query, notes, { maxResults: perIterK + masked.size });
+        const fresh = hits.filter((h) => !masked.has(h.noteId)).slice(0, perIterK);
+        if (fresh.length === 0) {
+            trace.push({ query, newNoteIds: [], sufficient: null, missing: '' });
+            break; // retrieval exhausted — nothing new to add
+        }
+        const newNoteIds = [];
+        for (const h of fresh) {
+            masked.add(h.noteId);
+            newNoteIds.push(h.noteId);
+            const body = noteById.get(h.noteId)?.body;
+            if (body)
+                excerpts.push(body);
+        }
+        // Final round: skip reflection (we won't retrieve again).
+        if (round === maxIter - 1) {
+            trace.push({ query, newNoteIds, sufficient: null, missing: '' });
+            break;
+        }
+        const refl = parseReflection(await complete(buildReflectPrompt(question.question, excerpts, {
+            conservative: opts.conservative,
+            keywordGuided: opts.keywordGuided,
+        })));
+        trace.push({ query, newNoteIds, sufficient: refl.sufficient, missing: refl.missing });
+        if (refl.sufficient)
+            break;
+        if (opts.keywordGuided && refl.keywords.length > 0) {
+            // MGRetrieval-style: ADD the missing-slot keywords to the question (guide,
+            // don't replace). An earlier variant that dropped the question
+            // (`keywords.join(' ')`) retrieved keyword-matching but contextually-WRONG
+            // turns and was net-negative (§4i / EXP-07) — keep the question's signal.
+            query = `${question.question} ${refl.keywords.join(' ')}`;
+        }
+        else {
+            // MemR3-style: append the free-form sub-query to the original question.
+            query = refl.refinedQuery ? `${question.question}. ${refl.refinedQuery}` : question.question;
+        }
+    }
+    const answer = (await complete(buildAnswerPrompt(question, excerpts))).trim();
+    return { answer, excerpts, iterations: trace.length, trace };
+}
+//# sourceMappingURL=iterative.js.map

package/dist/eval/memory-qa/iterative.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterative.js","sourceRoot":"","sources":["../../../src/eval/memory-qa/iterative.ts"],"names":[],"mappings":"AA6EA,wFAAwF;AACxF,SAAS,kBAAkB,CACzB,QAAgB,EAChB,QAAkB,EAClB,OAA4D,EAAE;IAE9D,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,oFAAoF;IACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY;QAChC,CAAC,CAAC,+OAA+O;QACjP,CAAC,CAAC,yFAAyF,CAAC;IAC9F,iFAAiF;IACjF,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa;QACnC,CAAC,CAAC,qHAAqH;QACvH,CAAC,CAAC,wEAAwE,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa;QACjC,CAAC,CAAC,2GAA2G;QAC7G,CAAC,CAAC,8HAA8H,CAAC;IACnI,OAAO;QACL,2EAA2E;QAC3E,QAAQ;QACR,UAAU;QACV,EAAE;QACF,aAAa,QAAQ,EAAE;QACvB,EAAE;QACF,qBAAqB,QAAQ,IAAI,YAAY,EAAE;QAC/C,EAAE;QACF,sDAAsD;QACtD,QAAQ;KACT,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,CAAC;QACN,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC;YACtD,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAClC,CAAC,CAAE,CAAC,CAAC,QAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBACjE,CAAC,CAAC,EAAE,CAAC;YACP,OAAO;gBACL,UAAU,EAAE,CAAC,CAAC,UAAU,KAAK,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;gBACrF,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;gBAChC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC;gBAC7D,QAAQ,EAAE,EAAE;aACb,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,mCAAmC;QACrC,CAAC;IACH,CAAC;IACD,8EAA8E;IAC9E,uEAAuE;IACvE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AAC5E,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,QAAqB,EACrB,KAAsB,EACtB,QAAoC,EACpC,SAAyB,EACzB,QAAoB,EACpB,iBAAiE,EACjE,OAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;IAEpC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC;IAE9B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC;QAC7C,2FAA2F;QAC3F,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1F,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC3E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,2CAA2C;QACpD,CAAC;QACD,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACrB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC;YAC1C,IAAI,IAAI;gBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC;QAED,0DAA0D;QAC1D,IAAI,KAAK,KAAK,OAAO,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACjE,MAAM;QACR,CAAC;QAED,MAAM,IAAI,GAAG,eAAe,CAC1B,MAAM,QAAQ,CACZ,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE;YAC9C,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC,CACH,CACF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACtF,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM;QAC3B,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,2EAA2E;YAC3E,+DAA+D;YAC/D,2EAA2E;YAC3E,0EAA0E;YAC1E,KAAK,GAAG,GAAG,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,wEAAwE;YACxE,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC/F,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,MAAM,QAAQ,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9E,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;AAC/D,CAAC"}

package/dist/eval/memory-qa/loaders.d.ts

@@ -0,0 +1,11 @@
+import { instanceToDocuments, loadLocomo, loadLongMemEval, scopedTurnId } from 'swarmkit-eval';
+import type { KnowledgeNote } from '../../types/knowledge.js';
+import type { MemQAInstance } from './types.js';
+export { loadLocomo, loadLongMemEval, scopedTurnId, instanceToDocuments };
+/**
+ * Convert one instance's conversation history into cognitive-core knowledge
+ * notes. The benchmark parsing and store-neutral document shape live in
+ * swarmkit-eval; this wrapper is the cognitive-core-specific ingest step.
+ */
+export declare function instanceToNotes(instance: MemQAInstance): KnowledgeNote[];
+//# sourceMappingURL=loaders.d.ts.map

package/dist/eval/memory-qa/loaders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loaders.d.ts","sourceRoot":"","sources":["../../../src/eval/memory-qa/loaders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC/F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY,EAAE,mBAAmB,EAAE,CAAC;AAE1E;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,aAAa,GAAG,aAAa,EAAE,CAaxE"}

package/dist/eval/memory-qa/loaders.js

@@ -0,0 +1,21 @@
+import { instanceToDocuments, loadLocomo, loadLongMemEval, scopedTurnId } from 'swarmkit-eval';
+import { createObservation } from '../../types/knowledge.js';
+export { loadLocomo, loadLongMemEval, scopedTurnId, instanceToDocuments };
+/**
+ * Convert one instance's conversation history into cognitive-core knowledge
+ * notes. The benchmark parsing and store-neutral document shape live in
+ * swarmkit-eval; this wrapper is the cognitive-core-specific ingest step.
+ */
+export function instanceToNotes(instance) {
+    return instanceToDocuments(instance).map((doc) => createObservation({
+        id: doc.id,
+        title: doc.turnId,
+        body: doc.text,
+        domain: [doc.instanceId],
+        entities: [],
+        tags: [doc.sessionId],
+        confidence: 0.8,
+        source: { origin: 'imported' },
+    }));
+}
+//# sourceMappingURL=loaders.js.map