@saihm/shm 0.0.0-stage → 0.38.1

package/dist/adapter/markdown_dir_port.js

@@ -0,0 +1,175 @@
+import { readdir, readFile, stat } from "node:fs/promises";
+import { extname, join } from "node:path";
+import { describeDocument } from "./frontmatter.js";
+import { hourlyEpoch } from "./epoch.js";
+const DEFAULTS = {
+    extensions: [".md"],
+    labelChars: 120,
+    maxFileBytes: 1_048_576,
+    recallCap: 200,
+    tier: "file",
+};
+function posInt(v, dflt) {
+    return Number.isFinite(v) && v >= 1 ? Math.floor(v) : dflt;
+}
+export class MarkdownDirPort {
+    tenantId;
+    dir;
+    extensions;
+    exclude;
+    labelChars;
+    maxFileBytes;
+    recallCap;
+    docs = [];
+    stampMap = new Map();
+    skippedTooLarge = 0;
+    skippedVanished = 0;
+    datedCount = 0;
+    constructor(opts) {
+        this.dir = opts.dir;
+        this.tenantId = opts.tenantId ?? "local";
+        this.extensions = (opts.extensions ?? DEFAULTS.extensions).map((e) => e.toLowerCase());
+        this.exclude = new Set(opts.exclude ?? []);
+        this.labelChars = posInt(opts.labelChars, DEFAULTS.labelChars);
+        this.maxFileBytes = posInt(opts.maxFileBytes, DEFAULTS.maxFileBytes);
+        this.recallCap = posInt(opts.recallCap, DEFAULTS.recallCap);
+    }
+    stampOf = (cellId) => this.stampMap.get(cellId) ?? {};
+    stamps() {
+        return this.stampMap;
+    }
+    get stats() {
+        let stamped = 0;
+        for (const s of this.stampMap.values()) {
+            if (s.thread !== undefined || s.status !== undefined || s.kind !== undefined)
+                stamped++;
+        }
+        return {
+            files: this.docs.length,
+            skippedTooLarge: this.skippedTooLarge,
+            skippedVanished: this.skippedVanished,
+            stamped,
+            dated: this.datedCount,
+        };
+    }
+    async readIndex() {
+        const entries = await readdir(this.dir, { withFileTypes: true });
+        const names = entries
+            .filter((d) => d.isFile())
+            .map((d) => d.name)
+            .filter((n) => this.extensions.includes(extname(n).toLowerCase()) && !this.exclude.has(n))
+            .sort();
+        const docs = [];
+        const stamps = new Map();
+        const seen = new Map();
+        let skipped = 0;
+        let vanished = 0;
+        let dated = 0;
+        for (const name of names) {
+            const path = join(this.dir, name);
+            let st;
+            try {
+                st = await stat(path);
+            }
+            catch (e) {
+                if (e.code === "ENOENT") {
+                    vanished++;
+                    continue;
+                }
+                throw e;
+            }
+            if (st.size > this.maxFileBytes) {
+                skipped++;
+                continue;
+            }
+            const cellId = stemOf(name);
+            const prior = seen.get(cellId);
+            if (prior !== undefined) {
+                throw new Error(`MarkdownDirPort: "${name}" and "${prior}" both map to cellId "${cellId}" in ${this.dir} — ` +
+                    `rename one, or narrow \`extensions\``);
+            }
+            seen.set(cellId, name);
+            let text;
+            try {
+                text = await readFile(path, "utf8");
+            }
+            catch (e) {
+                if (e.code === "ENOENT") {
+                    seen.delete(cellId);
+                    vanished++;
+                    continue;
+                }
+                throw e;
+            }
+            const { label, stamp, body, epoch: authored } = describeDocument(text, this.labelChars);
+            if (authored !== undefined)
+                dated++;
+            docs.push({
+                cellId,
+                file: name,
+                label,
+                body: text,
+                searchLower: `${label}\n${body}`.toLowerCase(),
+                epoch: authored ?? hourlyEpoch(st.mtimeMs),
+                byteSize: Buffer.byteLength(text, "utf8"),
+            });
+            stamps.set(cellId, stamp);
+        }
+        this.docs = docs;
+        this.stampMap = stamps;
+        this.skippedTooLarge = skipped;
+        this.skippedVanished = vanished;
+        this.datedCount = dated;
+        return docs.map((d) => ({
+            cellId: d.cellId,
+            label: d.label,
+            epoch: d.epoch,
+            byteSize: d.byteSize,
+            tier: DEFAULTS.tier,
+            forgotten: false,
+        }));
+    }
+    async recallByQuery(query) {
+        const q = query.toLowerCase().trim();
+        if (q === "")
+            return [];
+        const out = [];
+        for (const d of this.docs) {
+            if (out.length >= this.recallCap)
+                break;
+            if (d.searchLower.includes(q)) {
+                out.push({ cellId: d.cellId, plaintext: d.body, epoch: d.epoch, tier: DEFAULTS.tier });
+            }
+        }
+        return out;
+    }
+}
+function stemOf(name) {
+    const ext = extname(name);
+    return ext === "" ? name : name.slice(0, -ext.length);
+}
+export { hourlyEpoch } from "./epoch.js";
+export function federatedStampOf(sources, opts = {}) {
+    const sep = opts.separator ?? "::";
+    if (sep.length === 0)
+        throw new Error("federatedStampOf: separator must be non-empty");
+    const seen = new Set();
+    for (const s of sources) {
+        if (s.tag.length === 0)
+            throw new Error("federatedStampOf: source tag must be non-empty");
+        if (s.tag.includes(sep))
+            throw new Error(`federatedStampOf: source tag "${s.tag}" must not contain "${sep}"`);
+        if (seen.has(s.tag))
+            throw new Error(`federatedStampOf: duplicate source tag "${s.tag}"`);
+        seen.add(s.tag);
+    }
+    const prefixed = sources.map((s) => ({ prefix: `${s.tag}${sep}`, port: s.port }));
+    return (cellId) => {
+        for (const p of prefixed) {
+            if (cellId.startsWith(p.prefix))
+                return p.port.stamps().get(cellId.slice(p.prefix.length)) ?? {};
+        }
+        return {};
+    };
+}
+//# sourceMappingURL=markdown_dir_port.js.map

package/dist/adapter/markdown_dir_port.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown_dir_port.js","sourceRoot":"","sources":["../../src/adapter/markdown_dir_port.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,QAAQ,GAAG;IACf,UAAU,EAAE,CAAC,KAAK,CAAsB;IACxC,UAAU,EAAE,GAAG;IACf,YAAY,EAAE,SAAS;IACvB,SAAS,EAAE,GAAG;IACd,IAAI,EAAE,MAAM;CACJ,CAAC;AAEX,SAAS,MAAM,CAAC,CAAqB,EAAE,IAAY;IACjD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAK,CAAY,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACnF,CAAC;AAgCD,MAAM,OAAO,eAAe;IACjB,QAAQ,CAAS;IACT,GAAG,CAAS;IACZ,UAAU,CAAoB;IAC9B,OAAO,CAAsB;IAC7B,UAAU,CAAS;IACnB,YAAY,CAAS;IACrB,SAAS,CAAS;IAC3B,IAAI,GAAmB,EAAE,CAAC;IAC1B,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IAC7C,eAAe,GAAG,CAAC,CAAC;IACpB,eAAe,GAAG,CAAC,CAAC;IACpB,UAAU,GAAG,CAAC,CAAC;IAEvB,YAAY,IAA4B;QACtC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC/D,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;QACrE,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC;IAEQ,OAAO,GAAgB,CAAC,MAAc,EAAkB,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAEpG,MAAM;QACJ,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,IAAI,KAAK;QACP,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACvC,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,EAAE,CAAC;QAC1F,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;YACvB,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,OAAO;YACP,KAAK,EAAE,IAAI,CAAC,UAAU;SACvB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,OAAO;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACzB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;aACzF,IAAI,EAAE,CAAC;QAEV,MAAM,IAAI,GAAU,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;QACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;QACvC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAClC,IAAI,EAAE,CAAC;YACP,IAAI,CAAC;gBACH,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACnD,QAAQ,EAAE,CAAC;oBACX,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,CAAC;YACV,CAAC;YACD,IAAI,EAAE,CAAC,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChC,OAAO,EAAE,CAAC;gBACV,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,qBAAqB,IAAI,UAAU,KAAK,yBAAyB,MAAM,QAAQ,IAAI,CAAC,GAAG,KAAK;oBAC1F,sCAAsC,CACzC,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACvB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACtC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACnD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACpB,QAAQ,EAAE,CAAC;oBACX,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,CAAC;YACV,CAAC;YACD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACxF,IAAI,QAAQ,KAAK,SAAS;gBAAE,KAAK,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC;gBACR,MAAM;gBACN,IAAI,EAAE,IAAI;gBACV,KAAK;gBACL,IAAI,EAAE,IAAI;gBACV,WAAW,EAAE,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC9C,KAAK,EAAE,QAAQ,IAAI,WAAW,CAAC,EAAE,CAAC,OAAO,CAAC;gBAC1C,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC;aAC1C,CAAC,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC;QACvB,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAC/B,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAExB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,MAAM,CAAC,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAmB,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;gBAAE,MAAM;YACxC,IAAI,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YACzF,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED,SAAS,MAAM,CAAC,IAAY;IAC1B,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAOzC,MAAM,UAAU,gBAAgB,CAC9B,OAAiC,EACjC,OAAwC,EAAE;IAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC;IACnC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC1F,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,GAAG,uBAAuB,GAAG,GAAG,CAAC,CAAC;QAC9G,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1F,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClF,OAAO,CAAC,MAAc,EAAkB,EAAE;QACxC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;gBAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACnG,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;AACJ,CAAC"}

package/dist/aggregate/document_aggregation.d.ts

@@ -0,0 +1,40 @@
+import type { RankedSource } from "../rerank/reranking_retriever.js";
+export type DocumentLookup = (cellId: string) => string | undefined;
+export interface AggregationConfig {
+    readonly poolK: number;
+    readonly anchorJ: number;
+    readonly alpha: number;
+    readonly rrfK: number;
+}
+export declare const DEFAULT_AGGREGATION: AggregationConfig;
+export interface CalibrationSample {
+    readonly pool: readonly string[];
+    readonly isRelevant: readonly boolean[];
+    readonly documentOf: DocumentLookup;
+}
+export interface CalibrationResult {
+    readonly anchorJ: number | undefined;
+    readonly sweep: readonly {
+        readonly anchorJ: number;
+        readonly dHit: number;
+        readonly dPrecision: number;
+    }[];
+    readonly samples: number;
+    readonly vacuous: boolean;
+}
+export declare function calibrateAnchor(samples: readonly CalibrationSample[], opts?: {
+    readonly k?: number;
+    readonly maxHitDrop?: number;
+    readonly cfg?: Partial<AggregationConfig>;
+}): CalibrationResult;
+export declare function normalizeAggregation(cfg: Partial<AggregationConfig>): AggregationConfig;
+export declare function aggregateOrder(pool: readonly string[], documentOf: DocumentLookup, cfg: AggregationConfig): string[];
+export declare class DocumentAggregatingRetriever implements RankedSource {
+    private readonly base;
+    private readonly documentOf;
+    readonly name: string;
+    private readonly cfg;
+    constructor(base: RankedSource, documentOf: DocumentLookup, cfg?: Partial<AggregationConfig>);
+    retrieve(query: string, k: number): Promise<string[]>;
+}
+//# sourceMappingURL=document_aggregation.d.ts.map

package/dist/aggregate/document_aggregation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document_aggregation.d.ts","sourceRoot":"","sources":["../../src/aggregate/document_aggregation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAErE,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;AAEpE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,eAAO,MAAM,mBAAmB,EAAE,iBAAoE,CAAC;AAEvG,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,SAAS,OAAO,EAAE,CAAC;IACxC,QAAQ,CAAC,UAAU,EAAE,cAAc,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,SAAS;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5G,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,SAAS,iBAAiB,EAAE,EACrC,IAAI,GAAE;IAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAA;CAAO,GAC1G,iBAAiB,CAqCnB;AAED,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,iBAAiB,CAWvF;AAED,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,UAAU,EAAE,cAAc,EAC1B,GAAG,EAAE,iBAAiB,GACrB,MAAM,EAAE,CAiBV;AAED,qBAAa,4BAA6B,YAAW,YAAY;IAK7D,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAL7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;gBAGrB,IAAI,EAAE,YAAY,EAClB,UAAU,EAAE,cAAc,EAC3C,GAAG,GAAE,OAAO,CAAC,iBAAiB,CAAM;IAehC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAO5D"}

package/dist/aggregate/document_aggregation.js

@@ -0,0 +1,101 @@
+export const DEFAULT_AGGREGATION = { poolK: 30, anchorJ: 8, alpha: 0.25, rrfK: 60 };
+export function calibrateAnchor(samples, opts = {}) {
+    const k = Number.isFinite(opts.k) && opts.k >= 1 ? Math.floor(opts.k) : 10;
+    const maxHitDrop = Number.isFinite(opts.maxHitDrop) ? opts.maxHitDrop : -0.02;
+    const cfg = normalizeAggregation(opts.cfg ?? {});
+    const usable = samples.filter((s) => s.pool.length > 1 && s.pool.length === s.isRelevant.length);
+    const sweep = [];
+    if (usable.length === 0)
+        return { anchorJ: undefined, sweep, samples: 0, vacuous: true };
+    const vacuous = usable.every((s) => {
+        const seen = new Set();
+        return !s.pool.some((id) => { const d = s.documentOf(id); if (d === undefined)
+            return false; if (seen.has(d))
+            return true; seen.add(d); return false; });
+    });
+    const rel = new Map();
+    for (const s of usable) {
+        const m = new Map();
+        s.pool.forEach((id, i) => { if (s.isRelevant[i])
+            m.set(id, true); });
+        rel.set(s, (id) => m.get(id) === true);
+    }
+    const measure = (order) => {
+        let hit = 0, prec = 0;
+        for (const s of usable) {
+            const top = order(s).slice(0, k);
+            const n = top.filter(rel.get(s)).length;
+            hit += n > 0 ? 1 : 0;
+            prec += top.length ? n / top.length : 0;
+        }
+        return { hit: hit / usable.length, prec: prec / usable.length };
+    };
+    const base = measure((s) => s.pool);
+    let chosen;
+    for (let j = 1; j < cfg.poolK; j++) {
+        const m = measure((s) => aggregateOrder(s.pool, s.documentOf, { ...cfg, anchorJ: j }));
+        sweep.push({ anchorJ: j, dHit: m.hit - base.hit, dPrecision: m.prec - base.prec });
+        if (chosen === undefined && m.hit - base.hit >= maxHitDrop)
+            chosen = j;
+    }
+    return { anchorJ: vacuous ? undefined : chosen, sweep, samples: usable.length, vacuous };
+}
+export function normalizeAggregation(cfg) {
+    const num = (v, min, fallback, floor = true) => {
+        if (typeof v !== "number" || !Number.isFinite(v) || v < min)
+            return fallback;
+        return floor ? Math.floor(v) : v;
+    };
+    return {
+        poolK: num(cfg.poolK, 1, DEFAULT_AGGREGATION.poolK),
+        anchorJ: num(cfg.anchorJ, 0, DEFAULT_AGGREGATION.anchorJ),
+        alpha: num(cfg.alpha, 0, DEFAULT_AGGREGATION.alpha, false),
+        rrfK: num(cfg.rrfK, 1, DEFAULT_AGGREGATION.rrfK),
+    };
+}
+export function aggregateOrder(pool, documentOf, cfg) {
+    const n = pool.length;
+    if (n <= 1)
+        return [...pool];
+    const c = normalizeAggregation(cfg);
+    const j = Math.min(c.anchorJ, n);
+    if (j >= n)
+        return [...pool];
+    const w = (i) => 1 / (c.rrfK + i);
+    const key = pool.map((id, i) => documentOf(id) ?? i);
+    const support = new Map();
+    for (let i = 0; i < n; i++)
+        support.set(key[i], (support.get(key[i]) ?? 0) + w(i));
+    const score = (i) => w(i) + c.alpha * support.get(key[i]);
+    const movable = [];
+    for (let i = j; i < n; i++)
+        movable.push(i);
+    movable.sort((a, b) => score(b) - score(a) || a - b);
+    return [...pool.slice(0, j), ...movable.map((i) => pool[i])];
+}
+export class DocumentAggregatingRetriever {
+    base;
+    documentOf;
+    name;
+    cfg;
+    constructor(base, documentOf, cfg = {}) {
+        this.base = base;
+        this.documentOf = documentOf;
+        const { poolK, anchorJ, alpha, rrfK } = normalizeAggregation(cfg);
+        if (anchorJ >= poolK) {
+            throw new RangeError(`document_aggregation: anchorJ (${anchorJ}) >= poolK (${poolK}) anchors the whole pool, ` +
+                `which disables aggregation silently. Lower anchorJ or raise poolK.`);
+        }
+        this.cfg = { poolK, anchorJ, alpha, rrfK };
+        this.name = `docagg(J=${anchorJ},a=${alpha})/${base.name}`;
+    }
+    async retrieve(query, k) {
+        const depth = Math.max(k, this.cfg.poolK);
+        const base = await this.base.retrieve(query, depth);
+        if (base.length <= 1)
+            return base;
+        const poolN = Math.min(this.cfg.poolK, base.length);
+        return [...aggregateOrder(base.slice(0, poolN), this.documentOf, this.cfg), ...base.slice(poolN)];
+    }
+}
+//# sourceMappingURL=document_aggregation.js.map

package/dist/aggregate/document_aggregation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document_aggregation.js","sourceRoot":"","sources":["../../src/aggregate/document_aggregation.ts"],"names":[],"mappings":"AAWA,MAAM,CAAC,MAAM,mBAAmB,GAAsB,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AAevG,MAAM,UAAU,eAAe,CAC7B,OAAqC,EACrC,OAAyG,EAAE;IAE3G,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAK,IAAI,CAAC,CAAY,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACjG,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,UAAqB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1F,MAAM,GAAG,GAAG,oBAAoB,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IACjG,MAAM,KAAK,GAA4D,EAAE,CAAC;IAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACzF,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3J,CAAC,CAAC,CAAC;IAEH,MAAM,GAAG,GAAG,IAAI,GAAG,EAA8C,CAAC;IAClE,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,IAAI,GAAG,EAAmB,CAAC;QACrC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,KAAkD,EAAiC,EAAE;QACpG,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACjC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,MAAM,CAAC;YACzC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAClE,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,MAA0B,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,UAAU,EAAE,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACvF,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,UAAU;YAAE,MAAM,GAAG,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,GAA+B;IAClE,MAAM,GAAG,GAAG,CAAC,CAAU,EAAE,GAAW,EAAE,QAAgB,EAAE,KAAK,GAAG,IAAI,EAAU,EAAE;QAC9E,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,QAAQ,CAAC;QAC7E,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,mBAAmB,CAAC,KAAK,CAAC;QACnD,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,EAAE,mBAAmB,CAAC,OAAO,CAAC;QACzD,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC;QAC1D,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,mBAAmB,CAAC,IAAI,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAuB,EACvB,UAA0B,EAC1B,GAAsB;IAEtB,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7B,MAAM,CAAC,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IAE7B,MAAM,CAAC,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAClD,MAAM,GAAG,GAAwB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErF,MAAM,KAAK,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,CAAE,CAAC;IAC5E,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,OAAO,4BAA4B;IAKpB;IACA;IALV,IAAI,CAAS;IACL,GAAG,CAAoB;IAExC,YACmB,IAAkB,EAClB,UAA0B,EAC3C,MAAkC,EAAE;QAFnB,SAAI,GAAJ,IAAI,CAAc;QAClB,eAAU,GAAV,UAAU,CAAgB;QAG3C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAElE,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YACrB,MAAM,IAAI,UAAU,CAClB,kCAAkC,OAAO,eAAe,KAAK,4BAA4B;gBACvF,oEAAoE,CACvE,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,GAAG,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,YAAY,OAAO,MAAM,KAAK,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,CAAS;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAClC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACpG,CAAC;CACF"}

package/dist/aggregate/index.d.ts

@@ -0,0 +1,2 @@
+export { DEFAULT_AGGREGATION, DocumentAggregatingRetriever, aggregateOrder, calibrateAnchor, normalizeAggregation, type AggregationConfig, type CalibrationSample, type CalibrationResult, type DocumentLookup, } from "./document_aggregation.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/aggregate/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/aggregate/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,4BAA4B,EAC5B,cAAc,EACd,eAAe,EACf,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,cAAc,GACpB,MAAM,2BAA2B,CAAC"}

package/dist/aggregate/index.js

@@ -0,0 +1,2 @@
+export { DEFAULT_AGGREGATION, DocumentAggregatingRetriever, aggregateOrder, calibrateAnchor, normalizeAggregation, } from "./document_aggregation.js";
+//# sourceMappingURL=index.js.map

package/dist/aggregate/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/aggregate/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,4BAA4B,EAC5B,cAAc,EACd,eAAe,EACf,oBAAoB,GAKrB,MAAM,2BAA2B,CAAC"}

package/dist/answer/answer_layer.d.ts

@@ -0,0 +1,33 @@
+import type { Scope, SelectOpts } from "../retrieval/scope.js";
+import type { AbstentionConfig, Answerer, BodySource, GroundedAnswer } from "./types.js";
+import { type TokenEstimate } from "../inject/token_estimate.js";
+import type { RankedHit } from "../retrieval/semantic.js";
+export interface ScoredHit {
+    readonly cellId: string;
+    readonly score: number;
+}
+export interface RankRetriever {
+    select(query: string, opts?: SelectOpts): Promise<string[]>;
+    selectScored?(query: string, opts?: SelectOpts): Promise<readonly ScoredHit[]>;
+    selectDetailed?(query: string, opts?: SelectOpts): Promise<readonly RankedHit[]>;
+}
+export interface AnswerLayerConfig {
+    readonly topK: number;
+    readonly abstention: AbstentionConfig;
+    readonly maxTokens?: number;
+    readonly tokenEstimate?: TokenEstimate;
+}
+export declare const DEFAULT_ANSWER_CONFIG: AnswerLayerConfig;
+export declare class AnswerLayer {
+    private readonly retriever;
+    private readonly bodies;
+    private readonly answerer;
+    private readonly cfg;
+    constructor(retriever: RankRetriever, bodies: BodySource, answerer: Answerer, cfg?: AnswerLayerConfig);
+    ask(question: string, opts?: {
+        asOfEpoch?: number;
+        scope?: Scope;
+    }): Promise<GroundedAnswer>;
+    private applyTokenCap;
+}
+//# sourceMappingURL=answer_layer.d.ts.map

package/dist/answer/answer_layer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer_layer.d.ts","sourceRoot":"","sources":["../../src/answer/answer_layer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,EAAwB,MAAM,YAAY,CAAC;AAC/G,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAKjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAE1D,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,YAAY,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC;IAC/E,cAAc,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC;CAClF;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IACtC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;CACxC;AAED,eAAO,MAAM,qBAAqB,EAAE,iBAAgE,CAAC;AAErG,qBAAa,WAAW;IAEpB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAHH,SAAS,EAAE,aAAa,EACxB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,QAAQ,EAClB,GAAG,GAAE,iBAAyC;IAG3D,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,KAAK,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,CAAC;IA0BlG,OAAO,CAAC,aAAa;CA6BtB"}

package/dist/answer/answer_layer.js

@@ -0,0 +1,75 @@
+import { estimateTokens } from "../inject/token_estimate.js";
+import { DEFAULT_ABSTENTION } from "./types.js";
+import { buildContext } from "./context.js";
+import { decideAbstention } from "./gate.js";
+export const DEFAULT_ANSWER_CONFIG = { topK: 10, abstention: DEFAULT_ABSTENTION };
+export class AnswerLayer {
+    retriever;
+    bodies;
+    answerer;
+    cfg;
+    constructor(retriever, bodies, answerer, cfg = DEFAULT_ANSWER_CONFIG) {
+        this.retriever = retriever;
+        this.bodies = bodies;
+        this.answerer = answerer;
+        this.cfg = cfg;
+    }
+    async ask(question, opts) {
+        const selOpts = {
+            maxK: this.cfg.topK,
+            ...(opts?.asOfEpoch !== undefined ? { asOfEpoch: opts.asOfEpoch } : {}),
+            ...(opts?.scope !== undefined ? { scope: opts.scope } : {}),
+        };
+        const ranked = await this.retriever.select(question, selOpts);
+        const seen = new Set();
+        const topIds = [];
+        for (const id of ranked) {
+            if (seen.has(id))
+                continue;
+            seen.add(id);
+            topIds.push(id);
+            if (topIds.length === this.cfg.topK)
+                break;
+        }
+        const fetched = await this.bodies.fetchByIds(topIds, { query: question });
+        const byId = new Map(fetched.map((b) => [b.cellId, b]));
+        const ordered = topIds.map((id) => byId.get(id)).filter((b) => !!b);
+        const capped = this.applyTokenCap(ordered);
+        const ctx = buildContext(question, capped.bodies);
+        const raw = await this.answerer.answer({ question, context: ctx });
+        const decided = decideAbstention(raw, ctx.cellIds, this.cfg.abstention);
+        return capped.audit === undefined ? decided : { ...decided, budget: capped.audit };
+    }
+    applyTokenCap(ordered) {
+        const cap = this.cfg.maxTokens;
+        if (cap === undefined || !Number.isFinite(cap) || cap < 0)
+            return { bodies: ordered };
+        const count = this.cfg.tokenEstimate ?? estimateTokens;
+        const kept = [];
+        const dropped = [];
+        let used = 0;
+        let cut = false;
+        for (const b of ordered) {
+            if (cut) {
+                dropped.push(b.cellId);
+                continue;
+            }
+            let t;
+            try {
+                t = count(b.plaintext);
+            }
+            catch {
+                t = Number.NaN;
+            }
+            if (!Number.isFinite(t) || t < 0 || used + t > cap) {
+                cut = true;
+                dropped.push(b.cellId);
+                continue;
+            }
+            used += t;
+            kept.push(b);
+        }
+        return { bodies: kept, audit: { usedTokens: used, droppedForBudget: dropped } };
+    }
+}
+//# sourceMappingURL=answer_layer.js.map

package/dist/answer/answer_layer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"answer_layer.js","sourceRoot":"","sources":["../../src/answer/answer_layer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAsB,MAAM,6BAA6B,CAAC;AAEjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAqB7C,MAAM,CAAC,MAAM,qBAAqB,GAAsB,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,kBAAkB,EAAE,CAAC;AAErG,MAAM,OAAO,WAAW;IAEH;IACA;IACA;IACA;IAJnB,YACmB,SAAwB,EACxB,MAAkB,EAClB,QAAkB,EAClB,MAAyB,qBAAqB;QAH9C,cAAS,GAAT,SAAS,CAAe;QACxB,WAAM,GAAN,MAAM,CAAY;QAClB,aAAQ,GAAR,QAAQ,CAAU;QAClB,QAAG,GAAH,GAAG,CAA2C;IAC9D,CAAC;IAEJ,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,IAA4C;QACtE,MAAM,OAAO,GAAe;YAC1B,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;YACnB,GAAG,CAAC,IAAI,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,GAAG,CAAC,IAAI,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAC3B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAChB,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI;gBAAE,MAAM;QAC7C,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC1E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAA8B,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhG,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC;QACnE,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACxE,OAAO,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;IACrF,CAAC;IAEO,aAAa,CAAC,OAAgC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;QAC/B,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QACtF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,cAAc,CAAC;QACvD,MAAM,IAAI,GAAmB,EAAE,CAAC;QAChC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,IAAI,CAAS,CAAC;YACd,IAAI,CAAC;gBACH,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC;YAAC,MAAM,CAAC;gBACP,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;YACjB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC;gBACnD,GAAG,GAAG,IAAI,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;gBACvB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,CAAC;YACV,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACf,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,EAAE,CAAC;IAClF,CAAC;CACF"}

package/dist/answer/caching_body_source.d.ts

@@ -0,0 +1,42 @@
+import type { RecalledBody } from "../saihm_client/types.js";
+import type { BodyFetchHint, BodySource, ErasableBodySource, ServeStep } from "./types.js";
+export interface CachingBodySourceConfig {
+    readonly maxEntries: number;
+    readonly onEvict?: (cellId: string) => void;
+    readonly onServed?: (step: ServeStep, ms: number) => void;
+    readonly clock?: () => number;
+}
+export declare const DEFAULT_CACHE_CONFIG: CachingBodySourceConfig;
+export interface CacheStats {
+    readonly hits: number;
+    readonly misses: number;
+    readonly evictions: number;
+    readonly entries: number;
+    readonly olderNotCached: number;
+}
+export declare class CachingBodySource implements ErasableBodySource {
+    private readonly inner;
+    private readonly cfg;
+    private fenced;
+    private readonly cache;
+    private readonly erased;
+    private readonly superseded;
+    private hits;
+    private misses;
+    private evictions;
+    private olderNotCached;
+    constructor(inner: BodySource, cfg?: CachingBodySourceConfig);
+    fetchByIds(cellIds: readonly string[], hint?: BodyFetchHint): Promise<RecalledBody[]>;
+    fence(): void;
+    forget(cellId: string): void;
+    supersede(cellId: string, seq?: string): void;
+    has(cellId: string): boolean;
+    peek(cellId: string): RecalledBody | undefined;
+    evict(cellId: string): boolean;
+    invalidate(): void;
+    get stats(): CacheStats;
+    private put;
+    private served;
+    private touch;
+}
+//# sourceMappingURL=caching_body_source.d.ts.map

package/dist/answer/caching_body_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"caching_body_source.d.ts","sourceRoot":"","sources":["../../src/answer/caching_body_source.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE3F,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IAC1D,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;CAC/B;AAED,eAAO,MAAM,oBAAoB,EAAE,uBAA8C,CAAC;AAElF,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,qBAAa,iBAAkB,YAAW,kBAAkB;IAWxD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,GAAG;IAXtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAmC;IACzD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAyB;IACpD,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,cAAc,CAAK;gBAGR,KAAK,EAAE,UAAU,EACjB,GAAG,GAAE,uBAA8C;IAGhE,UAAU,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAsC3F,KAAK,IAAI,IAAI;IAIb,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK5B,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI;IAK7C,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAK9C,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI9B,UAAU,IAAI,IAAI;IAIlB,IAAI,KAAK,IAAI,UAAU,CAEtB;IAED,OAAO,CAAC,GAAG;IAoBX,OAAO,CAAC,MAAM;IAOd,OAAO,CAAC,KAAK;CAId"}

package/dist/answer/caching_body_source.js

@@ -0,0 +1,120 @@
+import { SessionClosedError } from "../retrieval/session_closed.js";
+import { SupersedeLedger } from "./supersede.js";
+export const DEFAULT_CACHE_CONFIG = { maxEntries: 4096 };
+export class CachingBodySource {
+    inner;
+    cfg;
+    fenced = false;
+    cache = new Map();
+    erased = new Set();
+    superseded = new SupersedeLedger();
+    hits = 0;
+    misses = 0;
+    evictions = 0;
+    olderNotCached = 0;
+    constructor(inner, cfg = DEFAULT_CACHE_CONFIG) {
+        this.inner = inner;
+        this.cfg = cfg;
+    }
+    async fetchByIds(cellIds, hint) {
+        if (this.fenced)
+            throw new SessionClosedError("CachingBodySource.fetchByIds");
+        const clock = this.cfg.clock ?? (() => performance.now());
+        const t0 = clock();
+        const resolved = new Map();
+        const missing = [];
+        const missingSeen = new Set();
+        for (const id of cellIds) {
+            if (resolved.has(id) || missingSeen.has(id))
+                continue;
+            const cached = this.cache.get(id);
+            if (cached !== undefined) {
+                this.hits++;
+                this.touch(id, cached);
+                resolved.set(id, cached);
+            }
+            else {
+                this.misses++;
+                missingSeen.add(id);
+                missing.push(id);
+            }
+        }
+        if (resolved.size > 0)
+            this.served("own", clock() - t0);
+        if (missing.length > 0) {
+            const t1 = clock();
+            const since = this.superseded.stamp();
+            const fetched = await this.inner.fetchByIds(missing, hint);
+            const innerMs = clock() - t1;
+            for (const b of fetched) {
+                this.put(b.cellId, b, since);
+                resolved.set(b.cellId, b);
+            }
+            if (fetched.length > 0)
+                this.served("inner", innerMs);
+        }
+        return cellIds.map((id) => resolved.get(id)).filter((b) => b !== undefined);
+    }
+    fence() {
+        this.fenced = true;
+    }
+    forget(cellId) {
+        this.erased.add(cellId);
+        this.cache.delete(cellId);
+    }
+    supersede(cellId, seq) {
+        this.superseded.supersede(cellId, seq);
+        this.cache.delete(cellId);
+    }
+    has(cellId) {
+        return this.cache.has(cellId);
+    }
+    peek(cellId) {
+        if (this.fenced)
+            throw new SessionClosedError("CachingBodySource.peek");
+        return this.cache.get(cellId);
+    }
+    evict(cellId) {
+        return this.cache.delete(cellId);
+    }
+    invalidate() {
+        this.cache.clear();
+    }
+    get stats() {
+        return { hits: this.hits, misses: this.misses, evictions: this.evictions, entries: this.cache.size, olderNotCached: this.olderNotCached };
+    }
+    put(cellId, body, since) {
+        if (this.erased.has(cellId))
+            return;
+        if (!this.superseded.admits(body, since)) {
+            this.olderNotCached++;
+            return;
+        }
+        this.cache.delete(cellId);
+        this.cache.set(cellId, body);
+        while (this.cache.size > this.cfg.maxEntries) {
+            const oldest = this.cache.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.cache.delete(oldest);
+            this.evictions++;
+            try {
+                this.cfg.onEvict?.(oldest);
+            }
+            catch {
+            }
+        }
+    }
+    served(step, ms) {
+        try {
+            this.cfg.onServed?.(step, ms);
+        }
+        catch {
+        }
+    }
+    touch(cellId, body) {
+        this.cache.delete(cellId);
+        this.cache.set(cellId, body);
+    }
+}
+//# sourceMappingURL=caching_body_source.js.map