@noy-db/hub 0.7.1-pre.0 → 0.8.0

package/dist/chunk-M5B2W6YY.js

@@ -0,0 +1,293 @@
+import {
+  getAtPath
+} from "./chunk-OKSTSITU.js";
+import {
+  EmbeddingModelMismatchError
+} from "./chunk-37EPTBBR.js";
+
+// src/with-lookup/embeddings/descriptor.ts
+function embeddingSourceText(record, source) {
+  const fields = typeof source === "string" ? [source] : source;
+  const parts = [];
+  for (const f of fields) {
+    for (const leaf of getAtPath(record, f)) {
+      if (typeof leaf === "string" && leaf !== "") parts.push(leaf);
+    }
+  }
+  return parts.join(" ");
+}
+
+// src/with-lookup/embeddings/vec-id.ts
+function encodeVecId(collection, recordId) {
+  if (collection.includes("/")) {
+    throw new Error(
+      `encodeVecId: collection name "${collection}" must not contain "/" \u2014 it would make the "${collection}/${recordId}" _vec id ambiguous against another collection's rows.`
+    );
+  }
+  return `${collection}/${recordId}`;
+}
+function decodeVecId(collection, vecId) {
+  const prefix = `${collection}/`;
+  if (!vecId.startsWith(prefix)) return null;
+  return vecId.slice(prefix.length);
+}
+function isVecIdFor(collection, vecId) {
+  return vecId.startsWith(`${collection}/`);
+}
+
+// src/with-lookup/embeddings/chunks.ts
+function normalizeChunkSpans(spans, textLength) {
+  const out = [];
+  for (const s of spans) {
+    if (!Number.isInteger(s.start) || !Number.isInteger(s.end)) continue;
+    if (s.start < 0 || s.end > textLength || s.end <= s.start) continue;
+    out.push({ id: s.id ?? `c${out.length}`, start: s.start, end: s.end });
+  }
+  return out;
+}
+async function deriveChunkVectors(text, descriptor) {
+  if (!descriptor.chunk) return [];
+  const spans = normalizeChunkSpans(descriptor.chunk(text), text.length);
+  if (spans.length === 0) return [];
+  const out = [];
+  for (const span of spans) {
+    out.push({ ...span, vec: await descriptor.encode(text.slice(span.start, span.end)) });
+  }
+  return out;
+}
+
+// src/with-lookup/embeddings/cosine.ts
+function cosine(a, b) {
+  if (a.length !== b.length) return 0;
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    const x = a[i], y = b[i];
+    dot += x * y;
+    na += x * x;
+    nb += y * y;
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+// src/with-lookup/embeddings/vector-set.ts
+function pointsIn(v) {
+  if (v.chunks && v.chunks.length > 0) return v.chunks.length;
+  return v.vec ? 1 : 0;
+}
+var VectorSet = class {
+  /**
+   * `indexConfig` is `withVectorIndex()`'s return value, taken from the
+   * collection's `embeddings.index`. Absent — the overwhelmingly common case —
+   * and this class behaves exactly as it did before #1360 part 2.
+   */
+  constructor(indexConfig) {
+    this.indexConfig = indexConfig;
+  }
+  indexConfig;
+  vectors;
+  /** id → position in `vectors`, so an incremental write is O(1), not O(n). */
+  posById = /* @__PURE__ */ new Map();
+  /** model → live record count, so the model guard is O(1) per query on the indexed path. */
+  modelCounts = /* @__PURE__ */ new Map();
+  points = 0;
+  index;
+  indexing;
+  /**
+   * Set when a build produced NO partitions — a corpus of nothing but
+   * zero-norm or mismatched-dimension vectors. An empty index answers every
+   * query with `[]`, which is a WRONG answer rather than a slow one (the exact
+   * path scores those vectors 0 and returns them), so the set falls back to
+   * brute force and does not retry the build on every query.
+   */
+  indexUnusable = false;
+  get loaded() {
+    return this.vectors !== void 0;
+  }
+  /** Live VECTOR count (chunks included) — what `minVectors` is compared against. */
+  get pointCount() {
+    return this.points;
+  }
+  /** Test/benchmark introspection: whether an approximate index is currently serving queries. */
+  get indexed() {
+    return this.index !== void 0;
+  }
+  async ensureLoaded(load) {
+    if (this.vectors !== void 0) return;
+    const loaded = await load();
+    this.vectors = loaded;
+    this.posById = /* @__PURE__ */ new Map();
+    this.modelCounts = /* @__PURE__ */ new Map();
+    this.points = 0;
+    for (let i = 0; i < loaded.length; i++) {
+      const v = loaded[i];
+      this.posById.set(v.id, i);
+      this.points += pointsIn(v);
+      this.modelCounts.set(v.model, (this.modelCounts.get(v.model) ?? 0) + 1);
+    }
+    this.index = void 0;
+    this.indexUnusable = false;
+  }
+  markDirty() {
+    this.vectors = void 0;
+    this.posById = /* @__PURE__ */ new Map();
+    this.modelCounts = /* @__PURE__ */ new Map();
+    this.points = 0;
+    this.index = void 0;
+    this.indexing = void 0;
+    this.indexUnusable = false;
+  }
+  /**
+   * Incrementally insert or replace one record's vectors.
+   *
+   * This is what makes an approximate index viable at all here. `markDirty()`
+   * drops the whole set, so before #1360 part 2 every `put()` cost a full
+   * re-load on the next query — survivable for a brute-force scan that has no
+   * build step, fatal for an index whose build is tens of query-equivalents.
+   * `embedOnWrite` already holds the vectors it just encoded, so it can hand
+   * them over instead of throwing the set away.
+   *
+   * No-op when the set is not loaded: an unloaded set is already "dirty", and
+   * the next query loads everything from the sidecars anyway.
+   */
+  upsert(vector) {
+    const all = this.vectors;
+    if (all === void 0) return;
+    const at = this.posById.get(vector.id);
+    if (at !== void 0) {
+      const prev = all[at];
+      this.points -= pointsIn(prev);
+      this.decModel(prev.model);
+      all[at] = vector;
+    } else {
+      this.posById.set(vector.id, all.length);
+      all.push(vector);
+    }
+    this.points += pointsIn(vector);
+    this.modelCounts.set(vector.model, (this.modelCounts.get(vector.model) ?? 0) + 1);
+    this.index?.upsert(vector);
+  }
+  /**
+   * Incrementally drop one record's vectors. Removal is a SWAP with the last
+   * element, not a splice: `vectors` order is whatever `adapter.list()`
+   * returned and has never been part of any contract, so preserving it would
+   * buy nothing and cost an O(n) index rebuild on every `forget()`.
+   */
+  removeRecord(id) {
+    const all = this.vectors;
+    if (all === void 0) return;
+    const at = this.posById.get(id);
+    if (at === void 0) return;
+    const removed = all[at];
+    this.points -= pointsIn(removed);
+    this.decModel(removed.model);
+    const last = all.pop();
+    if (at < all.length) {
+      all[at] = last;
+      this.posById.set(last.id, at);
+    }
+    this.posById.delete(id);
+    this.index?.remove(id);
+  }
+  decModel(model) {
+    const n = (this.modelCounts.get(model) ?? 0) - 1;
+    if (n <= 0) this.modelCounts.delete(model);
+    else this.modelCounts.set(model, n);
+  }
+  /**
+   * Brute-force cosine kNN, one hit per RECORD. **Always exact.**
+   *
+   * #1360 best-chunk folding: a chunked record contributes its single BEST
+   * chunk score, never one hit per chunk and never a sum. Summing would make a
+   * long document outrank a short exact match purely for having more chunks —
+   * the folding is what keeps the score comparable across record lengths, and
+   * what keeps `k` a count of records rather than of fragments.
+   */
+  cosineTopK(query, k, opts = {}) {
+    const all = this.vectors ?? [];
+    const minScore = opts.minScore ?? -Infinity;
+    const hits = [];
+    for (const v of all) {
+      if (opts.expectModel !== void 0 && v.model !== opts.expectModel) {
+        throw new EmbeddingModelMismatchError(opts.expectModel, v.model);
+      }
+      if (v.chunks && v.chunks.length > 0) {
+        let best;
+        let bestScore = -Infinity;
+        for (const c of v.chunks) {
+          const s = cosine(query, c.vec);
+          if (s > bestScore) {
+            bestScore = s;
+            best = c;
+          }
+        }
+        if (best !== void 0 && bestScore >= minScore) {
+          hits.push({ id: v.id, score: bestScore, chunk: { id: best.id, start: best.start, end: best.end } });
+        }
+        continue;
+      }
+      if (!v.vec) continue;
+      const score = cosine(query, v.vec);
+      if (score >= minScore) hits.push({ id: v.id, score });
+    }
+    hits.sort((a, b) => b.score - a.score);
+    return hits.slice(0, k);
+  }
+  /**
+   * kNN under the collection's policy: approximate when an index is opted in
+   * AND the corpus is at or above `minVectors`, exact otherwise.
+   *
+   * ⭐ The default below the threshold is EXACT, deliberately: an index that
+   * builds on a 300-vector collection pays a build the queries can never
+   * amortise, and buys a recall cliff for a scan that was already sub-millisecond.
+   */
+  async topK(query, k, opts = {}) {
+    const cfg = this.indexConfig;
+    if (opts.exact === true || cfg === void 0 || this.indexUnusable || this.points < cfg.minVectors) {
+      return this.cosineTopK(query, k, opts);
+    }
+    if (opts.expectModel !== void 0) {
+      for (const model of this.modelCounts.keys()) {
+        if (model !== opts.expectModel) throw new EmbeddingModelMismatchError(opts.expectModel, model);
+      }
+    }
+    await this.ensureIndex(cfg);
+    const idx = this.index;
+    if (idx === void 0) return this.cosineTopK(query, k, opts);
+    return idx.search(query, k, {
+      ...opts.minScore !== void 0 ? { minScore: opts.minScore } : {},
+      ...opts.nprobe !== void 0 ? { nprobe: opts.nprobe } : {}
+    });
+  }
+  /** Build (or rebuild, once churn has made it stale) the index. Concurrency-safe. */
+  async ensureIndex(cfg) {
+    if (this.index !== void 0 && !this.index.stale) return;
+    if (this.indexing !== void 0) return this.indexing;
+    const build = (async () => {
+      const idx = await cfg.create();
+      idx.build(this.vectors ?? []);
+      if (idx.nlist === 0) {
+        this.indexUnusable = true;
+        this.index = void 0;
+        return;
+      }
+      this.index = idx;
+    })();
+    this.indexing = build;
+    try {
+      await build;
+    } finally {
+      this.indexing = void 0;
+    }
+  }
+};
+
+export {
+  embeddingSourceText,
+  VectorSet,
+  encodeVecId,
+  decodeVecId,
+  isVecIdFor,
+  deriveChunkVectors
+};
+//# sourceMappingURL=chunk-M5B2W6YY.js.map

package/dist/chunk-M5B2W6YY.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/with-lookup/embeddings/descriptor.ts","../src/with-lookup/embeddings/vec-id.ts","../src/with-lookup/embeddings/chunks.ts","../src/with-lookup/embeddings/cosine.ts","../src/with-lookup/embeddings/vector-set.ts"],"sourcesContent":["/** Per-collection embedding config (L2). The encode hook is host/remote — no bundled model. */\nimport { getAtPath } from '../../via/i18n/core.js'\nimport type { EmbeddingChunkSpan } from './chunks.js'\nimport type { VectorIndexConfig } from './vector-index.js'\n\nexport interface EmbeddingDescriptor {\n  readonly source: string | readonly string[]\n  readonly encode: (text: string) => Promise<Float32Array>\n  readonly dim: number\n  readonly model: string\n  /**\n   * Optional sub-document splitter (#1360). Given the record's joined source\n   * text (`embeddingSourceText(record, source)`), return the spans to encode\n   * separately — `start`/`end` are character offsets into THAT string. When\n   * declared and non-empty, the record is scored by its BEST chunk and the\n   * winning span comes back on the hit; when absent or empty, the record keeps\n   * exactly one whole-text vector, as before.\n   */\n  readonly chunk?: (text: string) => readonly EmbeddingChunkSpan[]\n  /**\n   * Opt into the approximate vector index (#1360 part 2) with\n   * `index: withVectorIndex()`. Absent — the default — and every semantic\n   * query is an exact brute-force cosine scan, unchanged.\n   *\n   * Opting in is not an instruction to index NOW: the index is built only once\n   * the collection holds at least `minVectors` vectors, and any single query\n   * can still demand exactness with `{ exact: true }`.\n   */\n  readonly index?: VectorIndexConfig\n}\n\n/** Concatenate the record's source-field text (skips empties; supports nested/[]-wildcard paths). */\nexport function embeddingSourceText(record: Record<string, unknown>, source: string | readonly string[]): string {\n  const fields = typeof source === 'string' ? [source] : source\n  const parts: string[] = []\n  for (const f of fields) {\n    for (const leaf of getAtPath(record, f)) {\n      if (typeof leaf === 'string' && leaf !== '') parts.push(leaf)\n    }\n  }\n  return parts.join(' ')\n}\n","/**\n * Collection-namespaced `_vec` side-car ids (#726).\n *\n * `_vec` embedding side-cars used to be keyed by the bare record id,\n * vault-wide — two collections sharing a record id shared one `_vec` row.\n * This was verified to be a clobber/crash bug, NOT a confidentiality leak:\n * every collection has its own DEK, and AES-GCM auth-tag verification means\n * decrypting a foreign collection's `_vec` row under the wrong DEK throws\n * `TamperedError` rather than returning wrong plaintext — no disclosure\n * mechanism exists. The real bug was id collision: a same-id write/forget in\n * one collection could clobber or delete the other collection's row, and a\n * collection whose `similarTo()` / cold semantic `retrieve()` encountered a\n * foreign same-id row crashed with an uncaught `TamperedError` (a DoS). The\n * store-collection bucket stays the literal `'_vec'`; the id becomes\n * composite: `${collection}/${recordId}`. Mirrors `encodeIdxId`\n * (`../indexing/persisted-indexes.js`).\n *\n * Decode is deliberately NOT split-on-`/`: the collection is always known\n * at the decode site (`ctx.name`), so `decodeVecId` strips that known\n * prefix — correct even when `recordId` itself contains `/`.\n *\n * Precondition: `collection` itself must not contain `/` — otherwise\n * `${collection}/${recordId}` is ambiguous (e.g. collection `'a'` would\n * misattribute collection `'a/b'`'s rows via prefix match in `isVecIdFor`).\n * Collection names are NOT currently restricted to a `/`-free charset by\n * `vault.collection()`, so `encodeVecId` enforces the precondition itself,\n * failing loud at the write boundary rather than silently misencoding.\n */\n\n/**\n * Encode the `_vec` side-car id for a (collection, recordId) pair.\n * Format: `<collection>/<recordId>` — no escaping; `recordId` may contain `/`.\n *\n * Throws if `collection` contains `/`: that would make the composite id\n * ambiguous against `isVecIdFor`'s prefix match (collection `'a'` would\n * appear to own collection `'a/b'`'s rows), reintroducing the clobber/crash\n * risk the namespacing fix exists to close.\n */\nexport function encodeVecId(collection: string, recordId: string): string {\n  if (collection.includes('/')) {\n    throw new Error(\n      `encodeVecId: collection name \"${collection}\" must not contain \"/\" — ` +\n        `it would make the \"${collection}/${recordId}\" _vec id ambiguous against another collection's rows.`,\n    )\n  }\n  return `${collection}/${recordId}`\n}\n\n/**\n * Decode a `_vec` id back to the bare record id, given the KNOWN owning\n * collection — strips the `<collection>/` prefix rather than splitting on\n * `/`, so a `recordId` containing `/` round-trips correctly. Returns `null`\n * if `vecId` does not belong to `collection`.\n */\nexport function decodeVecId(collection: string, vecId: string): string | null {\n  const prefix = `${collection}/`\n  if (!vecId.startsWith(prefix)) return null\n  return vecId.slice(prefix.length)\n}\n\n/** Predicate form of {@link decodeVecId} — used to filter `list()` results. */\nexport function isVecIdFor(collection: string, vecId: string): boolean {\n  return vecId.startsWith(`${collection}/`)\n}\n","/**\n * Sub-document chunking (#1360) — multiple vectors per record.\n *\n * A long document encoded as ONE vector averages its topics away: a contract\n * whose fortieth clause is the only relevant one scores like a contract about\n * nothing in particular. Chunking encodes SPANS of the record's source text\n * separately, and scores a record by its BEST chunk.\n *\n * ## What a span indexes into — the load-bearing definition\n *\n * `start` / `end` are **character offsets into\n * `embeddingSourceText(record, descriptor.source)`** — the joined source text,\n * NOT into any single field. When `source` names several fields the text is\n * their `' '`-join, so an offset can sit past a join boundary; that is exactly\n * why the offsets are defined against the join and not against a field. The\n * consumer reconstructs the snippet with the same call:\n *\n * ```ts\n * const text = embeddingSourceText(record, descriptor.source)\n * const snippet = text.slice(hit.chunk.start, hit.chunk.end)\n * ```\n *\n * `similarTo()` does precisely that to fill `hit.snippet`, so a returned span\n * is checkable against the record the consumer already holds.\n *\n * ## What is deliberately NOT here\n *\n * No bundled splitter. The splitter is host-supplied like `encode` — chunking\n * a legal clause, a markdown section and a chat transcript are different\n * problems and hub has no business guessing.\n *\n * The approximate index that was deferred here HAS since been built — it is\n * `ivf-flat.ts`, opted into with `withVectorIndex()`, and it is OFF by default\n * (see `vector-set.ts`'s `topK`). This file's own multiplier is what gated it:\n * chunking stores one vector per chunk, so a corpus of R records at C chunks\n * each brute-forces R·C vectors, and the index threshold counts VECTORS.\n * ⛔ Managed plaintext vector backends (pgvector, Vectorize, Qdrant) remain\n * excluded BY DECISION, and building an in-hub index does not reopen that: the\n * backend would hold plaintext vectors, and embedding inversion leaks the text\n * back out. The index built here never leaves the hub and is never persisted.\n */\nimport type { EmbeddingDescriptor } from './descriptor.js'\n\n/** A span the host's splitter proposes: `[start, end)` into the joined source text. */\nexport interface EmbeddingChunkSpan {\n  readonly start: number\n  readonly end: number\n  /** Stable chunk id; defaults to `c<index>` when the splitter does not supply one. */\n  readonly id?: string\n}\n\n/** A chunk as it is stored and returned — a span with a resolved id, no vector. */\nexport interface EmbeddingChunk {\n  readonly id: string\n  readonly start: number\n  readonly end: number\n}\n\n/** A chunk plus its vector, as held in memory by {@link VectorSet}. */\nexport interface StoredChunk extends EmbeddingChunk {\n  readonly vec: Float32Array\n}\n\n/**\n * Normalise the splitter's output: drop empty/inverted/out-of-range spans,\n * resolve ids. Invalid spans are DROPPED rather than thrown on — a splitter is\n * host code running over host text, and one degenerate span (a trailing\n * separator, a zero-width match) must not fail the write; a record that ends up\n * with no usable span falls back to the whole-record vector, which is the\n * pre-#1360 behaviour and never worse than not indexing the record at all.\n */\nexport function normalizeChunkSpans(spans: readonly EmbeddingChunkSpan[], textLength: number): EmbeddingChunk[] {\n  const out: EmbeddingChunk[] = []\n  for (const s of spans) {\n    if (!Number.isInteger(s.start) || !Number.isInteger(s.end)) continue\n    if (s.start < 0 || s.end > textLength || s.end <= s.start) continue\n    out.push({ id: s.id ?? `c${out.length}`, start: s.start, end: s.end })\n  }\n  return out\n}\n\n/**\n * Encode each chunk of `text` under the descriptor's `encode` hook.\n * Returns `[]` when no `chunk` hook is declared or it yields no usable span —\n * the caller then stores the single whole-record vector, unchanged.\n *\n * Dimension is checked per chunk by the caller (it owns the error's field\n * name), so this stays a pure derive.\n */\nexport async function deriveChunkVectors(\n  text: string,\n  descriptor: EmbeddingDescriptor,\n): Promise<(EmbeddingChunk & { vec: Float32Array })[]> {\n  if (!descriptor.chunk) return []\n  const spans = normalizeChunkSpans(descriptor.chunk(text), text.length)\n  if (spans.length === 0) return []\n  const out: (EmbeddingChunk & { vec: Float32Array })[] = []\n  for (const span of spans) {\n    out.push({ ...span, vec: await descriptor.encode(text.slice(span.start, span.end)) })\n  }\n  return out\n}\n","/** Cosine similarity in [-1,1]; 0 on zero-norm or length mismatch (no NaN). (L2) */\nexport function cosine(a: Float32Array | number[], b: Float32Array | number[]): number {\n  if (a.length !== b.length) return 0\n  let dot = 0, na = 0, nb = 0\n  for (let i = 0; i < a.length; i++) {\n    const x = a[i]!, y = b[i]!\n    dot += x * y; na += x * x; nb += y * y\n  }\n  if (na === 0 || nb === 0) return 0\n  return dot / (Math.sqrt(na) * Math.sqrt(nb))\n}\n","/** In-memory vector set for L2 semantic retrieval. Loaded once per session from\n *  decrypted _vec sidecars (injected loader), brute-force cosine kNN, model-guarded.\n *  #1360 part 2 adds an OPT-IN approximate index in front of the brute force. */\nimport { cosine } from './cosine.js'\nimport type { EmbeddingChunk, StoredChunk } from './chunks.js'\nimport type { VectorIndex, VectorIndexConfig } from './vector-index.js'\nimport { EmbeddingModelMismatchError } from '../../kernel/errors.js'\n\n/**\n * One record's vectors. Exactly one of `vec` (whole-record, the pre-#1360\n * shape) and `chunks` (sub-document, #1360) is populated by the loader; a row\n * with neither is inert and scores nothing.\n */\nexport interface StoredVector {\n  readonly id: string\n  readonly model: string\n  readonly vec?: Float32Array\n  /** #1360 — sub-document chunks. When present, the record is scored by its BEST chunk. */\n  readonly chunks?: readonly StoredChunk[]\n}\nexport interface VectorHit {\n  readonly id: string\n  readonly score: number\n  /** #1360 — the winning chunk's span, when the record was scored via chunks. */\n  readonly chunk?: EmbeddingChunk\n}\n\n/** Options accepted by {@link VectorSet.topK}. */\nexport interface VectorTopKOptions {\n  readonly minScore?: number\n  readonly expectModel?: string\n  /**\n   * Force the exact brute-force scan for this query, whatever the corpus size\n   * and whatever `index` the descriptor declares. Exactness stays REACHABLE:\n   * an approximate default that cannot be turned off is a correctness change\n   * dressed as a performance one.\n   */\n  readonly exact?: boolean\n  /** Per-query recall dial; overrides the configured `nprobe`. Ignored on the exact path. */\n  readonly nprobe?: number\n}\n\n/** Points contributed by one stored record — chunked records contribute one per chunk. */\nfunction pointsIn(v: StoredVector): number {\n  if (v.chunks && v.chunks.length > 0) return v.chunks.length\n  return v.vec ? 1 : 0\n}\n\nexport class VectorSet {\n  private vectors: StoredVector[] | undefined\n  /** id → position in `vectors`, so an incremental write is O(1), not O(n). */\n  private posById = new Map<string, number>()\n  /** model → live record count, so the model guard is O(1) per query on the indexed path. */\n  private modelCounts = new Map<string, number>()\n  private points = 0\n  private index: VectorIndex | undefined\n  private indexing: Promise<void> | undefined\n  /**\n   * Set when a build produced NO partitions — a corpus of nothing but\n   * zero-norm or mismatched-dimension vectors. An empty index answers every\n   * query with `[]`, which is a WRONG answer rather than a slow one (the exact\n   * path scores those vectors 0 and returns them), so the set falls back to\n   * brute force and does not retry the build on every query.\n   */\n  private indexUnusable = false\n\n  /**\n   * `indexConfig` is `withVectorIndex()`'s return value, taken from the\n   * collection's `embeddings.index`. Absent — the overwhelmingly common case —\n   * and this class behaves exactly as it did before #1360 part 2.\n   */\n  constructor(private readonly indexConfig?: VectorIndexConfig) {}\n\n  get loaded(): boolean { return this.vectors !== undefined }\n  /** Live VECTOR count (chunks included) — what `minVectors` is compared against. */\n  get pointCount(): number { return this.points }\n  /** Test/benchmark introspection: whether an approximate index is currently serving queries. */\n  get indexed(): boolean { return this.index !== undefined }\n\n  async ensureLoaded(load: () => Promise<StoredVector[]>): Promise<void> {\n    if (this.vectors !== undefined) return\n    const loaded = await load()\n    this.vectors = loaded\n    this.posById = new Map()\n    this.modelCounts = new Map()\n    this.points = 0\n    for (let i = 0; i < loaded.length; i++) {\n      const v = loaded[i]!\n      this.posById.set(v.id, i)\n      this.points += pointsIn(v)\n      this.modelCounts.set(v.model, (this.modelCounts.get(v.model) ?? 0) + 1)\n    }\n    this.index = undefined\n    this.indexUnusable = false\n  }\n\n  markDirty(): void {\n    this.vectors = undefined\n    this.posById = new Map()\n    this.modelCounts = new Map()\n    this.points = 0\n    this.index = undefined\n    this.indexing = undefined\n    this.indexUnusable = false\n  }\n\n  /**\n   * Incrementally insert or replace one record's vectors.\n   *\n   * This is what makes an approximate index viable at all here. `markDirty()`\n   * drops the whole set, so before #1360 part 2 every `put()` cost a full\n   * re-load on the next query — survivable for a brute-force scan that has no\n   * build step, fatal for an index whose build is tens of query-equivalents.\n   * `embedOnWrite` already holds the vectors it just encoded, so it can hand\n   * them over instead of throwing the set away.\n   *\n   * No-op when the set is not loaded: an unloaded set is already \"dirty\", and\n   * the next query loads everything from the sidecars anyway.\n   */\n  upsert(vector: StoredVector): void {\n    const all = this.vectors\n    if (all === undefined) return\n    const at = this.posById.get(vector.id)\n    if (at !== undefined) {\n      const prev = all[at]!\n      this.points -= pointsIn(prev)\n      this.decModel(prev.model)\n      all[at] = vector\n    } else {\n      this.posById.set(vector.id, all.length)\n      all.push(vector)\n    }\n    this.points += pointsIn(vector)\n    this.modelCounts.set(vector.model, (this.modelCounts.get(vector.model) ?? 0) + 1)\n    this.index?.upsert(vector)\n  }\n\n  /**\n   * Incrementally drop one record's vectors. Removal is a SWAP with the last\n   * element, not a splice: `vectors` order is whatever `adapter.list()`\n   * returned and has never been part of any contract, so preserving it would\n   * buy nothing and cost an O(n) index rebuild on every `forget()`.\n   */\n  removeRecord(id: string): void {\n    const all = this.vectors\n    if (all === undefined) return\n    const at = this.posById.get(id)\n    if (at === undefined) return\n    const removed = all[at]!\n    this.points -= pointsIn(removed)\n    this.decModel(removed.model)\n    const last = all.pop()!\n    if (at < all.length) { all[at] = last; this.posById.set(last.id, at) }\n    this.posById.delete(id)\n    this.index?.remove(id)\n  }\n\n  private decModel(model: string): void {\n    const n = (this.modelCounts.get(model) ?? 0) - 1\n    if (n <= 0) this.modelCounts.delete(model)\n    else this.modelCounts.set(model, n)\n  }\n\n  /**\n   * Brute-force cosine kNN, one hit per RECORD. **Always exact.**\n   *\n   * #1360 best-chunk folding: a chunked record contributes its single BEST\n   * chunk score, never one hit per chunk and never a sum. Summing would make a\n   * long document outrank a short exact match purely for having more chunks —\n   * the folding is what keeps the score comparable across record lengths, and\n   * what keeps `k` a count of records rather than of fragments.\n   */\n  cosineTopK(query: Float32Array, k: number, opts: { minScore?: number; expectModel?: string } = {}): VectorHit[] {\n    const all = this.vectors ?? []\n    const minScore = opts.minScore ?? -Infinity\n    const hits: VectorHit[] = []\n    for (const v of all) {\n      if (opts.expectModel !== undefined && v.model !== opts.expectModel) {\n        throw new EmbeddingModelMismatchError(opts.expectModel, v.model)\n      }\n      if (v.chunks && v.chunks.length > 0) {\n        let best: StoredChunk | undefined\n        let bestScore = -Infinity\n        for (const c of v.chunks) {\n          const s = cosine(query, c.vec)\n          if (s > bestScore) { bestScore = s; best = c }\n        }\n        if (best !== undefined && bestScore >= minScore) {\n          hits.push({ id: v.id, score: bestScore, chunk: { id: best.id, start: best.start, end: best.end } })\n        }\n        continue\n      }\n      if (!v.vec) continue\n      const score = cosine(query, v.vec)\n      if (score >= minScore) hits.push({ id: v.id, score })\n    }\n    hits.sort((a, b) => b.score - a.score)\n    return hits.slice(0, k)\n  }\n\n  /**\n   * kNN under the collection's policy: approximate when an index is opted in\n   * AND the corpus is at or above `minVectors`, exact otherwise.\n   *\n   * ⭐ The default below the threshold is EXACT, deliberately: an index that\n   * builds on a 300-vector collection pays a build the queries can never\n   * amortise, and buys a recall cliff for a scan that was already sub-millisecond.\n   */\n  async topK(query: Float32Array, k: number, opts: VectorTopKOptions = {}): Promise<VectorHit[]> {\n    const cfg = this.indexConfig\n    if (opts.exact === true || cfg === undefined || this.indexUnusable || this.points < cfg.minVectors) {\n      return this.cosineTopK(query, k, opts)\n    }\n    // The exact path throws per-vector on a model mismatch as it scans; the\n    // indexed path never scans every vector, so the same guard has to be a set\n    // membership test. Same error, same trigger, O(1).\n    if (opts.expectModel !== undefined) {\n      for (const model of this.modelCounts.keys()) {\n        if (model !== opts.expectModel) throw new EmbeddingModelMismatchError(opts.expectModel, model)\n      }\n    }\n    await this.ensureIndex(cfg)\n    const idx = this.index\n    if (idx === undefined) return this.cosineTopK(query, k, opts)\n    return idx.search(query, k, {\n      ...(opts.minScore !== undefined ? { minScore: opts.minScore } : {}),\n      ...(opts.nprobe !== undefined ? { nprobe: opts.nprobe } : {}),\n    })\n  }\n\n  /** Build (or rebuild, once churn has made it stale) the index. Concurrency-safe. */\n  private async ensureIndex(cfg: VectorIndexConfig): Promise<void> {\n    if (this.index !== undefined && !this.index.stale) return\n    if (this.indexing !== undefined) return this.indexing\n    const build = (async () => {\n      const idx = await cfg.create()\n      idx.build(this.vectors ?? [])\n      if (idx.nlist === 0) { this.indexUnusable = true; this.index = undefined; return }\n      this.index = idx\n    })()\n    this.indexing = build\n    try { await build } finally { this.indexing = undefined }\n  }\n}\n"],"mappings":";;;;;;;;AAgCO,SAAS,oBAAoB,QAAiC,QAA4C;AAC/G,QAAM,SAAS,OAAO,WAAW,WAAW,CAAC,MAAM,IAAI;AACvD,QAAM,QAAkB,CAAC;AACzB,aAAW,KAAK,QAAQ;AACtB,eAAW,QAAQ,UAAU,QAAQ,CAAC,GAAG;AACvC,UAAI,OAAO,SAAS,YAAY,SAAS,GAAI,OAAM,KAAK,IAAI;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,MAAM,KAAK,GAAG;AACvB;;;ACHO,SAAS,YAAY,YAAoB,UAA0B;AACxE,MAAI,WAAW,SAAS,GAAG,GAAG;AAC5B,UAAM,IAAI;AAAA,MACR,iCAAiC,UAAU,oDACnB,UAAU,IAAI,QAAQ;AAAA,IAChD;AAAA,EACF;AACA,SAAO,GAAG,UAAU,IAAI,QAAQ;AAClC;AAQO,SAAS,YAAY,YAAoB,OAA8B;AAC5E,QAAM,SAAS,GAAG,UAAU;AAC5B,MAAI,CAAC,MAAM,WAAW,MAAM,EAAG,QAAO;AACtC,SAAO,MAAM,MAAM,OAAO,MAAM;AAClC;AAGO,SAAS,WAAW,YAAoB,OAAwB;AACrE,SAAO,MAAM,WAAW,GAAG,UAAU,GAAG;AAC1C;;;ACQO,SAAS,oBAAoB,OAAsC,YAAsC;AAC9G,QAAM,MAAwB,CAAC;AAC/B,aAAW,KAAK,OAAO;AACrB,QAAI,CAAC,OAAO,UAAU,EAAE,KAAK,KAAK,CAAC,OAAO,UAAU,EAAE,GAAG,EAAG;AAC5D,QAAI,EAAE,QAAQ,KAAK,EAAE,MAAM,cAAc,EAAE,OAAO,EAAE,MAAO;AAC3D,QAAI,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,IAAI,MAAM,IAAI,OAAO,EAAE,OAAO,KAAK,EAAE,IAAI,CAAC;AAAA,EACvE;AACA,SAAO;AACT;AAUA,eAAsB,mBACpB,MACA,YACqD;AACrD,MAAI,CAAC,WAAW,MAAO,QAAO,CAAC;AAC/B,QAAM,QAAQ,oBAAoB,WAAW,MAAM,IAAI,GAAG,KAAK,MAAM;AACrE,MAAI,MAAM,WAAW,EAAG,QAAO,CAAC;AAChC,QAAM,MAAkD,CAAC;AACzD,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,EAAE,GAAG,MAAM,KAAK,MAAM,WAAW,OAAO,KAAK,MAAM,KAAK,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;AAAA,EACtF;AACA,SAAO;AACT;;;ACpGO,SAAS,OAAO,GAA4B,GAAoC;AACrF,MAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,MAAI,MAAM,GAAG,KAAK,GAAG,KAAK;AAC1B,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,IAAI,EAAE,CAAC,GAAI,IAAI,EAAE,CAAC;AACxB,WAAO,IAAI;AAAG,UAAM,IAAI;AAAG,UAAM,IAAI;AAAA,EACvC;AACA,MAAI,OAAO,KAAK,OAAO,EAAG,QAAO;AACjC,SAAO,OAAO,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,EAAE;AAC5C;;;ACiCA,SAAS,SAAS,GAAyB;AACzC,MAAI,EAAE,UAAU,EAAE,OAAO,SAAS,EAAG,QAAO,EAAE,OAAO;AACrD,SAAO,EAAE,MAAM,IAAI;AACrB;AAEO,IAAM,YAAN,MAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAuBrB,YAA6B,aAAiC;AAAjC;AAAA,EAAkC;AAAA,EAAlC;AAAA,EAtBrB;AAAA;AAAA,EAEA,UAAU,oBAAI,IAAoB;AAAA;AAAA,EAElC,cAAc,oBAAI,IAAoB;AAAA,EACtC,SAAS;AAAA,EACT;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,gBAAgB;AAAA,EASxB,IAAI,SAAkB;AAAE,WAAO,KAAK,YAAY;AAAA,EAAU;AAAA;AAAA,EAE1D,IAAI,aAAqB;AAAE,WAAO,KAAK;AAAA,EAAO;AAAA;AAAA,EAE9C,IAAI,UAAmB;AAAE,WAAO,KAAK,UAAU;AAAA,EAAU;AAAA,EAEzD,MAAM,aAAa,MAAoD;AACrE,QAAI,KAAK,YAAY,OAAW;AAChC,UAAM,SAAS,MAAM,KAAK;AAC1B,SAAK,UAAU;AACf,SAAK,UAAU,oBAAI,IAAI;AACvB,SAAK,cAAc,oBAAI,IAAI;AAC3B,SAAK,SAAS;AACd,aAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,YAAM,IAAI,OAAO,CAAC;AAClB,WAAK,QAAQ,IAAI,EAAE,IAAI,CAAC;AACxB,WAAK,UAAU,SAAS,CAAC;AACzB,WAAK,YAAY,IAAI,EAAE,QAAQ,KAAK,YAAY,IAAI,EAAE,KAAK,KAAK,KAAK,CAAC;AAAA,IACxE;AACA,SAAK,QAAQ;AACb,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,YAAkB;AAChB,SAAK,UAAU;AACf,SAAK,UAAU,oBAAI,IAAI;AACvB,SAAK,cAAc,oBAAI,IAAI;AAC3B,SAAK,SAAS;AACd,SAAK,QAAQ;AACb,SAAK,WAAW;AAChB,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,OAAO,QAA4B;AACjC,UAAM,MAAM,KAAK;AACjB,QAAI,QAAQ,OAAW;AACvB,UAAM,KAAK,KAAK,QAAQ,IAAI,OAAO,EAAE;AACrC,QAAI,OAAO,QAAW;AACpB,YAAM,OAAO,IAAI,EAAE;AACnB,WAAK,UAAU,SAAS,IAAI;AAC5B,WAAK,SAAS,KAAK,KAAK;AACxB,UAAI,EAAE,IAAI;AAAA,IACZ,OAAO;AACL,WAAK,QAAQ,IAAI,OAAO,IAAI,IAAI,MAAM;AACtC,UAAI,KAAK,MAAM;AAAA,IACjB;AACA,SAAK,UAAU,SAAS,MAAM;AAC9B,SAAK,YAAY,IAAI,OAAO,QAAQ,KAAK,YAAY,IAAI,OAAO,KAAK,KAAK,KAAK,CAAC;AAChF,SAAK,OAAO,OAAO,MAAM;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,aAAa,IAAkB;AAC7B,UAAM,MAAM,KAAK;AACjB,QAAI,QAAQ,OAAW;AACvB,UAAM,KAAK,KAAK,QAAQ,IAAI,EAAE;AAC9B,QAAI,OAAO,OAAW;AACtB,UAAM,UAAU,IAAI,EAAE;AACtB,SAAK,UAAU,SAAS,OAAO;AAC/B,SAAK,SAAS,QAAQ,KAAK;AAC3B,UAAM,OAAO,IAAI,IAAI;AACrB,QAAI,KAAK,IAAI,QAAQ;AAAE,UAAI,EAAE,IAAI;AAAM,WAAK,QAAQ,IAAI,KAAK,IAAI,EAAE;AAAA,IAAE;AACrE,SAAK,QAAQ,OAAO,EAAE;AACtB,SAAK,OAAO,OAAO,EAAE;AAAA,EACvB;AAAA,EAEQ,SAAS,OAAqB;AACpC,UAAM,KAAK,KAAK,YAAY,IAAI,KAAK,KAAK,KAAK;AAC/C,QAAI,KAAK,EAAG,MAAK,YAAY,OAAO,KAAK;AAAA,QACpC,MAAK,YAAY,IAAI,OAAO,CAAC;AAAA,EACpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,WAAW,OAAqB,GAAW,OAAoD,CAAC,GAAgB;AAC9G,UAAM,MAAM,KAAK,WAAW,CAAC;AAC7B,UAAM,WAAW,KAAK,YAAY;AAClC,UAAM,OAAoB,CAAC;AAC3B,eAAW,KAAK,KAAK;AACnB,UAAI,KAAK,gBAAgB,UAAa,EAAE,UAAU,KAAK,aAAa;AAClE,cAAM,IAAI,4BAA4B,KAAK,aAAa,EAAE,KAAK;AAAA,MACjE;AACA,UAAI,EAAE,UAAU,EAAE,OAAO,SAAS,GAAG;AACnC,YAAI;AACJ,YAAI,YAAY;AAChB,mBAAW,KAAK,EAAE,QAAQ;AACxB,gBAAM,IAAI,OAAO,OAAO,EAAE,GAAG;AAC7B,cAAI,IAAI,WAAW;AAAE,wBAAY;AAAG,mBAAO;AAAA,UAAE;AAAA,QAC/C;AACA,YAAI,SAAS,UAAa,aAAa,UAAU;AAC/C,eAAK,KAAK,EAAE,IAAI,EAAE,IAAI,OAAO,WAAW,OAAO,EAAE,IAAI,KAAK,IAAI,OAAO,KAAK,OAAO,KAAK,KAAK,IAAI,EAAE,CAAC;AAAA,QACpG;AACA;AAAA,MACF;AACA,UAAI,CAAC,EAAE,IAAK;AACZ,YAAM,QAAQ,OAAO,OAAO,EAAE,GAAG;AACjC,UAAI,SAAS,SAAU,MAAK,KAAK,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC;AAAA,IACtD;AACA,SAAK,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACrC,WAAO,KAAK,MAAM,GAAG,CAAC;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,KAAK,OAAqB,GAAW,OAA0B,CAAC,GAAyB;AAC7F,UAAM,MAAM,KAAK;AACjB,QAAI,KAAK,UAAU,QAAQ,QAAQ,UAAa,KAAK,iBAAiB,KAAK,SAAS,IAAI,YAAY;AAClG,aAAO,KAAK,WAAW,OAAO,GAAG,IAAI;AAAA,IACvC;AAIA,QAAI,KAAK,gBAAgB,QAAW;AAClC,iBAAW,SAAS,KAAK,YAAY,KAAK,GAAG;AAC3C,YAAI,UAAU,KAAK,YAAa,OAAM,IAAI,4BAA4B,KAAK,aAAa,KAAK;AAAA,MAC/F;AAAA,IACF;AACA,UAAM,KAAK,YAAY,GAAG;AAC1B,UAAM,MAAM,KAAK;AACjB,QAAI,QAAQ,OAAW,QAAO,KAAK,WAAW,OAAO,GAAG,IAAI;AAC5D,WAAO,IAAI,OAAO,OAAO,GAAG;AAAA,MAC1B,GAAI,KAAK,aAAa,SAAY,EAAE,UAAU,KAAK,SAAS,IAAI,CAAC;AAAA,MACjE,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;AAAA,IAC7D,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,MAAc,YAAY,KAAuC;AAC/D,QAAI,KAAK,UAAU,UAAa,CAAC,KAAK,MAAM,MAAO;AACnD,QAAI,KAAK,aAAa,OAAW,QAAO,KAAK;AAC7C,UAAM,SAAS,YAAY;AACzB,YAAM,MAAM,MAAM,IAAI,OAAO;AAC7B,UAAI,MAAM,KAAK,WAAW,CAAC,CAAC;AAC5B,UAAI,IAAI,UAAU,GAAG;AAAE,aAAK,gBAAgB;AAAM,aAAK,QAAQ;AAAW;AAAA,MAAO;AACjF,WAAK,QAAQ;AAAA,IACf,GAAG;AACH,SAAK,WAAW;AAChB,QAAI;AAAE,YAAM;AAAA,IAAM,UAAE;AAAU,WAAK,WAAW;AAAA,IAAU;AAAA,EAC1D;AACF;","names":[]}