@vellumai/assistant 0.8.8 → 0.8.9-dev.202606091853.fbaa2ae

package/src/plugins/defaults/memory-v3-shadow/section-dense-store.ts

@@ -0,0 +1,236 @@
+// ---------------------------------------------------------------------------
+// Memory v3 — section-grain dense Qdrant collection
+// ---------------------------------------------------------------------------
+//
+// Owns a dedicated Qdrant collection holding one dense embedding per page
+// *section* (the lead block plus each `## `-delimited heading section, chunked
+// to fit the embedding window — see `sections.ts`). This is the dense lane for
+// the section-grain retrieval design: where the v2 `memory_v2_concept_pages`
+// collection embeds whole pages, this one embeds sub-page sections so a query
+// can match the single relevant block of a long article.
+//
+// Reuses the v2 Qdrant URL/config (`resolveQdrantUrl` + `config.memory.qdrant.*`)
+// and the shared embedding backend (`embedWithBackend`) rather than standing up
+// new infrastructure. The collection carries a single dense named vector sized
+// to the configured embedding backend (`config.memory.qdrant.vectorSize`) — no
+// sparse channel, since the section dense lane is dense-only.
+//
+// This module owns the write side (collection lifecycle + upserts) and the
+// shared Qdrant client; the read side (`dense.ts`) reuses that client to query
+// this collection.
+
+import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
+import { v5 as uuidv5 } from "uuid";
+
+import type { AssistantConfig } from "../../../config/types.js";
+import { embedWithBackend } from "../../../memory/embedding-backend.js";
+import { resolveQdrantUrl } from "../../../memory/qdrant-client.js";
+import { getLogger } from "../../../util/logger.js";
+import type { Section } from "./types.js";
+
+const log = getLogger("memory-v3-section-dense-store");
+
+/** Name of the dedicated Qdrant collection holding section-grain embeddings. */
+export const SECTION_COLLECTION = "memory_v3_sections";
+
+/**
+ * Stable UUIDv5 namespace used to derive a deterministic Qdrant point ID from a
+ * section's `(article, ordinal)` pair. The namespace itself is an arbitrary
+ * fixed UUID; what matters is that the same section always maps to the same
+ * point ID so re-upserts replace the prior point in place instead of
+ * accumulating duplicates.
+ */
+const SECTION_NAMESPACE = "1d2c3b4a-5e6f-4a7b-8c9d-0e1f2a3b4c5d";
+
+let _client: QdrantRestClient | null = null;
+let _collectionReady = false;
+
+/**
+ * Lazily create the shared Qdrant REST client bound to the resolved URL.
+ * Shared by both section-dense lanes (this store and the dense read lane in
+ * `dense.ts`) so they reuse one client instance and one reset hook.
+ */
+export function getSectionDenseClient(
+  config: AssistantConfig,
+): QdrantRestClient {
+  if (_client) return _client;
+  _client = new QdrantRestClient({
+    url: resolveQdrantUrl(config),
+    checkCompatibility: false,
+  });
+  return _client;
+}
+
+/**
+ * Derive the deterministic Qdrant point ID for a section. Qdrant requires
+ * UUID/integer IDs; UUIDv5 over `${article}#${ordinal}` keeps the mapping
+ * stable across processes so upserts replace in place.
+ */
+function pointIdForSection(article: string, ordinal: number): string {
+  return uuidv5(`${article}#${ordinal}`, SECTION_NAMESPACE);
+}
+
+/**
+ * Create the section-grain collection if it does not already exist, with a
+ * single dense vector sized to the configured embedding backend. Idempotent:
+ * an already-ready collection is a no-op, and a concurrent 409-on-create is
+ * treated as success.
+ */
+export async function ensureSectionCollection(
+  config: AssistantConfig,
+): Promise<void> {
+  if (_collectionReady) return;
+
+  const client = getSectionDenseClient(config);
+  const vectorSize = config.memory.qdrant.vectorSize;
+  const onDisk = config.memory.qdrant.onDisk;
+
+  const exists = await client.collectionExists(SECTION_COLLECTION);
+  if (!exists.exists) {
+    log.info(
+      { collection: SECTION_COLLECTION, vectorSize },
+      "Creating Qdrant collection for memory v3 sections",
+    );
+    try {
+      await client.createCollection(SECTION_COLLECTION, {
+        vectors: {
+          size: vectorSize,
+          distance: "Cosine",
+          on_disk: onDisk,
+        },
+        hnsw_config: {
+          on_disk: onDisk,
+          m: 16,
+          ef_construct: 100,
+        },
+        optimizers_config: {
+          default_segment_number: 2,
+        },
+        on_disk_payload: onDisk,
+      });
+    } catch (err) {
+      // 409 = a concurrent caller created the collection — fall through to
+      // ensure the payload index below rather than returning early.
+      const status =
+        err instanceof Error && "status" in err
+          ? (err as { status: number }).status
+          : undefined;
+      if (status !== 409) throw err;
+    }
+  }
+
+  // Always ensure the `article` payload index — on EVERY path (fresh create,
+  // concurrent 409, and a pre-existing collection whose index creation was
+  // interrupted by an earlier crash). `deleteSectionsForArticle`/pruning filter
+  // on `article`, and strict-mode Qdrant rejects filters on unindexed fields, so
+  // a missing index would make them fail until manual repair. createPayloadIndex
+  // is idempotent, so re-running it against an existing index is a no-op.
+  await client.createPayloadIndex(SECTION_COLLECTION, {
+    field_name: "article",
+    field_schema: "keyword",
+  });
+
+  _collectionReady = true;
+}
+
+/**
+ * Embed each section's `text` and upsert one point per section, keyed by a
+ * deterministic `(article, ordinal)`-derived ID. Stable IDs mean re-upserting
+ * the same sections overwrites in place rather than accumulating duplicates,
+ * so the operation is idempotent. Payload carries `{ article, ordinal, title }`
+ * for downstream filtering and rendering.
+ *
+ * An empty `sections` array is a no-op (no embedding round-trip).
+ */
+export async function upsertSections(
+  config: AssistantConfig,
+  sections: Section[],
+): Promise<void> {
+  if (sections.length === 0) return;
+
+  await ensureSectionCollection(config);
+
+  const { vectors } = await embedWithBackend(
+    config,
+    sections.map((s) => s.text),
+  );
+
+  const points = sections.map((section, i) => ({
+    id: pointIdForSection(section.article, section.ordinal),
+    vector: vectors[i]!,
+    payload: {
+      article: section.article,
+      ordinal: section.ordinal,
+      title: section.title,
+    },
+  }));
+
+  await getSectionDenseClient(config).upsert(SECTION_COLLECTION, {
+    wait: true,
+    points,
+  });
+}
+
+/**
+ * Delete every section point belonging to an article. Used by incremental
+ * rebuilds (in a later PR) to clear an article's stale sections before
+ * re-upserting its current ones. Idempotent: deleting an absent article's
+ * sections is a no-op server-side.
+ */
+export async function deleteSectionsForArticle(
+  config: AssistantConfig,
+  article: string,
+): Promise<void> {
+  await ensureSectionCollection(config);
+
+  await getSectionDenseClient(config).delete(SECTION_COLLECTION, {
+    wait: true,
+    filter: { must: [{ key: "article", match: { value: article } }] },
+  });
+}
+
+/**
+ * Return every distinct `article` slug that currently has at least one section
+ * point in the collection. Scrolls the whole collection (payload only, no
+ * vectors) in bounded batches and collects the distinct `article` values.
+ *
+ * Used by the maintain job's prune stage to find articles whose points linger
+ * after the page was deleted from the index — the change-delta selector never
+ * names a slug that is gone, so a deleted page's stale points are only
+ * observable by reading the collection back. An empty/absent collection yields
+ * an empty list.
+ */
+export async function listSectionArticles(
+  config: AssistantConfig,
+): Promise<string[]> {
+  await ensureSectionCollection(config);
+
+  const client = getSectionDenseClient(config);
+  const articles = new Set<string>();
+  let offset: string | number | undefined = undefined;
+  const maxIterations = 10_000;
+  const batchSize = 256;
+  for (let i = 0; i < maxIterations; i++) {
+    const result = await client.scroll(SECTION_COLLECTION, {
+      limit: batchSize,
+      with_payload: true,
+      with_vector: false,
+      ...(offset !== undefined ? { offset } : {}),
+    });
+    for (const point of result.points) {
+      const article = (point.payload as { article?: unknown } | null)?.article;
+      if (typeof article === "string") articles.add(article);
+    }
+    const next = result.next_page_offset;
+    if (next == null) break;
+    offset = typeof next === "string" ? next : (next as number);
+  }
+
+  return [...articles];
+}
+
+/** @internal Test-only: reset module-level singletons. */
+export function _resetSectionDenseStoreForTests(): void {
+  _client = null;
+  _collectionReady = false;
+}

package/src/plugins/defaults/memory-v3-shadow/section-needle.ts

@@ -0,0 +1,200 @@
+import type { SectionIndex, Slug } from "./types.js";
+
+/**
+ * Section-grain "needle" lane for memory-v3 retrieval: a lexical BM25F search
+ * over a {@link SectionIndex}. Each section is a tiny two-field document — a
+ * weighted `head` line (`${lastSlugSegment} — ${title}`) and the remaining
+ * `body` text — so a literal term in a heading outranks the same term buried in
+ * prose. Scoring happens at section grain; results are deduped to distinct
+ * articles, each tagged with its best-scoring section.
+ *
+ * Implementation notes:
+ * - Hand-rolled Okapi BM25F (no dependency). The corpus is bounded (one doc per
+ *   section), so a plain inverted index is plenty.
+ * - To match the validated harness the term stream includes adjacent-token
+ *   bigrams (`token[i] + "_" + token[i+1]`) in addition to unigrams.
+ */
+
+/** Okapi BM25 term-frequency saturation parameter. */
+const k1 = 1.5;
+/** Okapi BM25 length-normalization parameter. */
+const b = 0.75;
+
+/** `head`-field weight in the BM25F term-frequency blend. */
+const HEAD_WEIGHT = 2.5;
+/** `body`-field weight in the BM25F term-frequency blend. */
+const BODY_WEIGHT = 1;
+
+export interface SectionNeedle {
+  /**
+   * Returns up to `k` distinct articles ranked by BM25F score (descending),
+   * each tagged with its best-scoring section index (into the underlying
+   * `SectionIndex.sections`). Ties break by `(article, ordinal)`.
+   */
+  query(text: string, k: number): { article: Slug; section: number }[];
+  /**
+   * Highest-scoring section index (into `SectionIndex.sections`) for `article`
+   * against `queryText`. Returns the article's first section index when no term
+   * matches (or `-1` if the article has no sections). Used by other lanes to
+   * attach a matched-section descriptor to an article they surface.
+   */
+  bestSection(article: Slug, queryText: string): number;
+}
+
+/** Lowercase, split on non-alphanumeric, drop empties. */
+function tokenizeUnigrams(text: string): string[] {
+  return text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter((token) => token.length > 0);
+}
+
+/** Unigrams plus adjacent-token bigrams (`a_b`), matching the harness. */
+function tokenize(text: string): string[] {
+  const unigrams = tokenizeUnigrams(text);
+  const terms = [...unigrams];
+  for (let i = 0; i + 1 < unigrams.length; i++) {
+    terms.push(`${unigrams[i]}_${unigrams[i + 1]}`);
+  }
+  return terms;
+}
+
+/** Split a section's text into its head line and the remaining body. */
+function splitHeadBody(text: string): { head: string; body: string } {
+  const newline = text.indexOf("\n");
+  if (newline === -1) return { head: text, body: "" };
+  return { head: text.slice(0, newline), body: text.slice(newline + 1) };
+}
+
+interface Posting {
+  doc: number;
+  /** Field-weighted term frequency: HEAD_WEIGHT*tf_head + BODY_WEIGHT*tf_body. */
+  weightedTf: number;
+}
+
+export function buildSectionNeedle(index: SectionIndex): SectionNeedle {
+  const { sections } = index;
+  const docCount = sections.length;
+
+  // term -> postings (section index + weighted term frequency).
+  const postings = new Map<string, Posting[]>();
+  // BM25F effective document length = weighted token count per section.
+  const docLengths: number[] = new Array(docCount).fill(0);
+  let totalLength = 0;
+
+  for (let doc = 0; doc < docCount; doc++) {
+    const { head, body } = splitHeadBody(sections[doc]!.text);
+    const headTerms = tokenize(head);
+    const bodyTerms = tokenize(body);
+
+    const length =
+      HEAD_WEIGHT * headTerms.length + BODY_WEIGHT * bodyTerms.length;
+    docLengths[doc] = length;
+    totalLength += length;
+
+    const weightedTf = new Map<string, number>();
+    for (const term of headTerms) {
+      weightedTf.set(term, (weightedTf.get(term) ?? 0) + HEAD_WEIGHT);
+    }
+    for (const term of bodyTerms) {
+      weightedTf.set(term, (weightedTf.get(term) ?? 0) + BODY_WEIGHT);
+    }
+
+    for (const [term, tf] of weightedTf) {
+      let list = postings.get(term);
+      if (!list) {
+        list = [];
+        postings.set(term, list);
+      }
+      list.push({ doc, weightedTf: tf });
+    }
+  }
+
+  const avgDocLength = docCount > 0 ? totalLength / docCount : 0;
+
+  /** BM25F score per section for the given query terms. */
+  function scoreSections(queryTerms: Set<string>): Map<number, number> {
+    const scores = new Map<number, number>();
+    if (docCount === 0) return scores;
+
+    for (const term of queryTerms) {
+      const list = postings.get(term);
+      if (!list) continue;
+
+      const idf = Math.log(
+        1 + (docCount - list.length + 0.5) / (list.length + 0.5),
+      );
+
+      for (const { doc, weightedTf } of list) {
+        const norm = weightedTf * (k1 + 1);
+        const denom =
+          weightedTf + k1 * (1 - b + b * (docLengths[doc]! / avgDocLength));
+        scores.set(doc, (scores.get(doc) ?? 0) + idf * (norm / denom));
+      }
+    }
+
+    return scores;
+  }
+
+  /** Deterministic order: score desc, then (article, ordinal) asc. */
+  function rankSection(
+    a: number,
+    c: number,
+    scores: Map<number, number>,
+  ): number {
+    const byScore = (scores.get(c) ?? 0) - (scores.get(a) ?? 0);
+    if (byScore !== 0) return byScore;
+    const sa = sections[a]!;
+    const sc = sections[c]!;
+    return sa.article.localeCompare(sc.article) || sa.ordinal - sc.ordinal;
+  }
+
+  function query(
+    text: string,
+    k: number,
+  ): { article: Slug; section: number }[] {
+    if (k <= 0 || docCount === 0) return [];
+
+    const queryTerms = new Set(tokenize(text));
+    const scores = scoreSections(queryTerms);
+    if (scores.size === 0) return [];
+
+    const ranked = [...scores.keys()].sort((a, c) => rankSection(a, c, scores));
+
+    // Dedupe to distinct articles, keeping each article's best (first-ranked)
+    // section, until we have `k` articles.
+    const seen = new Set<Slug>();
+    const result: { article: Slug; section: number }[] = [];
+    for (const doc of ranked) {
+      const article = sections[doc]!.article;
+      if (seen.has(article)) continue;
+      seen.add(article);
+      result.push({ article, section: doc });
+      if (result.length >= k) break;
+    }
+    return result;
+  }
+
+  function bestSection(article: Slug, queryText: string): number {
+    const docs = index.byArticle.get(article);
+    if (!docs || docs.length === 0) return -1;
+
+    const queryTerms = new Set(tokenize(queryText));
+    const scores = scoreSections(queryTerms);
+
+    let best = docs[0]!;
+    let bestScore = scores.get(best) ?? 0;
+    for (const doc of docs) {
+      const score = scores.get(doc) ?? 0;
+      // Strictly-greater keeps the earliest (lowest ordinal) section on ties,
+      // since `docs` is in ascending section order.
+      if (score > bestScore) {
+        best = doc;
+        bestScore = score;
+      }
+    }
+    return best;
+  }
+
+  return { query, bestSection };
+}

package/src/plugins/defaults/memory-v3-shadow/sections.ts

@@ -0,0 +1,115 @@
+import type { Section, SectionIndex, Slug } from "./types.js";
+
+/**
+ * Section-grain index builder for memory-v3 lane retrieval.
+ *
+ * Splits each page body into sections at `## ` headings — text before the
+ * first heading is the page lead (ordinal 0) — and emits a flat, deterministic
+ * `SectionIndex`. Over-long sections are chunked so each `Section.text` fits a
+ * typical embedding window. The function is pure: it does no I/O of its own
+ * (the caller supplies `pageBody`), no LLM calls, and no Qdrant/daemon wiring.
+ */
+
+/**
+ * Max `Section.text` length in characters. Sections longer than this are split
+ * into multiple ordered `Section`s sharing the same `(article, ordinal)` so the
+ * embedding backend never receives an over-window input.
+ */
+export const SECTION_CHUNK_CHARS = 6000;
+
+/** Last `/`- or `.`-delimited segment of a slug (used for the head line). */
+function lastSlugSegment(slug: Slug): string {
+  const segments = slug.split(/[/.]/);
+  return segments[segments.length - 1] ?? slug;
+}
+
+interface RawSection {
+  title: string;
+  body: string;
+}
+
+/**
+ * Split a frontmatter-stripped markdown body into raw sections. The text before
+ * the first `## ` heading is the lead (title `""`); each subsequent `## `
+ * heading starts a new section whose title is the heading text.
+ */
+function splitIntoRawSections(body: string): RawSection[] {
+  // Always seed a lead section (title `""`). The lead may stay empty, so a
+  // headingless or empty page still yields a single ordinal-0 section.
+  const sections: { title: string; lines: string[] }[] = [
+    { title: "", lines: [] },
+  ];
+
+  for (const line of body.split("\n")) {
+    const heading = /^## (.*)$/.exec(line);
+    if (heading) {
+      sections.push({ title: heading[1]!.trim(), lines: [] });
+    } else {
+      sections[sections.length - 1]!.lines.push(line);
+    }
+  }
+
+  return sections.map((s) => ({ title: s.title, body: s.lines.join("\n") }));
+}
+
+/**
+ * Split `text` into chunks no longer than `SECTION_CHUNK_CHARS`, preferring to
+ * break on newlines so chunks stay readable. Order is preserved.
+ */
+function chunkText(text: string): string[] {
+  if (text.length <= SECTION_CHUNK_CHARS) return [text];
+
+  const chunks: string[] = [];
+  let remaining = text;
+  while (remaining.length > SECTION_CHUNK_CHARS) {
+    const window = remaining.slice(0, SECTION_CHUNK_CHARS);
+    const newlineBreak = window.lastIndexOf("\n");
+    // Only break on a newline if it leaves a non-trivial chunk; otherwise hard
+    // split at the window boundary to guarantee forward progress.
+    const breakAt = newlineBreak > 0 ? newlineBreak : SECTION_CHUNK_CHARS;
+    chunks.push(remaining.slice(0, breakAt));
+    remaining = remaining.slice(breakAt).replace(/^\n/, "");
+  }
+  if (remaining.length > 0) chunks.push(remaining);
+  return chunks;
+}
+
+export async function buildSectionIndex(
+  slugs: Slug[],
+  pageBody: (slug: Slug) => Promise<string>,
+): Promise<SectionIndex> {
+  const sections: Section[] = [];
+
+  // Sort slugs so the flat `sections` array is deterministic across runs.
+  for (const article of [...slugs].sort((a, b) => a.localeCompare(b))) {
+    const body = await pageBody(article);
+    const segment = lastSlugSegment(article);
+
+    let ordinal = 0;
+    for (const raw of splitIntoRawSections(body)) {
+      const head = `${segment} — ${raw.title}`;
+      const fullText = `${head}\n${raw.body}`;
+      for (const chunk of chunkText(fullText)) {
+        sections.push({
+          article,
+          title: raw.title,
+          text: chunk,
+          ordinal: ordinal++,
+        });
+      }
+    }
+  }
+
+  const byArticle = new Map<Slug, number[]>();
+  for (let i = 0; i < sections.length; i++) {
+    const article = sections[i]!.article;
+    let indices = byArticle.get(article);
+    if (!indices) {
+      indices = [];
+      byArticle.set(article, indices);
+    }
+    indices.push(i);
+  }
+
+  return { sections, byArticle };
+}