@vellumai/assistant 0.7.0 → 0.7.2

package/src/memory/v2/sparse-bm25.ts

@@ -0,0 +1,245 @@
+// ---------------------------------------------------------------------------
+// Memory v2 — BM25 sparse channel
+// ---------------------------------------------------------------------------
+//
+// Replaces the legacy TF-only sparse embedding (`generateSparseEmbedding` in
+// `../embedding-backend.ts`) with a real Okapi BM25 implementation. Common
+// words like "i", "am", "the" no longer dominate sparse matching the way they
+// did when every token was weighted equally.
+//
+// BM25 score for document `d` and query `q`:
+//
+//   score(d, q) = Σ_t∈q  IDF(t) · TF_sat(d, t)
+//   TF_sat(d, t) = tf(d, t) · (k1 + 1)
+//                / (tf(d, t) + k1 · (1 - b + b · |d| / avg_dl))
+//   IDF(t)       = log( (N - df(t) + 0.5) / (df(t) + 0.5) + 1 )
+//
+// `+1` inside the IDF log keeps the result non-negative even when df(t) > N/2,
+// matching the variant Lucene uses for `BM25Similarity`.
+//
+// **Asymmetric encoding**: documents carry the full BM25 weight per token
+// (IDF · TF_sat baked into the stored vector), and queries carry binary
+// occurrence per token. Qdrant's sparse dot product then reduces to the BM25
+// score directly. Putting BM25 on the doc side means the weights need
+// recomputing whenever the corpus DF or avg_dl changes — operators trigger
+// that with `assistant memory v2 reembed` after major content shifts.
+
+import { readFile } from "node:fs/promises";
+
+import type { SparseEmbedding } from "../embedding-types.js";
+import {
+  SPARSE_VOCAB_SIZE,
+  tokenHash,
+  tokenizeStemmed,
+} from "../sparse-tokenize.js";
+import { listPages } from "./page-store.js";
+
+/**
+ * Aggregate corpus statistics used to weight a BM25 document vector. Held in
+ * memory after a startup walk over `memory/concepts/`.
+ */
+export interface CorpusStats {
+  /** Total document count over which DF was accumulated. */
+  totalDocs: number;
+  /** hashedTokenIndex (in `[0, SPARSE_VOCAB_SIZE)`) → distinct-doc count. */
+  df: Map<number, number>;
+  /** Average document length in tokens, post-tokenize. */
+  avgDl: number;
+  /** Wall-clock millis at build time — used by diagnostics, not the formula. */
+  builtAt: number;
+}
+
+/** BM25 hyperparameters. Standard Lucene/Elasticsearch defaults. */
+export interface Bm25Params {
+  /** TF saturation curve. ~1.2 is standard. */
+  k1: number;
+  /** Length normalization. 0 = none, 1 = full. ~0.75 is standard. */
+  b: number;
+}
+
+let _conceptPageStats: CorpusStats | null = null;
+
+/**
+ * Latest in-memory corpus stats for `memory/concepts/`, or `null` if a build
+ * has not yet completed. Callers must handle `null` and fall back to legacy
+ * TF-only behavior so the daemon remains usable during the brief startup
+ * window before {@link rebuildConceptPageCorpusStats} finishes.
+ */
+export function getConceptPageCorpusStats(): CorpusStats | null {
+  return _conceptPageStats;
+}
+
+/**
+ * Walk every concept page on disk, accumulate document frequency per hashed
+ * token bucket, and average document length. Atomically swaps the result into
+ * the module-level cache when the walk succeeds. On error the previous stats
+ * stay live.
+ *
+ * Reads bodies via `readPage`-equivalent direct file reads to avoid paying for
+ * frontmatter parsing on every page (we only need the body for sparse).
+ */
+export async function rebuildConceptPageCorpusStats(
+  workspaceDir: string,
+): Promise<void> {
+  const slugs = await listPages(workspaceDir);
+  if (slugs.length === 0) {
+    _conceptPageStats = {
+      totalDocs: 0,
+      df: new Map(),
+      avgDl: 0,
+      builtAt: Date.now(),
+    };
+    return;
+  }
+
+  const df = new Map<number, number>();
+  let totalTokens = 0;
+  let docsCounted = 0;
+
+  for (const slug of slugs) {
+    const body = await readPageBodyForStats(workspaceDir, slug);
+    if (body === null) continue;
+    const tokens = tokenizeStemmed(body);
+    if (tokens.length === 0) continue;
+    totalTokens += tokens.length;
+    docsCounted += 1;
+    const seen = new Set<number>();
+    for (const token of tokens) {
+      const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+      if (seen.has(idx)) continue;
+      seen.add(idx);
+      df.set(idx, (df.get(idx) ?? 0) + 1);
+    }
+  }
+
+  _conceptPageStats = {
+    totalDocs: docsCounted,
+    df,
+    avgDl: docsCounted > 0 ? totalTokens / docsCounted : 0,
+    builtAt: Date.now(),
+  };
+}
+
+/**
+ * Read just the body of a page for stats accumulation. Skips the YAML
+ * frontmatter without invoking the schema-validating `readPage` parser, since
+ * any parse failure surfaced there would abort the whole rebuild — and we
+ * only need the prose half for tokenization.
+ */
+async function readPageBodyForStats(
+  workspaceDir: string,
+  slug: string,
+): Promise<string | null> {
+  const path = `${workspaceDir}/memory/concepts/${slug}.md`;
+  let raw: string;
+  try {
+    raw = await readFile(path, "utf-8");
+  } catch {
+    return null;
+  }
+  // Strip a leading `---\n...\n---\n` block if present; otherwise return raw.
+  if (raw.startsWith("---")) {
+    const closing = raw.indexOf("\n---", 3);
+    if (closing !== -1) {
+      const after = raw.indexOf("\n", closing + 4);
+      if (after !== -1) return raw.slice(after + 1);
+    }
+  }
+  return raw;
+}
+
+/**
+ * Compute the BM25 IDF weight for a hashed token bucket. Returns `0` when the
+ * token's df equals the corpus size (a token in every document carries no
+ * discriminating power).
+ */
+function computeIdf(stats: CorpusStats, hashIdx: number): number {
+  const df = stats.df.get(hashIdx) ?? 0;
+  const numerator = stats.totalDocs - df + 0.5;
+  const denominator = df + 0.5;
+  return Math.log(numerator / denominator + 1);
+}
+
+/**
+ * Document-side BM25-weighted sparse vector. Each emitted value is
+ * `IDF(t) · TF_sat(d, t)` so the dot product against a binary query vector
+ * (see {@link generateBm25QueryEmbedding}) yields the BM25 score.
+ *
+ * Returns an empty embedding for empty input or when the corpus is empty
+ * (every IDF would be zero anyway).
+ */
+export function generateBm25DocEmbedding(
+  text: string,
+  stats: CorpusStats,
+  params: Bm25Params,
+): SparseEmbedding {
+  const tokens = tokenizeStemmed(text);
+  if (tokens.length === 0 || stats.totalDocs === 0) {
+    return { indices: [], values: [] };
+  }
+
+  // Per-document term frequencies, keyed by hashed bucket.
+  const tf = new Map<number, number>();
+  for (const token of tokens) {
+    const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+    tf.set(idx, (tf.get(idx) ?? 0) + 1);
+  }
+
+  const docLen = tokens.length;
+  // avg_dl can be 0 only when totalDocs is 0, which we already short-circuited.
+  const lengthFactor = 1 - params.b + (params.b * docLen) / stats.avgDl;
+  const indices: number[] = [];
+  const values: number[] = [];
+
+  for (const [idx, freq] of tf) {
+    const idf = computeIdf(stats, idx);
+    if (idf === 0) continue; // Skip tokens that contribute nothing to scores.
+    const saturated =
+      (freq * (params.k1 + 1)) / (freq + params.k1 * lengthFactor);
+    const weight = idf * saturated;
+    if (weight === 0) continue;
+    indices.push(idx);
+    values.push(weight);
+  }
+
+  return { indices, values };
+}
+
+/**
+ * Query-side sparse vector — binary occurrence per distinct query token. The
+ * dot product `Σ_t v_q(t) · v_d(t)` against a BM25-weighted document vector
+ * is exactly the BM25 score, since `v_q(t) = 1` for tokens in the query and
+ * `0` otherwise.
+ *
+ * Stateless — does not need corpus stats, so callers can use this on every
+ * turn without coordinating with {@link rebuildConceptPageCorpusStats}.
+ */
+export function generateBm25QueryEmbedding(text: string): SparseEmbedding {
+  const tokens = tokenizeStemmed(text);
+  if (tokens.length === 0) {
+    return { indices: [], values: [] };
+  }
+
+  const seen = new Set<number>();
+  const indices: number[] = [];
+  const values: number[] = [];
+  for (const token of tokens) {
+    const idx = tokenHash(token, SPARSE_VOCAB_SIZE);
+    if (seen.has(idx)) continue;
+    seen.add(idx);
+    indices.push(idx);
+    values.push(1);
+  }
+
+  return { indices, values };
+}
+
+/** @internal Test-only: reset module-level singletons. */
+export function _resetCorpusStatsForTests(): void {
+  _conceptPageStats = null;
+}
+
+/** @internal Test-only: install a fixture stats table without disk I/O. */
+export function _setCorpusStatsForTests(stats: CorpusStats | null): void {
+  _conceptPageStats = stats;
+}

package/src/memory/v2/static-context.ts

@@ -0,0 +1,63 @@
+// ---------------------------------------------------------------------------
+// Memory v2 — Static context loader for user-message auto-injection
+// ---------------------------------------------------------------------------
+//
+// Reads the four top-level memory files (essentials/threads/recent/buffer)
+// and returns a concatenated, header-wrapped block ready to splice into the
+// current user message via the injector chain.
+//
+// Pairs with the v2 per-turn activation block (`maybeRouteV2Injection` in
+// `conversation-graph-memory.ts`, which threads through `injectTextBlock`)
+// — that block carries activated concept pages selected by the activation
+// pipeline; this static block carries the always-relevant aggregate views
+// written by consolidation and the user. Both land on the user message so
+// the system prompt stays cache-stable.
+//
+// Refresh cadence is owned by the caller: the agent loop only passes the
+// content through when `mode === "full"` (first turn / post-compaction),
+// matching the existing PKB auto-inject pattern.
+
+import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
+import { loadConfig } from "../../config/loader.js";
+import { readPromptFile } from "../../prompts/system-prompt.js";
+import { getWorkspacePromptPath } from "../../util/platform.js";
+
+interface MemoryV2StaticBlock {
+  heading: string;
+  file: string;
+}
+
+const MEMORY_V2_STATIC_BLOCKS: readonly MemoryV2StaticBlock[] = [
+  { heading: "## Essentials", file: "memory/essentials.md" },
+  { heading: "## Threads", file: "memory/threads.md" },
+  { heading: "## Recent", file: "memory/recent.md" },
+  { heading: "## Buffer", file: "memory/buffer.md" },
+];
+
+/**
+ * Build the v2 static memory block, gated on `memory-v2-enabled` +
+ * `config.memory.v2.enabled`. Empty/missing files are skipped; returns
+ * `null` when the gate is off or every file is empty.
+ */
+export function readMemoryV2StaticContent(): string | null {
+  let config;
+  try {
+    config = loadConfig();
+  } catch {
+    return null;
+  }
+  if (
+    !isAssistantFeatureFlagEnabled("memory-v2-enabled", config) ||
+    !config.memory.v2.enabled
+  ) {
+    return null;
+  }
+
+  const sections: string[] = [];
+  for (const { heading, file } of MEMORY_V2_STATIC_BLOCKS) {
+    const content = readPromptFile(getWorkspacePromptPath(file));
+    if (!content) continue;
+    sections.push(`${heading}\n\n${content}`);
+  }
+  return sections.length > 0 ? sections.join("\n\n") : null;
+}

package/src/memory/v2/types.ts

@@ -20,9 +20,12 @@ import { z } from "zod";
 /**
  * YAML frontmatter at the top of a concept page (`memory/concepts/<slug>.md`).
  *
- * `edges` is a derived view of `memory/edges.json` — it is regenerated by
- * the rebuild-edges backfill job and is not the source of truth for the graph
- * topology. `ref_files` lists paths to attached media (images, audio, etc.).
+ * `edges` is the canonical list of *outgoing* directed edges from this page.
+ * Each entry is the slug of another concept page; an entry of `B` in A's
+ * `edges:` means "activating A pulls in B" — activation flows A → B but not
+ * B → A. The full graph is the union of every page's `edges:` list — there
+ * is no separate edges-index file. `ref_files` lists paths to attached media
+ * (images, audio, etc.).
  */
 export const ConceptPageFrontmatterSchema = z.object({
   edges: z.array(z.string()).default([]),
@@ -34,8 +37,10 @@ export type ConceptPageFrontmatter = z.infer<
 >;
 
 /**
- * A single concept page on disk. The slug is the filename minus `.md` and
- * also the stable identity used in edges and activation state.
+ * A single concept page on disk. The slug is the relative path from
+ * `memory/concepts/` minus `.md`, using forward slashes — so `alice` and
+ * `people/alice` are both valid slugs. The slug is the stable identity used
+ * in edges and activation state.
  */
 export const ConceptPageSchema = z.object({
   slug: z.string(),
@@ -45,21 +50,6 @@ export const ConceptPageSchema = z.object({
 
 export type ConceptPage = z.infer<typeof ConceptPageSchema>;
 
-// ---------------------------------------------------------------------------
-// Edges index (`memory/edges.json`)
-// ---------------------------------------------------------------------------
-
-/**
- * Source of truth for v2 graph topology. Edges are unweighted, undirected,
- * and stored as canonicalized 2-tuples (alphabetical-first slug first).
- */
-export const EdgesIndexSchema = z.object({
-  version: z.literal(1),
-  edges: z.array(z.tuple([z.string(), z.string()])),
-});
-
-export type EdgesIndex = z.infer<typeof EdgesIndexSchema>;
-
 // ---------------------------------------------------------------------------
 // Activation state (per-conversation, persisted in SQLite)
 // ---------------------------------------------------------------------------

package/src/memory/validation.ts

@@ -1,19 +1,8 @@
-import { z } from "zod";
-
 /**
  * Unit interval [0, 1] — used for confidence and importance fields on memory items.
  * Coerces out-of-range numbers to the nearest bound rather than rejecting,
  * since LLM-generated values occasionally exceed the range.
  */
-const unitInterval = z
-  .number()
-  .transform((v) => Math.min(1, Math.max(0, v)));
-
-/** Zod schema for validating confidence/importance values on memory items. */
-export const memoryItemScores = z.object({
-  confidence: unitInterval,
-  importance: unitInterval,
-});
 
 /** Clamp a numeric value to [0, 1]. */
 export function clampUnitInterval(value: number): number {

package/src/messaging/draft-store.ts

@@ -62,12 +62,6 @@ export function createDraft(opts: {
   return draft;
 }
 
-export function getDraft(platform: string, id: string): Draft | null {
-  const path = getDraftPath(platform, id);
-  if (!pathExists(path)) return null;
-  return JSON.parse(readFileSync(path, "utf-8")) as Draft;
-}
-
 export function listDrafts(platform: string): Draft[] {
   const dir = getDraftsDir(platform);
   const files = readdirSync(dir).filter((f) => f.endsWith(".json"));

package/src/messaging/provider-types.ts

@@ -41,6 +41,12 @@ export interface SearchResult {
   nextCursor?: string;
 }
 
+export interface HistoryPageResult {
+  messages: Message[];
+  hasMore: boolean;
+  nextCursor?: string;
+}
+
 export interface SendResult {
   id: string;
   timestamp: number;
@@ -66,6 +72,8 @@ export interface HistoryOptions {
   limit?: number;
   before?: string;
   after?: string;
+  cursor?: string;
+  inclusive?: boolean;
 }
 
 export interface SearchOptions {

package/src/messaging/provider.ts

@@ -11,6 +11,7 @@ import type {
   ConnectionInfo,
   Conversation,
   HistoryOptions,
+  HistoryPageResult,
   ListOptions,
   Message,
   SearchOptions,
@@ -60,6 +61,12 @@ export interface MessagingProvider {
     threadId: string,
     options?: HistoryOptions,
   ): Promise<Message[]>;
+  getThreadRepliesPage?(
+    connection: OAuthConnection | undefined,
+    conversationId: string,
+    threadId: string,
+    options?: HistoryOptions,
+  ): Promise<HistoryPageResult>;
   markRead?(
     connection: OAuthConnection | undefined,
     conversationId: string,

package/src/messaging/providers/gmail/client.ts

@@ -3,12 +3,7 @@ import type {
   OAuthConnectionResponse,
 } from "../../../oauth/connection.js";
 import type {
-  GmailAttachment,
   GmailDraft,
-  GmailFilter,
-  GmailFilterAction,
-  GmailFilterCriteria,
-  GmailFiltersListResponse,
   GmailLabel,
   GmailLabelsListResponse,
   GmailMessage,
@@ -17,7 +12,6 @@ import type {
   GmailModifyRequest,
   GmailProfile,
   GmailThread,
-  GmailVacationSettings,
 } from "./types.js";
 
 const GMAIL_BATCH_URL = "https://www.googleapis.com/batch/gmail/v1";
@@ -237,7 +231,7 @@ export async function listMessages(
 }
 
 /** Get a single message by ID. */
-export async function getMessage(
+async function getMessage(
   connection: OAuthConnection,
   messageId: string,
   format: GmailMessageFormat = "full",
@@ -591,17 +585,6 @@ export async function batchModifyMessages(
   });
 }
 
-/** Move a message to trash. */
-export async function trashMessage(
-  connection: OAuthConnection,
-  messageId: string,
-): Promise<GmailMessage> {
-  return request<GmailMessage>(connection, `/messages/${messageId}/trash`, {
-    method: "POST",
-    retryable: true,
-  });
-}
-
 /** List all labels. */
 export async function listLabels(
   connection: OAuthConnection,
@@ -659,17 +642,6 @@ export async function createDraftRaw(
   });
 }
 
-/** Send an existing draft by ID. */
-export async function sendDraft(
-  connection: OAuthConnection,
-  draftId: string,
-): Promise<GmailMessage> {
-  return request<GmailMessage>(connection, "/drafts/send", {
-    method: "POST",
-    body: JSON.stringify({ id: draftId }),
-  });
-}
-
 /** Send an email. */
 export async function sendMessage(
   connection: OAuthConnection,
@@ -707,95 +679,3 @@ export async function getProfile(
 ): Promise<GmailProfile> {
   return request<GmailProfile>(connection, "/profile");
 }
-
-/** Get attachment data for a message. */
-export async function getAttachment(
-  connection: OAuthConnection,
-  messageId: string,
-  attachmentId: string,
-): Promise<GmailAttachment> {
-  return request<GmailAttachment>(
-    connection,
-    `/messages/${messageId}/attachments/${attachmentId}`,
-  );
-}
-
-/** Send an email with a pre-built raw MIME payload (for multipart/attachments). */
-export async function sendMessageRaw(
-  connection: OAuthConnection,
-  raw: string,
-  threadId?: string,
-): Promise<GmailMessage> {
-  const payload: Record<string, unknown> = { raw };
-  if (threadId) payload.threadId = threadId;
-  return request<GmailMessage>(connection, "/messages/send", {
-    method: "POST",
-    body: JSON.stringify(payload),
-  });
-}
-
-/** Create a user label. */
-export async function createLabel(
-  connection: OAuthConnection,
-  name: string,
-  opts?: {
-    messageListVisibility?: "show" | "hide";
-    labelListVisibility?: "labelShow" | "labelShowIfUnread" | "labelHide";
-  },
-): Promise<GmailLabel> {
-  return request<GmailLabel>(connection, "/labels", {
-    method: "POST",
-    body: JSON.stringify({ name, ...opts }),
-  });
-}
-
-/** List all Gmail filters. */
-export async function listFilters(
-  connection: OAuthConnection,
-): Promise<GmailFilter[]> {
-  const resp = await request<GmailFiltersListResponse>(
-    connection,
-    "/settings/filters",
-  );
-  return resp.filter ?? [];
-}
-
-/** Create a Gmail filter. */
-export async function createFilter(
-  connection: OAuthConnection,
-  criteria: GmailFilterCriteria,
-  action: GmailFilterAction,
-): Promise<GmailFilter> {
-  return request<GmailFilter>(connection, "/settings/filters", {
-    method: "POST",
-    body: JSON.stringify({ criteria, action }),
-  });
-}
-
-/** Delete a Gmail filter. */
-export async function deleteFilter(
-  connection: OAuthConnection,
-  filterId: string,
-): Promise<void> {
-  await request<void>(connection, `/settings/filters/${filterId}`, {
-    method: "DELETE",
-  });
-}
-
-/** Get vacation auto-reply settings. */
-export async function getVacation(
-  connection: OAuthConnection,
-): Promise<GmailVacationSettings> {
-  return request<GmailVacationSettings>(connection, "/settings/vacation");
-}
-
-/** Update vacation auto-reply settings. */
-export async function updateVacation(
-  connection: OAuthConnection,
-  settings: GmailVacationSettings,
-): Promise<GmailVacationSettings> {
-  return request<GmailVacationSettings>(connection, "/settings/vacation", {
-    method: "PUT",
-    body: JSON.stringify(settings),
-  });
-}

package/src/messaging/providers/gmail/types.ts

@@ -95,52 +95,3 @@ export interface GmailModifyRequest {
 
 /** Message format for GET requests */
 export type GmailMessageFormat = "minimal" | "full" | "raw" | "metadata";
-
-/** Attachment data from the Gmail API */
-export interface GmailAttachment {
-  size: number;
-  data: string; // base64url-encoded
-}
-
-/** Gmail filter criteria */
-export interface GmailFilterCriteria {
-  from?: string;
-  to?: string;
-  subject?: string;
-  query?: string;
-  hasAttachment?: boolean;
-  excludeChats?: boolean;
-  size?: number;
-  sizeComparison?: "larger" | "smaller";
-}
-
-/** Gmail filter action */
-export interface GmailFilterAction {
-  addLabelIds?: string[];
-  removeLabelIds?: string[];
-  forward?: string;
-}
-
-/** Gmail filter */
-export interface GmailFilter {
-  id: string;
-  criteria: GmailFilterCriteria;
-  action: GmailFilterAction;
-}
-
-/** Gmail filters list response */
-export interface GmailFiltersListResponse {
-  filter?: GmailFilter[];
-}
-
-/** Gmail vacation auto-reply settings */
-export interface GmailVacationSettings {
-  enableAutoReply: boolean;
-  responseSubject?: string;
-  responseBodyPlainText?: string;
-  responseBodyHtml?: string;
-  restrictToContacts?: boolean;
-  restrictToDomain?: boolean;
-  startTime?: string; // epoch ms as string
-  endTime?: string; // epoch ms as string
-}