@vellumai/assistant 0.7.3 → 0.8.1

package/src/memory/v2/skill-store.ts

@@ -34,6 +34,7 @@ import {
   embedWithBackend,
   generateSparseEmbedding,
 } from "../embedding-backend.js";
+import { invalidatePageIndex } from "./page-index.js";
 import {
   pruneSlugsWithPrefixExcept,
   upsertConceptPageEmbedding,
@@ -42,6 +43,10 @@ import {
   augmentMcpSetupDescription,
   buildSkillContent,
 } from "./skill-content.js";
+import {
+  generateBm25DocEmbedding,
+  getConceptPageCorpusStats,
+} from "./sparse-bm25.js";
 import type { SkillEntry } from "./types.js";
 
 const log = getLogger("memory-v2-skill-store");
@@ -54,6 +59,14 @@ const log = getLogger("memory-v2-skill-store");
  */
 export const SKILL_SLUG_PREFIX = "skills/";
 
+/**
+ * Payload discriminator written on every skill-seeded Qdrant point. Keeps
+ * skill rows distinguishable from user-authored concept pages that happen to
+ * be slugged under `skills/...`, so prefix pruning never deletes a hand-
+ * authored page sitting in the same namespace.
+ */
+const SKILL_PAYLOAD_KIND = "skill";
+
 /** Compose the unified-collection slug for a skill id. */
 export function skillSlugFor(id: string): string {
   return `${SKILL_SLUG_PREFIX}${id}`;
@@ -68,10 +81,39 @@ let entries: Map<string, SkillEntry> | null = null;
 
 /**
  * Seed (or re-seed) skill embeddings into the unified concept-page collection.
- * Idempotent: safe to call repeatedly. Best-effort: never throws — any
- * failure leaves the prior `entries` cache in place and logs a warning.
+ * Idempotent. Defaults to best-effort (errors are logged but swallowed) for
+ * background callers like daemon startup; pass `{ throwOnError: true }` from
+ * synchronous CLI-driven paths that need to surface failures to the operator.
  *
- * Steps:
+ * Single-flight + coalesced: at most one seed runs at a time, and concurrent
+ * callers are coalesced into one follow-up re-snapshot that runs after the
+ * in-flight seed completes. Without this, an older snapshot can finish after
+ * a newer one and overwrite the newer skill state. Strict callers observe
+ * the most recent run's outcome via `lastSeedError`.
+ */
+let seedTail: Promise<void> = Promise.resolve();
+let seedPending: Promise<void> | null = null;
+let lastSeedError: unknown = null;
+
+export async function seedV2SkillEntries(
+  opts: { throwOnError?: boolean } = {},
+): Promise<void> {
+  if (!seedPending) {
+    const next = seedTail.then(async () => {
+      seedPending = null;
+      await runSeedOnce();
+    });
+    seedTail = next.catch(() => {});
+    seedPending = next;
+  }
+  await seedPending;
+  if (opts.throwOnError && lastSeedError) {
+    throw lastSeedError;
+  }
+}
+
+/**
+ * Steps (per run):
  *   1. Enumerate the local skill catalog and resolve each skill's enabled
  *      state (`resolveSkillStates`).
  *   2. Build a `SkillEntry` per enabled skill, applying the mcp-setup
@@ -90,7 +132,7 @@ let entries: Map<string, SkillEntry> | null = null;
  *      stale points from prior catalog state (e.g. uninstalled skills).
  *   7. Replace the module-level `entries` cache with the freshly built map.
  */
-export async function seedV2SkillEntries(): Promise<void> {
+async function runSeedOnce(): Promise<void> {
   try {
     const config = getConfig();
     const catalog = loadSkillCatalog();
@@ -137,32 +179,53 @@ export async function seedV2SkillEntries(): Promise<void> {
       );
     }
 
-    // Embed all content strings in one batched call. Sparse vectors are
-    // computed in-process (no network).
-    const embedded = await embedWithBackend(
-      config,
-      seeds.map((s) => s.content),
-    );
-    const denseVectors = await Promise.all(
-      embedded.vectors.map((v) =>
-        applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
-      ),
-    );
-
-    const now = Date.now();
+    // Embed all content strings in one batched call when there is anything to
+    // embed. Skipping the call when `seeds` is empty avoids throwing on an
+    // unavailable embedding backend in the all-disabled case, so pruning and
+    // cache replacement still run and clear stale state.
     const nextEntries = new Map<string, SkillEntry>();
-    await Promise.all(
-      seeds.map((seed, i) =>
-        upsertConceptPageEmbedding({
-          slug: skillSlugFor(seed.id),
-          dense: denseVectors[i],
-          sparse: generateSparseEmbedding(seed.content),
-          updatedAt: now,
-        }),
-      ),
-    );
-    for (const seed of seeds) {
-      nextEntries.set(seed.id, seed);
+    if (seeds.length > 0) {
+      const embedded = await embedWithBackend(
+        config,
+        seeds.map((s) => s.content),
+      );
+      const denseVectors = await Promise.all(
+        embedded.vectors.map((v) =>
+          applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
+        ),
+      );
+
+      // Skills share the concept-page Qdrant collection, so the sparse vector
+      // must use the same stemmed BM25 encoding the concept-page documents
+      // carry — otherwise the stemmed BM25 query vectors used by callers (see
+      // `simBatch`, `activation.selectCandidates`, recall) hash to different
+      // buckets than the stored skill vectors and skip the sparse channel
+      // entirely. Fall back to the legacy TF encoder only during the cold-start
+      // window before corpus stats finish building.
+      const corpusStats = getConceptPageCorpusStats();
+      const encodeSparse = (input: string) =>
+        corpusStats
+          ? generateBm25DocEmbedding(input, corpusStats, {
+              k1: config.memory.v2.bm25_k1,
+              b: config.memory.v2.bm25_b,
+            })
+          : generateSparseEmbedding(input);
+
+      const now = Date.now();
+      await Promise.all(
+        seeds.map((seed, i) =>
+          upsertConceptPageEmbedding({
+            slug: skillSlugFor(seed.id),
+            dense: denseVectors[i],
+            sparse: encodeSparse(seed.content),
+            updatedAt: now,
+            kind: SKILL_PAYLOAD_KIND,
+          }),
+        ),
+      );
+      for (const seed of seeds) {
+        nextEntries.set(seed.id, seed);
+      }
     }
 
     // Prune stale skill slugs. When the catalog is unavailable (empty array
@@ -173,6 +236,7 @@ export async function seedV2SkillEntries(): Promise<void> {
       await pruneSlugsWithPrefixExcept(
         SKILL_SLUG_PREFIX,
         seeds.map((s) => s.id),
+        { kind: SKILL_PAYLOAD_KIND },
       );
     } else {
       log.info(
@@ -182,7 +246,13 @@ export async function seedV2SkillEntries(): Promise<void> {
 
     // Atomically replace the cache only after every step above succeeds.
     entries = nextEntries;
+    // Drop the page-index cache so the next router invocation observes the
+    // freshly seeded skill set (skill entries share the unified concept-page
+    // collection and surface in the same index).
+    invalidatePageIndex();
+    lastSeedError = null;
   } catch (err) {
+    lastSeedError = err;
     log.warn({ err }, "Failed to seed v2 skill entries");
   }
 }
@@ -208,7 +278,26 @@ export function isSkillSlug(slug: string): boolean {
   return slug.startsWith(SKILL_SLUG_PREFIX);
 }
 
+/**
+ * Snapshot of the in-process skill cache, sorted by skill id (ASCII order)
+ * for determinism. Returns a freshly allocated array on each call so callers
+ * cannot mutate the underlying cache.
+ *
+ * The cache is replaced atomically by `seedV2SkillEntries`, so a snapshot
+ * may be stale once a subsequent seed run completes. Callers that need
+ * up-to-the-moment state must re-call this after awaiting the seed.
+ */
+export function listSkillEntries(): SkillEntry[] {
+  if (!entries) return [];
+  return [...entries.values()].sort((a, b) =>
+    a.id < b.id ? -1 : a.id > b.id ? 1 : 0,
+  );
+}
+
 /** @internal Test-only: clear the module-level cache. */
 export function _resetSkillStoreForTests(): void {
   entries = null;
+  seedTail = Promise.resolve();
+  seedPending = null;
+  lastSeedError = null;
 }

package/src/memory/v2/static-context.ts

@@ -18,7 +18,6 @@
 // matching the existing PKB auto-inject pattern.
 
 import type { ChannelId } from "../../channels/types.js";
-import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
 import { loadConfig } from "../../config/loader.js";
 import { readPromptFile } from "../../prompts/system-prompt.js";
 import { getWorkspacePromptPath } from "../../util/platform.js";
@@ -36,9 +35,9 @@ const MEMORY_V2_STATIC_BLOCKS: readonly MemoryV2StaticBlock[] = [
 ];
 
 /**
- * Build the v2 static memory block, gated on `memory-v2-enabled` +
- * `config.memory.v2.enabled`. Empty/missing files are skipped; returns
- * `null` when the gate is off or every file is empty.
+ * Build the v2 static memory block, gated on `config.memory.v2.enabled`.
+ * Empty/missing files are skipped; returns `null` when the gate is off or
+ * every file is empty.
  */
 export function readMemoryV2StaticContent(): string | null {
   let config;
@@ -47,10 +46,7 @@ export function readMemoryV2StaticContent(): string | null {
   } catch {
     return null;
   }
-  if (
-    !isAssistantFeatureFlagEnabled("memory-v2-enabled", config) ||
-    !config.memory.v2.enabled
-  ) {
+  if (!config.memory.v2.enabled) {
     return null;
   }
 
@@ -64,19 +60,26 @@ export function readMemoryV2StaticContent(): string | null {
 }
 
 /**
- * Static memory holds the user's aggregate personal pages
- * (essentials/threads/recent/buffer). Block injection when a non-guardian
- * actor reaches the assistant over a remote channel — otherwise the model
- * can be prompt-injected into reciting private memory. Internal flows
- * (`sourceChannel: "vellum"`) and turns with no trust context pass through
- * unchanged; this gate exists only to keep remote untrusted actors out.
+ * Trust-class predicate for personal-memory injection. Personal memory
+ * spans v2 static blocks (essentials/threads/recent/buffer), the PKB
+ * context, and NOW.md — all of which can hold private user content. Block
+ * injection when a non-guardian actor reaches the assistant over a remote
+ * channel — otherwise the model can be prompt-injected into reciting
+ * private memory. Internal flows (`sourceChannel: "vellum"`) and turns
+ * with no trust context pass through unchanged; this gate exists only to
+ * keep remote untrusted actors out.
+ *
+ * This is the trust-only gate. Cadence (first-turn / post-compaction) is
+ * applied separately by the caller so that the freshest content remains
+ * available for re-injection after a mid-turn reducer-triggered compaction
+ * — the initial-injection turn may not have been a `shouldInjectNowAndPkb`
+ * turn, but compaction strips the existing personal-memory blocks and we
+ * still need the freshest content to re-inject.
  */
-export function shouldLoadMemoryV2Static(args: {
-  shouldInjectNowAndPkb: boolean;
+export function shouldExposePersonalMemory(args: {
   sourceChannel: ChannelId | undefined;
   isTrustedActor: boolean;
 }): boolean {
-  if (!args.shouldInjectNowAndPkb) return false;
   const isRemoteUntrustedActor =
     args.sourceChannel !== undefined &&
     args.sourceChannel !== "vellum" &&

package/src/memory/v2/sweep-job.ts

@@ -13,21 +13,24 @@
  * extraction-trigger path. Until then this handler is invoked only by
  * `memory_v2_sweep` rows enqueued explicitly (tests, future CLI).
  *
- * Skipped entirely when the `memory-v2-enabled` feature flag is off, or when
+ * Skipped entirely when `config.memory.v2.enabled` is false, or when
  * `config.memory.v2.sweep_enabled` is false — keeps the sweep dormant in
  * v1-only workspaces and in v2 workspaces that haven't opted in, even if a
- * stale row sits in the queue at flag-flip time.
+ * stale row sits in the queue when v2 is disabled.
  */
 
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
 
-import { desc, gt } from "drizzle-orm";
+import { and, desc, eq, gt, notInArray } from "drizzle-orm";
 import { z } from "zod";
 
-import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-flags.js";
 import type { AssistantConfig } from "../../config/types.js";
-import { getAssistantName } from "../../daemon/identity-helpers.js";
+import {
+  getAssistantName,
+  resolveUserName,
+} from "../../daemon/identity-helpers.js";
+import { emitNotificationSignal } from "../../notifications/emit-signal.js";
 import {
   extractToolUse,
   getConfiguredProvider,
@@ -42,11 +45,15 @@ import {
   formatRememberEntry,
 } from "../graph/tool-handlers.js";
 import type { MemoryJob } from "../jobs-store.js";
-import { messages } from "../schema.js";
+import { stringifyMessageContent } from "../message-content.js";
+import { conversations, messages } from "../schema.js";
 import { renderSweepPrompt } from "./prompts/sweep.js";
 
 const log = getLogger("memory-v2-sweep");
 
+/** Stable job identifier surfaced in `activity.failed` notifications. */
+const JOB_NAME = "memory.v2.sweep";
+
 /** Window of conversation history the sweep inspects on each run. */
 const RECENT_MESSAGES_WINDOW_MS = 30 * 60 * 1000;
 
@@ -101,74 +108,131 @@ const SweepResultSchema = z.object({
  * progress without inspecting the filesystem.
  */
 export async function memoryV2SweepJob(
-  _job: MemoryJob,
+  job: MemoryJob,
   config: AssistantConfig,
 ): Promise<number> {
-  if (!isAssistantFeatureFlagEnabled("memory-v2-enabled", config)) {
-    log.debug("memory-v2-enabled flag off; sweep skipped");
+  if (!config.memory?.v2?.enabled) {
+    log.debug("memory.v2.enabled is false; sweep skipped");
     return 0;
   }
 
-  if (!config.memory?.v2?.sweep_enabled) {
+  if (!config.memory.v2.sweep_enabled) {
     log.debug("memory.v2.sweep_enabled is false; sweep skipped");
     return 0;
   }
 
   const workspaceDir = getWorkspaceDir();
   const memoryDir = join(workspaceDir, "memory");
-  const recentText = loadRecentMessagesText(Date.now());
-  if (!recentText) {
-    log.debug("No recent messages in window; sweep skipped");
-    return 0;
-  }
 
-  const existingBuffer = readBufferText(memoryDir);
+  // Once we're committed to running (past the flag/feature gates), any
+  // unexpected error is surfaced via an `activity.failed` notification —
+  // mirrors v1 filing's post-migration treatment, but hand-rolled because
+  // the sweep makes a single forced-tool `provider.sendMessage` call rather
+  // than driving a conversation through `runBackgroundJob`. The function
+  // continues to return 0 on caught failures (preserving the existing
+  // silent-failure contract); only the notification side-effect is new.
+  try {
+    const recentText = loadRecentMessagesText(Date.now());
+    if (!recentText) {
+      log.debug("No recent messages in window; sweep skipped");
+      return 0;
+    }
+
+    const existingBuffer = readBufferText(memoryDir);
 
-  const provider = await getConfiguredProvider("memoryV2Sweep");
-  if (!provider) {
-    log.warn("memoryV2Sweep provider unavailable; sweep skipped");
-    return 0;
-  }
+    const provider = await getConfiguredProvider("memoryV2Sweep");
+    if (!provider) {
+      log.warn("memoryV2Sweep provider unavailable; sweep skipped");
+      return 0;
+    }
 
-  const systemPrompt = renderSweepPrompt({
-    assistantName: getAssistantName(),
-    userName: resolveUserName(workspaceDir),
-  });
-  const userText =
-    `## existingBuffer\n\n${existingBuffer || "(empty)"}\n\n` +
-    `## recentMessages\n\n${recentText}`;
+    const systemPrompt = renderSweepPrompt({
+      assistantName: getAssistantName(),
+      userName: resolveUserName(workspaceDir),
+    });
+    const userText =
+      `## existingBuffer\n\n${existingBuffer || "(empty)"}\n\n` +
+      `## recentMessages\n\n${recentText}`;
 
-  const response = await provider.sendMessage(
-    [userMessage(userText)],
-    [SWEEP_TOOL],
-    systemPrompt,
-    {
-      config: {
-        callSite: "memoryV2Sweep" as const,
-        tool_choice: { type: "tool" as const, name: SWEEP_TOOL_NAME },
+    const response = await provider.sendMessage(
+      [userMessage(userText)],
+      [SWEEP_TOOL],
+      systemPrompt,
+      {
+        config: {
+          callSite: "memoryV2Sweep" as const,
+          tool_choice: { type: "tool" as const, name: SWEEP_TOOL_NAME },
+        },
       },
-    },
-  );
+    );
+
+    const toolBlock = extractToolUse(response);
+    if (!toolBlock || toolBlock.name !== SWEEP_TOOL_NAME) {
+      log.debug("Sweep model returned no tool_use block");
+      return 0;
+    }
+    const parsed = SweepResultSchema.safeParse(toolBlock.input);
+    if (!parsed.success) {
+      log.warn(
+        { error: parsed.error.message },
+        "Sweep tool input did not match schema",
+      );
+      return 0;
+    }
 
-  const toolBlock = extractToolUse(response);
-  if (!toolBlock || toolBlock.name !== SWEEP_TOOL_NAME) {
-    log.debug("Sweep model returned no tool_use block");
+    const written = appendEntries(memoryDir, parsed.data.entries);
+    if (written > 0) {
+      log.info({ written }, "Memory v2 sweep wrote new buffer entries");
+    }
+    return written;
+  } catch (err) {
+    const errorMessage = err instanceof Error ? err.message : String(err);
+    log.error({ err }, "memory v2 sweep failed");
+    emitSweepActivityFailed({
+      jobId: job.id,
+      errorMessage,
+    });
     return 0;
   }
-  const parsed = SweepResultSchema.safeParse(toolBlock.input);
-  if (!parsed.success) {
+}
+
+/**
+ * Emit an `activity.failed` notification for a failed sweep run. Mirrors
+ * the shape `runBackgroundJob` produces for its own failures so the home
+ * feed and native notifications stay consistent regardless of which code
+ * path executed the work. Fire-and-forget — a notification failure must
+ * never break sweep operation.
+ */
+function emitSweepActivityFailed(args: {
+  jobId: string;
+  errorMessage: string;
+}): void {
+  const day = new Date().toISOString().slice(0, 10);
+  emitNotificationSignal({
+    sourceChannel: "scheduler",
+    sourceContextId: args.jobId,
+    sourceEventName: "activity.failed",
+    dedupeKey: `activity-failed:${JOB_NAME}:${day}`,
+    contextPayload: {
+      jobName: JOB_NAME,
+      errorMessage: args.errorMessage,
+      errorKind: "exception",
+    },
+    attentionHints: {
+      requiresAction: false,
+      urgency: "medium",
+      isAsyncBackground: true,
+      visibleInSourceNow: false,
+    },
+  }).catch((emitErr) => {
     log.warn(
-      { error: parsed.error.message },
-      "Sweep tool input did not match schema",
+      {
+        err: emitErr instanceof Error ? emitErr.message : String(emitErr),
+        jobId: args.jobId,
+      },
+      "Failed to emit activity.failed notification for memory v2 sweep",
     );
-    return 0;
-  }
-
-  const written = appendEntries(memoryDir, parsed.data.entries);
-  if (written > 0) {
-    log.info({ written }, "Memory v2 sweep wrote new buffer entries");
-  }
-  return written;
+  });
 }
 
 /**
@@ -220,14 +284,22 @@ function loadRecentMessagesText(nowMs: number): string {
   const db = getDb();
   // Pull newest-first then reverse for chronological output. Bounding the
   // initial limit (1000) defends against pathological busy windows where a
-  // naive scan would touch every recent message.
+  // naive scan would touch every recent message. Joining conversations and
+  // excluding background/scheduled types keeps automation chatter
+  // (heartbeats, filing, update bulletins, scheduled jobs) out of buffer.md.
   const rows = db
     .select({
       role: messages.role,
       content: messages.content,
     })
     .from(messages)
-    .where(gt(messages.createdAt, cutoff))
+    .innerJoin(conversations, eq(messages.conversationId, conversations.id))
+    .where(
+      and(
+        gt(messages.createdAt, cutoff),
+        notInArray(conversations.conversationType, ["background", "scheduled"]),
+      ),
+    )
     .orderBy(desc(messages.createdAt))
     .limit(1000)
     .all();
@@ -249,50 +321,3 @@ function loadRecentMessagesText(nowMs: number): string {
   }
   return joined;
 }
-
-/**
- * Coerce stored message content (JSON-serialized `ContentBlock[]` *or* plain
- * string in legacy rows) into a single text string. Image / tool blocks are
- * dropped — the sweep model only needs the spoken context.
- */
-function stringifyMessageContent(stored: string): string {
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(stored);
-  } catch {
-    return stored.trim();
-  }
-  if (typeof parsed === "string") return parsed.trim();
-  if (!Array.isArray(parsed)) return "";
-  const parts: string[] = [];
-  for (const block of parsed) {
-    if (
-      block &&
-      typeof block === "object" &&
-      (block as { type?: string }).type === "text" &&
-      typeof (block as { text?: unknown }).text === "string"
-    ) {
-      parts.push((block as { text: string }).text);
-    }
-  }
-  return parts.join("\n").trim();
-}
-
-/**
- * Read the guardian's display name from `users/default.md`. We look for the
- * markdown-bold "Name" label (matching the IDENTITY.md convention) and fall
- * back to `null` on any miss; the prompt template substitutes a generic
- * label.
- */
-function resolveUserName(workspaceDir: string): string | null {
-  try {
-    const content = readFileSync(
-      join(workspaceDir, "users", "default.md"),
-      "utf-8",
-    );
-    const match = content.match(/\*\*Name:\*\*\s*(.+)/);
-    return match?.[1]?.trim() || null;
-  } catch {
-    return null;
-  }
-}

package/src/memory/v2/types.ts

@@ -25,12 +25,23 @@ import { z } from "zod";
  * `edges:` means "activating A pulls in B" — activation flows A → B but not
  * B → A. The full graph is the union of every page's `edges:` list — there
  * is no separate edges-index file. `ref_files` lists paths to attached media
- * (images, audio, etc.).
+ * (images, audio, etc.). `ref_urls` lists external URL references (e.g.
+ * citations, source links).
+ *
+ * `summary` is a 1-4 sentence prose description of the page. When present,
+ * retrieval injects the path + summary instead of the full page so the agent
+ * can decide whether to read the file. Optional because legacy pages predating
+ * the summary field still parse — those fall back to full-page injection and
+ * full-page-only similarity.
  */
-export const ConceptPageFrontmatterSchema = z.object({
-  edges: z.array(z.string()).default([]),
-  ref_files: z.array(z.string()).default([]),
-});
+export const ConceptPageFrontmatterSchema = z
+  .object({
+    edges: z.array(z.string()).default([]),
+    ref_files: z.array(z.string()).default([]),
+    ref_urls: z.array(z.string().url()).default([]),
+    summary: z.string().optional(),
+  })
+  .strict();
 
 export type ConceptPageFrontmatter = z.infer<
   typeof ConceptPageFrontmatterSchema

package/src/memory/validation.ts

@@ -8,3 +8,16 @@
 export function clampUnitInterval(value: number): number {
   return Math.min(1, Math.max(0, value));
 }
+
+/**
+ * Map cosine similarity [-1, 1] into the unit interval [0, 1] via
+ * `(x + 1) / 2`, then clamp. Single-channel display normalization for the
+ * legacy v1 semantic-search path; do **not** use before hybrid fusion —
+ * the affine rescale halves pairwise dense differences and shifts ranking
+ * toward sparse. Hybrid fusion (`v2/sim.ts`) instead clamps negative
+ * cosines to 0 with `Math.max(0, x)` and lets positive cosines pass
+ * through unchanged.
+ */
+export function mapCosineToUnit(value: number): number {
+  return clampUnitInterval((value + 1) / 2);
+}