@vellumai/assistant 0.7.3 → 0.8.1

package/src/memory/context-search/search.ts

@@ -1,3 +1,4 @@
+import { isAbortReason } from "../../util/abort-reasons.js";
 import {
   ALL_RECALL_SOURCES,
   type NormalizedRecallInput,
@@ -30,23 +31,12 @@ export interface DeterministicRecallSearchResult extends RecallSearchResult {
 
 export interface DeterministicRecallSearchOptions {
   adapters?: readonly RecallSourceAdapter[];
-  readPkbContextEvidence?: (
-    context: RecallSearchContext,
-  ) => Promise<readonly RecallEvidence[]> | readonly RecallEvidence[];
 }
 
 const SOURCE_PRIORITY = new Map<RecallSource, number>(
   ALL_RECALL_SOURCES.map((source, index) => [source, index]),
 );
 
-const AUTO_INJECTED_PKB_CONTEXT_EVIDENCE_IDS = new Set([
-  "pkb:auto-inject",
-  "pkb:NOW.md",
-]);
-const AUTO_INJECTED_PKB_CONTEXT_TEXT_CAP = Math.floor(
-  RECALL_EVIDENCE_TEXT_CAP_PER_SOURCE / 2,
-);
-
 export async function runDeterministicRecallSearch(
   input: RecallInput,
   context: RecallSearchContext,
@@ -64,18 +54,6 @@ export async function runDeterministicRecallSearch(
   );
   const errorsBySource = new Map<RecallSource, string>();
 
-  if (normalizedInput.sources.includes("pkb")) {
-    try {
-      const contextEvidence = await readPkbContextEvidenceForSearch(
-        context,
-        options,
-      );
-      appendEvidence(evidenceBySource, "pkb", contextEvidence);
-    } catch (err) {
-      errorsBySource.set("pkb", errorToMessage(err));
-    }
-  }
-
   const selectedAdapters = normalizedInput.sources.flatMap((source) => {
     const adapter = adapterBySource.get(source);
     return adapter ? [adapter] : [];
@@ -88,17 +66,20 @@ export async function runDeterministicRecallSearch(
     ),
   );
 
-  adapterResults.forEach((settledResult, index) => {
+  for (const [index, settledResult] of adapterResults.entries()) {
     const source = selectedAdapters[index]?.source;
-    if (!source) return;
+    if (!source) continue;
 
     if (settledResult.status === "fulfilled") {
       appendEvidence(evidenceBySource, source, settledResult.value.evidence);
-      return;
+      continue;
     }
 
+    if (isAbortLikeError(settledResult.reason)) {
+      throw settledResult.reason;
+    }
     errorsBySource.set(source, errorToMessage(settledResult.reason));
-  });
+  }
 
   const evidence = capEvidence(
     dedupeEvidence(sortEvidence([...evidenceBySource.values()].flat())),
@@ -142,10 +123,6 @@ async function loadDefaultRecallSourceAdapter(
       const { searchMemorySource } = await import("./sources/memory.js");
       return { source, search: searchMemorySource };
     }
-    case "pkb": {
-      const { searchPkbSource } = await import("./sources/pkb.js");
-      return { source, search: searchPkbSource };
-    }
     case "conversations": {
       const { searchConversationSource } =
         await import("./sources/conversations.js");
@@ -156,21 +133,6 @@ async function loadDefaultRecallSourceAdapter(
       return { source, search: searchWorkspaceSource };
     }
   }
-
-  const exhaustiveSource: never = source;
-  throw new Error(`Unknown recall source: ${String(exhaustiveSource)}`);
-}
-
-async function readPkbContextEvidenceForSearch(
-  context: RecallSearchContext,
-  options: DeterministicRecallSearchOptions,
-): Promise<readonly RecallEvidence[]> {
-  if (options.readPkbContextEvidence) {
-    return options.readPkbContextEvidence(context);
-  }
-
-  const { readPkbContextEvidence } = await import("./sources/pkb.js");
-  return readPkbContextEvidence(context);
 }
 
 function appendEvidence(
@@ -221,7 +183,7 @@ function retrievalRankBoost(item: RecallEvidence): number {
   if (retrieval === "path") return 0.45;
   if (retrieval === "structured-json") return 0.4;
   if (retrieval === "section") return 0.35;
-  if (retrieval === "lexical") return item.source === "pkb" ? 1.5 : 0.25;
+  if (retrieval === "lexical") return 0.25;
   return 0;
 }
 
@@ -253,17 +215,11 @@ function capEvidence(
   evidence: readonly RecallEvidence[],
   maxResults: number,
 ): RecallEvidence[] {
-  const autoInjectedContextEvidence = evidence.filter(
-    isAutoInjectedPkbContextEvidence,
-  );
-  const regularEvidence = evidence.filter(
-    (item) => !isAutoInjectedPkbContextEvidence(item),
-  );
   const capped: RecallEvidence[] = [];
   const textSizeBySource = new Map<RecallSource, number>();
   let totalTextSize = 0;
 
-  for (const item of regularEvidence) {
+  for (const item of evidence) {
     if (capped.length >= maxResults) {
       break;
     }
@@ -277,79 +233,21 @@ function capEvidence(
     if (appended.totalRemaining <= 0) break;
   }
 
-  for (let index = 0; index < autoInjectedContextEvidence.length; index += 1) {
-    if (capped.length >= maxResults) {
-      break;
-    }
-
-    const item = autoInjectedContextEvidence[index];
-    if (!item) continue;
-
-    const autoInjectedRemaining = autoInjectedContextEvidence.length - index;
-    const reservedTextBudget = getReservedAutoInjectedTextBudget(item, {
-      textSizeBySource,
-      totalTextSize,
-      autoInjectedRemaining,
-    });
-    const appended = appendCappedEvidence(item, {
-      capped,
-      textSizeBySource,
-      totalTextSize,
-      textBudget: reservedTextBudget,
-    });
-    totalTextSize = appended.totalTextSize;
-    if (appended.totalRemaining <= 0) {
-      break;
-    }
-  }
-
   return capped;
 }
 
-export function isAutoInjectedPkbContextEvidence(
-  item: RecallEvidence,
-): boolean {
-  return (
-    item.source === "pkb" && AUTO_INJECTED_PKB_CONTEXT_EVIDENCE_IDS.has(item.id)
-  );
-}
-
-function getReservedAutoInjectedTextBudget(
-  item: RecallEvidence,
-  state: {
-    textSizeBySource: Map<RecallSource, number>;
-    totalTextSize: number;
-    autoInjectedRemaining: number;
-  },
-): number {
-  const sourceTextSize = state.textSizeBySource.get(item.source) ?? 0;
-  const autoInjectedSourceCap =
-    item.source === "pkb"
-      ? AUTO_INJECTED_PKB_CONTEXT_TEXT_CAP
-      : RECALL_EVIDENCE_TEXT_CAP_PER_SOURCE;
-  const sourceRemaining = autoInjectedSourceCap - sourceTextSize;
-  const totalRemaining = RECALL_TOTAL_EVIDENCE_TEXT_CAP - state.totalTextSize;
-  const remaining = Math.min(sourceRemaining, totalRemaining);
-  return Math.max(1, Math.floor(remaining / state.autoInjectedRemaining));
-}
-
 function appendCappedEvidence(
   item: RecallEvidence,
   state: {
     capped: RecallEvidence[];
     textSizeBySource: Map<RecallSource, number>;
     totalTextSize: number;
-    textBudget?: number;
   },
 ): { totalTextSize: number; totalRemaining: number } {
   const sourceTextSize = state.textSizeBySource.get(item.source) ?? 0;
   const sourceRemaining = RECALL_EVIDENCE_TEXT_CAP_PER_SOURCE - sourceTextSize;
   const totalRemaining = RECALL_TOTAL_EVIDENCE_TEXT_CAP - state.totalTextSize;
-  const remaining = Math.min(
-    sourceRemaining,
-    totalRemaining,
-    state.textBudget ?? Number.MAX_SAFE_INTEGER,
-  );
+  const remaining = Math.min(sourceRemaining, totalRemaining);
   if (remaining <= 0) {
     return { totalTextSize: state.totalTextSize, totalRemaining };
   }
@@ -385,3 +283,15 @@ function errorToMessage(err: unknown): string {
   }
   return String(err);
 }
+
+function isAbortLikeError(err: unknown): boolean {
+  if (err instanceof Error && err.name === "AbortError") return true;
+  if (
+    typeof DOMException !== "undefined" &&
+    err instanceof DOMException &&
+    err.name === "AbortError"
+  ) {
+    return true;
+  }
+  return isAbortReason(err);
+}

package/src/memory/context-search/sources/conversations.ts

@@ -7,6 +7,7 @@ import { rawAll } from "../../raw-query.js";
 import type { RecallSearchContext, RecallSearchResult } from "../types.js";
 
 const SUBAGENT_SOURCE = "subagent";
+const NOTIFICATION_SOURCE = "notification";
 
 interface ConversationEvidenceRow {
   message_id: string;
@@ -88,7 +89,10 @@ export async function searchConversationSource(
 
   for (const ftsMatch of ftsMatches) {
     try {
-      rows = mergeConversationRows(rows, searchWithFts(ftsMatch, queryLimit));
+      rows = mergeConversationRows(
+        rows,
+        searchWithFts(ftsMatch, queryLimit, context.conversationId),
+      );
     } catch {
       // Try the next, broader query shape.
     }
@@ -97,7 +101,7 @@ export async function searchConversationSource(
   }
 
   if (rows.length === 0) {
-    rows = searchWithLike(trimmedQuery, queryLimit);
+    rows = searchWithLike(trimmedQuery, queryLimit, context.conversationId);
   }
 
   const sortedRows = rows
@@ -128,6 +132,7 @@ export async function searchConversationSource(
 function searchWithFts(
   ftsMatch: string,
   limit: number,
+  excludedConversationId: string,
 ): ConversationEvidenceRow[] {
   return rawAll<ConversationEvidenceRow>(
     `
@@ -142,14 +147,17 @@ function searchWithFts(
     JOIN messages m ON m.id = messages_fts.message_id
     JOIN conversations c ON c.id = m.conversation_id
     WHERE messages_fts MATCH ?
-      AND (c.conversation_type IS NULL OR c.conversation_type != 'private')
-      AND (c.source IS NULL OR c.source NOT IN (?, ?))
+      AND (c.source IS NULL OR c.source NOT IN (?, ?, ?))
+      AND c.id != ?
+      AND c.conversation_type != 'private'
     ORDER BY bm25(messages_fts), m.created_at DESC
     LIMIT ?
     `,
     ftsMatch,
     SUBAGENT_SOURCE,
     AUTO_ANALYSIS_SOURCE,
+    NOTIFICATION_SOURCE,
+    excludedConversationId,
     limit,
   );
 }
@@ -157,6 +165,7 @@ function searchWithFts(
 function searchWithLike(
   query: string,
   limit: number,
+  excludedConversationId: string,
 ): ConversationEvidenceRow[] {
   return rawAll<ConversationEvidenceRow>(
     `
@@ -170,14 +179,17 @@ function searchWithLike(
     FROM messages m
     JOIN conversations c ON c.id = m.conversation_id
     WHERE m.content LIKE ? ESCAPE '\\'
-      AND (c.conversation_type IS NULL OR c.conversation_type != 'private')
-      AND (c.source IS NULL OR c.source NOT IN (?, ?))
+      AND (c.source IS NULL OR c.source NOT IN (?, ?, ?))
+      AND c.id != ?
+      AND c.conversation_type != 'private'
     ORDER BY m.created_at DESC
     LIMIT ?
     `,
     buildLikePattern(query),
     SUBAGENT_SOURCE,
     AUTO_ANALYSIS_SOURCE,
+    NOTIFICATION_SOURCE,
+    excludedConversationId,
     limit,
   );
 }

package/src/memory/context-search/sources/memory-v2.ts

@@ -2,7 +2,7 @@
 // Memory v2 — `recall` adapter for the `memory` source
 // ---------------------------------------------------------------------------
 //
-// When the v2 flag is on, the `memory` recall source reads from the v2
+// When v2 is enabled, the `memory` recall source reads from the v2
 // concept-page subsystem (under `<workspace>/memory/concepts/`) instead of
 // the legacy graph. Two retrieval paths run in parallel and merge:
 //
@@ -26,11 +26,8 @@
 import { readdir, readFile, realpath, stat } from "node:fs/promises";
 import { extname, isAbsolute, join, relative } from "node:path";
 
-import { isAssistantFeatureFlagEnabled } from "../../../config/assistant-feature-flags.js";
-import type { AssistantConfig } from "../../../config/schema.js";
 import { getLogger } from "../../../util/logger.js";
 import { embedWithRetry } from "../../embed.js";
-import { generateSparseEmbedding } from "../../embedding-backend.js";
 import { spreadActivation } from "../../v2/activation.js";
 import { getEdgeIndex } from "../../v2/edge-index.js";
 import {
@@ -39,27 +36,14 @@ import {
   slugFromConceptPath,
 } from "../../v2/page-store.js";
 import { hybridQueryConceptPages } from "../../v2/qdrant.js";
-import { clampUnitInterval } from "../../validation.js";
+import { fuseHalf } from "../../v2/sim.js";
+import { generateBm25QueryEmbedding } from "../../v2/sparse-bm25.js";
 import type {
   RecallEvidence,
   RecallSearchContext,
   RecallSearchResult,
 } from "../types.js";
 
-/**
- * True when both v2 gates are on. Single source of truth for whether v2 is
- * the active memory subsystem — used by recall sources (`memory`, `pkb`),
- * per-turn injectors, the indexer's v1 graph_extract suppression, and the
- * v1/v2 maintenance scheduler. The historical name retains "Read" but the
- * predicate now gates both read and write paths.
- */
-export function isMemoryV2ReadActive(config: AssistantConfig): boolean {
-  return (
-    isAssistantFeatureFlagEnabled("memory-v2-enabled", config) &&
-    config.memory.v2.enabled
-  );
-}
-
 const log = getLogger("context-search-memory-v2-source");
 
 /**
@@ -189,7 +173,7 @@ async function activationEvidence(
   });
   const denseVector = denseResult.vectors[0];
   if (!denseVector || denseVector.length === 0) return [];
-  const sparseVector = generateSparseEmbedding(trimmedQuery);
+  const sparseVector = generateBm25QueryEmbedding(trimmedQuery);
 
   const annLimit =
     context.config.memory.v2.ann_candidate_limit ??
@@ -204,24 +188,43 @@ async function activationEvidence(
   const { dense_weight: denseWeight, sparse_weight: sparseWeight } =
     context.config.memory.v2;
 
-  let maxSparse = 0;
+  // Mirror sim.ts: normalize body and summary sparse channels against their
+  // own per-batch maxima, fuse each half via clamp01(dense·w_d + sparse·w_s),
+  // then take max(body, summary) per slug. Pages without a summary embedding
+  // return undefined for both summary scores; the max collapses cleanly to
+  // the body score so legacy pages keep their pre-summary ranking.
+  let maxBodySparse = 0;
+  let maxSummarySparse = 0;
   for (const hit of hits) {
-    if (hit.sparseScore !== undefined && hit.sparseScore > maxSparse) {
-      maxSparse = hit.sparseScore;
+    if (hit.sparseScore !== undefined && hit.sparseScore > maxBodySparse) {
+      maxBodySparse = hit.sparseScore;
+    }
+    if (
+      hit.summarySparseScore !== undefined &&
+      hit.summarySparseScore > maxSummarySparse
+    ) {
+      maxSummarySparse = hit.summarySparseScore;
     }
   }
 
   const ownActivation = new Map<string, number>();
   for (const hit of hits) {
-    const dense = hit.denseScore ?? 0;
-    const sparseNormalized =
-      hit.sparseScore !== undefined && maxSparse > 0
-        ? hit.sparseScore / maxSparse
-        : 0;
-    ownActivation.set(
-      hit.slug,
-      clampUnitInterval(denseWeight * dense + sparseWeight * sparseNormalized),
+    const bodyScore = fuseHalf(
+      hit.denseScore,
+      hit.sparseScore,
+      maxBodySparse,
+      denseWeight,
+      sparseWeight,
+    );
+    const summaryScore = fuseHalf(
+      hit.summaryDenseScore,
+      hit.summarySparseScore,
+      maxSummarySparse,
+      denseWeight,
+      sparseWeight,
     );
+    const score = Math.max(bodyScore ?? 0, summaryScore ?? bodyScore ?? 0);
+    ownActivation.set(hit.slug, score);
   }
 
   const edgeIndex = await getEdgeIndex(context.workingDir);
@@ -311,11 +314,13 @@ async function lexicalEvidence(
   if (!conceptsRoot) return [];
 
   const matches: MemoryV2LexicalMatch[] = [];
+  const visitedDirectories = new Set<string>([conceptsRoot]);
   await walkConceptsDirectory(
     conceptsRoot,
     conceptsRoot,
     queryTerms,
     matches,
+    visitedDirectories,
     context.signal,
   );
 
@@ -346,6 +351,7 @@ async function walkConceptsDirectory(
   conceptsRoot: string,
   queryTerms: ReadonlySet<string>,
   matches: MemoryV2LexicalMatch[],
+  visitedDirectories: Set<string>,
   signal: AbortSignal | undefined,
 ): Promise<void> {
   throwIfAborted(signal);
@@ -380,11 +386,14 @@ async function walkConceptsDirectory(
     }
 
     if (entryStats.isDirectory()) {
+      if (visitedDirectories.has(entryRealPath)) continue;
+      visitedDirectories.add(entryRealPath);
       await walkConceptsDirectory(
         entryRealPath,
         conceptsRoot,
         queryTerms,
         matches,
+        visitedDirectories,
         signal,
       );
       continue;

package/src/memory/context-search/sources/memory.ts

@@ -9,7 +9,7 @@ import type {
   RecallSearchContext,
   RecallSearchResult,
 } from "../types.js";
-import { isMemoryV2ReadActive, searchMemoryV2Source } from "./memory-v2.js";
+import { searchMemoryV2Source } from "./memory-v2.js";
 
 const log = getLogger("context-search-memory-source");
 
@@ -23,7 +23,7 @@ export async function searchMemorySource(
     return { evidence: [] };
   }
 
-  if (isMemoryV2ReadActive(context.config)) {
+  if (context.config.memory.v2.enabled) {
     return searchMemoryV2Source(query, context, normalizedLimit);
   }
 
@@ -34,6 +34,7 @@ export async function searchMemorySource(
     });
     queryVector = result.vectors[0] ?? null;
   } catch (err) {
+    if (context.signal?.aborted || isAbortError(err)) throw err;
     log.warn({ err }, "Failed to embed memory recall query");
     return { evidence: [] };
   }
@@ -67,11 +68,17 @@ export async function searchMemorySource(
 
     return { evidence };
   } catch (err) {
+    if (context.signal?.aborted || isAbortError(err)) throw err;
     log.warn({ err }, "Failed to search memory graph for recall");
     return { evidence: [] };
   }
 }
 
+function isAbortError(err: unknown): boolean {
+  if (!(err instanceof Error)) return false;
+  return err.name === "AbortError" || err.name === "APIUserAbortError";
+}
+
 function memoryNodeToEvidence(node: MemoryNode, score: number): RecallEvidence {
   return {
     id: `memory:${node.id}`,

package/src/memory/context-search/sources/workspace.ts

@@ -23,7 +23,6 @@ const PATH_LITERAL_PATTERN =
 
 const WORKSPACE_BUCKETS: readonly WorkspaceBucket[] = [
   { name: "root", relativePath: "", budget: 100, rootFilesOnly: true },
-  { name: "pkb", relativePath: "pkb", budget: 500 },
   { name: "memory", relativePath: "memory", budget: 500 },
   { name: "journal", relativePath: "journal", budget: 250 },
   { name: "scratch", relativePath: "scratch", budget: 500 },
@@ -66,6 +65,9 @@ const SECRET_SEGMENT_NAMES = new Set([
   "ces-security",
 ]);
 
+const SECRET_TOKEN_PATTERN =
+  /(?:^|[-_.])(?:keys?|secrets?|tokens?)(?:[-_.]|$)/i;
+
 const QUERY_STOP_WORDS = new Set([
   "a",
   "about",
@@ -591,8 +593,13 @@ async function maybeSearchResolvedFile(
   matches: WorkspaceMatch[],
   state: WalkState,
 ): Promise<void> {
+  if (state.scannedRelativePaths.has(relativePath)) {
+    return;
+  }
+  state.scannedRelativePaths.add(relativePath);
+  state.scannedFiles += 1;
+
   if (
-    state.scannedRelativePaths.has(relativePath) ||
     shouldSkipWorkspaceFile(relativePath) ||
     shouldSkipFilePath(relativePath) ||
     fileSizeBytes > WORKSPACE_SOURCE_MAX_FILE_SIZE_BYTES
@@ -600,8 +607,6 @@ async function maybeSearchResolvedFile(
     return;
   }
 
-  state.scannedRelativePaths.add(relativePath);
-  state.scannedFiles += 1;
   matches.push(
     ...(await searchFile(realPath, relativePath, fileSizeBytes, queryTerms)),
   );
@@ -1049,8 +1054,6 @@ function getPathPriorityBoost(relativePath: string): number {
   if (relativePath.startsWith("scratch/")) return 0.35;
   if (relativePath.startsWith("users/")) return 0.3;
   if (relativePath.startsWith("journal/")) return 0.25;
-  if (relativePath.startsWith("pkb/archive/")) return -0.15;
-  if (relativePath.startsWith("pkb/")) return 0.2;
   if (relativePath.startsWith("memory/concepts/")) return 0.4;
   if (relativePath.startsWith("memory/")) return 0.2;
   if (
@@ -1181,15 +1184,15 @@ function shouldSkipSegmentName(name: string): boolean {
   return (
     GENERATED_OR_DEPENDENCY_DIR_NAMES.has(lowerName) ||
     lowerName.startsWith(".env") ||
-    lowerName.includes("key") ||
-    lowerName.includes("secret") ||
-    lowerName.includes("token") ||
+    SECRET_TOKEN_PATTERN.test(lowerName) ||
     lowerName.startsWith("credentials") ||
     SECRET_SEGMENT_NAMES.has(lowerName)
   );
 }
 
-function normalizeWorkspacePathLiteral(pathLiteral: string): string | null {
+export function normalizeWorkspacePathLiteral(
+  pathLiteral: string,
+): string | null {
   const trimmed = pathLiteral
     .trim()
     .replace(/^["'`]+|["'`.,;:)>\]}]+$/g, "")

package/src/memory/context-search/types.ts

@@ -1,6 +1,6 @@
 import type { AssistantConfig } from "../../config/schema.js";
 
-export type RecallSource = "memory" | "pkb" | "conversations" | "workspace";
+export type RecallSource = "memory" | "conversations" | "workspace";
 
 export type RecallDepth = "fast" | "standard" | "deep";
 

package/src/memory/conversation-bootstrap.ts

@@ -12,6 +12,14 @@ export interface BootstrapConversationOptions {
   systemHint: string;
   scheduleJobId?: string;
   groupId?: string;
+  /**
+   * When set, the new conversation is linked to its parent via the
+   * `fork_parent_conversation_id` column. Used by background jobs that
+   * spawn analysis conversations off a source conversation (auto-analyze,
+   * memory-retrospective) so the parent → child relationship is queryable
+   * later (e.g. "find the most recent retrospective for this source").
+   */
+  forkParentConversationId?: string;
 }
 
 export function bootstrapConversation(opts: BootstrapConversationOptions) {
@@ -21,6 +29,9 @@ export function bootstrapConversation(opts: BootstrapConversationOptions) {
     ...(opts.source && { source: opts.source }),
     ...(opts.scheduleJobId && { scheduleJobId: opts.scheduleJobId }),
     ...(opts.groupId && { groupId: opts.groupId }),
+    ...(opts.forkParentConversationId && {
+      forkParentConversationId: opts.forkParentConversationId,
+    }),
   });
   queueGenerateConversationTitle({
     conversationId: conversation.id,