@vellumai/assistant 0.8.1 → 0.8.3

package/src/memory/v2/migration.ts

@@ -643,12 +643,15 @@ export async function runMemoryV2Migration(
 }
 
 /**
- * HTML marker embedded in each appended block so a crash-recovery rerun can
- * detect already-applied promotions and skip the append. `appendLines` is a
- * read-modify-write — without this, a crash between `appendPromotions` and
- * `writeSentinel` would let the next boot duplicate every promotion line.
+ * Paired HTML markers wrapped around each appended block. The opening marker
+ * also serves as the idempotency guard: `appendLines` is a read-modify-write,
+ * and without it a crash between `appendPromotions` and `writeSentinel` would
+ * let the next boot duplicate every promotion line. The closing marker
+ * delimits the migration-inserted region so a force-rerun strip can remove
+ * exactly that block without touching user/assistant edits appended below.
  */
-const PROMOTION_MARKER = "<!-- migration:v1-to-v2 -->";
+const PROMOTION_MARKER_OPEN = "<!-- migration:v1-to-v2 -->";
+const PROMOTION_MARKER_CLOSE = "<!-- /migration:v1-to-v2 -->";
 
 /**
  * Append each promotion bucket to its target file. Files are created if
@@ -680,8 +683,8 @@ async function appendPromotions(
 
 /**
  * Append `lines` to `path`, creating it (with a trailing newline) if absent.
- * If the file already contains `PROMOTION_MARKER`, the append is skipped — a
- * prior partially-completed migration already wrote this block.
+ * If the file already contains `PROMOTION_MARKER_OPEN`, the append is skipped
+ * — a prior partially-completed migration already wrote this block.
  */
 async function appendLines(path: string, lines: string[]): Promise<void> {
   let existing = "";
@@ -690,16 +693,17 @@ async function appendLines(path: string, lines: string[]): Promise<void> {
   } catch (err) {
     if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
   }
-  if (existing.includes(PROMOTION_MARKER)) return;
+  if (existing.includes(PROMOTION_MARKER_OPEN)) return;
   const trailing = existing.length === 0 || existing.endsWith("\n") ? "" : "\n";
-  const next = `${existing}${trailing}${PROMOTION_MARKER}\n${lines.join("\n")}\n`;
+  const block = `${PROMOTION_MARKER_OPEN}\n${lines.join("\n")}\n${PROMOTION_MARKER_CLOSE}\n`;
+  const next = `${existing}${trailing}${block}`;
   await writeFile(path, next, "utf-8");
 }
 
 /**
- * Strip any prior migration-block (everything from the marker line through
- * end of file) from each promotion target. Called on force-reruns so the
- * marker guard in `appendLines` doesn't skip the new promotions.
+ * Strip any prior migration-block from each promotion target. Called on
+ * force-reruns so the marker guard in `appendLines` doesn't skip the new
+ * promotions.
  */
 async function stripPromotionMarkerBlocks(workspaceDir: string): Promise<void> {
   const memoryDir = join(workspaceDir, "memory");
@@ -718,6 +722,16 @@ async function stripPromotionMarkerBlocks(workspaceDir: string): Promise<void> {
   await Promise.all(candidates.map(stripMarkerBlock));
 }
 
+/**
+ * Remove the migration-inserted block from `path` while preserving content
+ * outside it. The block is identified by the
+ * `PROMOTION_MARKER_OPEN ... PROMOTION_MARKER_CLOSE` envelope.
+ *
+ * If an opening marker is present without a matching close, strip from the
+ * opening marker to the next blank line, or to EOF if none is found. Content
+ * appended after such an unclosed block without a blank-line separator can
+ * be dropped on that fallback path.
+ */
 async function stripMarkerBlock(path: string): Promise<void> {
   let existing: string;
   try {
@@ -726,13 +740,29 @@ async function stripMarkerBlock(path: string): Promise<void> {
     if ((err as NodeJS.ErrnoException).code === "ENOENT") return;
     throw err;
   }
-  const idx = existing.indexOf(PROMOTION_MARKER);
-  if (idx === -1) return;
-  // Cut from the start of the marker line. `idx` already points at the marker,
-  // which `appendLines` always wrote at the start of its own line, so a plain
-  // slice here also drops the leading newline that preceded it (if any).
-  const trimmed = existing.slice(0, idx).replace(/\n+$/, "");
-  const next = trimmed.length === 0 ? "" : `${trimmed}\n`;
+  const openIdx = existing.indexOf(PROMOTION_MARKER_OPEN);
+  if (openIdx === -1) return;
+
+  let endIdx: number;
+  const closeIdx = existing.indexOf(PROMOTION_MARKER_CLOSE, openIdx);
+  if (closeIdx !== -1) {
+    endIdx = closeIdx + PROMOTION_MARKER_CLOSE.length;
+    if (existing[endIdx] === "\n") endIdx += 1;
+  } else {
+    const blankIdx = existing.indexOf("\n\n", openIdx);
+    endIdx = blankIdx === -1 ? existing.length : blankIdx + 2;
+  }
+
+  const head = existing.slice(0, openIdx).replace(/\n+$/, "");
+  const tail = existing.slice(endIdx);
+  let next: string;
+  if (head.length === 0) {
+    next = tail;
+  } else if (tail.length === 0) {
+    next = `${head}\n`;
+  } else {
+    next = `${head}\n${tail}`;
+  }
   await writeFile(path, next, "utf-8");
 }
 

package/src/memory/v2/page-index.ts

@@ -30,6 +30,16 @@ const log = getLogger("memory-v2-page-index");
 
 const SUMMARY_MAX_LENGTH = 200;
 
+/**
+ * Collapse every run of whitespace (including embedded newlines and tabs) to a
+ * single space and trim. The router prompt renders one entry per line, so an
+ * embedded newline anywhere in `summary` would split that entry across lines
+ * and corrupt the format the router parses.
+ */
+function normalizeSummary(raw: string): string {
+  return raw.replace(/\s+/g, " ").trim().slice(0, SUMMARY_MAX_LENGTH);
+}
+
 /**
  * One row in the rendered page index. `id` is a 1-based dense integer that is
  * stable within a single index version (i.e. a single build). It changes when
@@ -70,11 +80,12 @@ let cache: CachedIndex | null = null;
 /**
  * Return a `PageIndex` for `workspaceDir`. Cached module-locally; the cache
  * is invalidated by `invalidatePageIndex` (called by daemon-side hooks when
- * concept pages or skill entries change).
+ * concept pages, skill entries, or CLI-command entries change).
  *
  * Cold builds list every concept page in parallel, drop pages whose read
- * rejects, append seeded skill entries from `listSkillEntries()`, sort by
- * slug for deterministic IDs, then resolve outgoing edges to numeric IDs.
+ * rejects, append seeded skill entries from `listSkillEntries()` and CLI
+ * command entries from `listCliCommandEntries()`, sort by slug for
+ * deterministic IDs, then resolve outgoing edges to numeric IDs.
  */
 export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
   if (cache && cache.workspaceDir === workspaceDir) {
@@ -97,6 +108,30 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
     outgoingSlugs: string[];
   }
 
+  const [
+    { listSkillEntries, SKILL_SLUG_PREFIX },
+    { listCliCommandEntries, CLI_COMMAND_SLUG_PREFIX },
+  ] = await Promise.all([
+    import("./skill-store.js"),
+    import("./cli-command-store.js"),
+  ]);
+
+  // Build the synthetic-slug sets first so we can drop colliding concept
+  // pages. Collision policy: **synthetic entries win**. Skill and CLI rows
+  // are seeded from authoritative in-process catalogs; a hand-authored page
+  // sitting under `skills/<id>` or `cli-commands/<name>` is either a stale
+  // leftover from a prior write or a user mistake. `bySlug` is last-writer-
+  // wins, so without explicit dedupe one side would silently shadow the
+  // other depending on iteration order.
+  const skillEntries = listSkillEntries();
+  const skillSlugs = new Set(
+    skillEntries.map((entry) => `${SKILL_SLUG_PREFIX}${entry.id}`),
+  );
+  const cliCommandEntries = listCliCommandEntries();
+  const cliCommandSlugs = new Set(
+    cliCommandEntries.map((entry) => `${CLI_COMMAND_SLUG_PREFIX}${entry.id}`),
+  );
+
   const drafts: DraftEntry[] = [];
   for (let i = 0; i < settled.length; i++) {
     const result = settled[i];
@@ -110,20 +145,40 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
     }
     const page = result.value;
     if (!page) continue;
+    if (skillSlugs.has(slug)) {
+      log.warn(
+        { slug },
+        "Dropping concept page from index — slug collides with a seeded skill entry; skill wins",
+      );
+      continue;
+    }
+    if (cliCommandSlugs.has(slug)) {
+      log.warn(
+        { slug },
+        "Dropping concept page from index — slug collides with a seeded CLI-command entry; CLI command wins",
+      );
+      continue;
+    }
     const summarySource = page.frontmatter.summary?.trim() || page.body.trim();
     drafts.push({
       slug,
-      summary: summarySource.slice(0, SUMMARY_MAX_LENGTH),
+      summary: normalizeSummary(summarySource),
       outgoingSlugs: page.frontmatter.edges,
     });
   }
 
-  const { listSkillEntries, SKILL_SLUG_PREFIX } =
-    await import("./skill-store.js");
-  for (const entry of listSkillEntries()) {
+  for (const entry of skillEntries) {
     drafts.push({
       slug: `${SKILL_SLUG_PREFIX}${entry.id}`,
-      summary: entry.content.trim().slice(0, SUMMARY_MAX_LENGTH),
+      summary: normalizeSummary(entry.content),
+      outgoingSlugs: [],
+    });
+  }
+
+  for (const entry of cliCommandEntries) {
+    drafts.push({
+      slug: `${CLI_COMMAND_SLUG_PREFIX}${entry.id}`,
+      summary: normalizeSummary(entry.description),
       outgoingSlugs: [],
     });
   }

package/src/memory/v2/prompts/router.ts

@@ -27,7 +27,6 @@ import { homedir } from "node:os";
 import { isAbsolute, join } from "node:path";
 
 import { getLogger } from "../../../util/logger.js";
-import { getWorkspaceDir } from "../../../util/platform.js";
 
 const log = getLogger("memory-v2-router-prompt");
 
@@ -102,7 +101,7 @@ export function renderRouterPrompt(opts: RenderRouterPromptOpts): string {
  * referenced by `memory.v2.router.router_prompt_path`, then substitute the
  * standard placeholders. Path-resolution rules mirror the consolidation
  * prompt override: absolute paths used as-is, leading `~/` expanded to home,
- * relative paths resolved under the workspace root.
+ * relative paths resolved under `workspaceDir`.
  *
  * Failure handling is intentionally permissive — missing file, read error,
  * oversized file, or empty/whitespace-only body all log a warning and fall
@@ -111,11 +110,12 @@ export function renderRouterPrompt(opts: RenderRouterPromptOpts): string {
  */
 export function resolveRouterPrompt(
   overridePath: string | null,
+  workspaceDir: string,
   opts: RenderRouterPromptOpts,
 ): string {
   if (overridePath === null) return renderRouterPrompt(opts);
 
-  const resolvedPath = resolveOverridePath(overridePath);
+  const resolvedPath = resolveOverridePath(overridePath, workspaceDir);
   let contents: string;
   try {
     const stat = lstatSync(resolvedPath);
@@ -178,15 +178,18 @@ function substitutePlaceholders(
   const assistant = opts.assistantName?.trim() || "the assistant";
   const user = opts.userName?.trim() || "the user";
   return template
-    .replaceAll(ASSISTANT_NAME_PLACEHOLDER, assistant)
-    .replaceAll(USER_NAME_PLACEHOLDER, user)
-    .replaceAll(PAGE_INDEX_PLACEHOLDER, opts.pageIndexBlock);
+    .replaceAll(ASSISTANT_NAME_PLACEHOLDER, () => assistant)
+    .replaceAll(USER_NAME_PLACEHOLDER, () => user)
+    .replaceAll(PAGE_INDEX_PLACEHOLDER, () => opts.pageIndexBlock);
 }
 
-function resolveOverridePath(overridePath: string): string {
+function resolveOverridePath(
+  overridePath: string,
+  workspaceDir: string,
+): string {
   if (overridePath.startsWith("~/")) {
     return join(homedir(), overridePath.slice(2));
   }
   if (isAbsolute(overridePath)) return overridePath;
-  return join(getWorkspaceDir(), overridePath);
+  return join(workspaceDir, overridePath);
 }

package/src/memory/v2/prompts/sweep.ts

@@ -51,6 +51,6 @@ export function renderSweepPrompt(opts: {
   const user = opts.userName?.trim() || "the user";
   return SWEEP_PROMPT.replaceAll(
     ASSISTANT_NAME_PLACEHOLDER,
-    assistant,
-  ).replaceAll(USER_NAME_PLACEHOLDER, user);
+    () => assistant,
+  ).replaceAll(USER_NAME_PLACEHOLDER, () => user);
 }

package/src/memory/v2/qdrant.ts

@@ -200,6 +200,9 @@ async function ensureConceptPageCollectionOnce(): Promise<{
 
       const missing = missingNamedVectors(info);
       if (missing.length === 0) {
+        // Long-lived installs may predate the `kind` payload index; ensure
+        // every required index exists before declaring the collection ready.
+        await ensurePayloadIndexes();
         _collectionReady = true;
         return { migrated: false };
       }
@@ -271,17 +274,62 @@ async function ensureConceptPageCollectionOnce(): Promise<{
     throw err;
   }
 
-  // Slug is the only payload field we filter on; index it once at create-time
-  // so upserts and slug-restricted queries don't pay a per-call indexing cost.
-  await client.createPayloadIndex(MEMORY_V2_COLLECTION, {
-    field_name: "slug",
-    field_schema: "keyword",
-  });
+  await ensurePayloadIndexes();
 
   _collectionReady = true;
   return { migrated };
 }
 
+/**
+ * Idempotently create the payload indexes the collection's query and
+ * filter paths rely on:
+ *
+ *   - `slug` (keyword): every slug-restricted query and prefix scan filters on it.
+ *   - `kind` (keyword): the skill-backfill scroll filters with `is_empty` on
+ *     `kind`. Strict-mode Qdrant deployments reject filters on unindexed
+ *     payload fields, so without this the backfill consistently fails and
+ *     legacy skill points remain untagged.
+ *
+ * Same-schema `createPayloadIndex` calls are idempotent server-side in
+ * Qdrant (200 OK), so the only "already exists" failures we expect are
+ * narrow races where a concurrent caller created the same index a moment
+ * earlier. Those are benign and swallowed. Every other failure — strict-mode
+ * rejection, index-limit, transient network blip — must propagate so the
+ * caller does not latch readiness on a collection whose `slug`/`kind`
+ * filters will keep rejecting queries until the next daemon restart.
+ */
+async function ensurePayloadIndexes(): Promise<void> {
+  const client = getClient();
+  const indexes = [
+    { field_name: "slug", field_schema: "keyword" as const },
+    { field_name: "kind", field_schema: "keyword" as const },
+  ];
+  // Parallel so one "already exists" race on a single index doesn't stall
+  // the other create round-trip. v1's `qdrant-client.ts` uses the same
+  // Promise.all shape.
+  await Promise.all(
+    indexes.map(async (index) => {
+      try {
+        await client.createPayloadIndex(MEMORY_V2_COLLECTION, index);
+      } catch (err) {
+        if (isPayloadIndexAlreadyExists(err)) return;
+        throw err;
+      }
+    }),
+  );
+}
+
+/**
+ * True when a `createPayloadIndex` error indicates the index already
+ * exists with matching parameters — the only failure shape it is safe to
+ * swallow. Qdrant returns 4xx with messages like
+ * `"Wrong input: Payload field 'kind' already exists ..."`.
+ */
+function isPayloadIndexAlreadyExists(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /already exists/i.test(msg);
+}
+
 /**
  * Return the names of required named vectors absent from the collection's
  * current schema. An empty array means the collection is fully migrated.
@@ -422,7 +470,10 @@ export async function deleteConceptPageEmbedding(slug: string): Promise<void> {
  * `payload.kind` matches are eligible for deletion. This is critical because
  * `validateSlug` permits user-authored concept pages slugged like
  * `skills/foo`; without a kind filter they would collide with the skill
- * namespace and be repeatedly pruned every seed run.
+ * namespace and be repeatedly pruned every seed run. The companion
+ * {@link backfillKindOnPointsWithPrefix} preserves this invariant for legacy
+ * untagged rows by tagging only suffixes the caller knows are skills —
+ * user-authored `skills/<slug>` rows stay kindless and outside this prune.
  *
  * Idempotent: when the live `<prefix>*` slugs already match `activeSuffixes`,
  * the function performs a single scroll and no deletes.
@@ -491,6 +542,83 @@ export async function pruneSlugsWithPrefixExcept(
   }
 }
 
+/**
+ * Set `payload.kind` on every point whose slug starts with `prefix`, whose
+ * suffix is in `allowedSuffixes`, and is currently missing the `kind`
+ * discriminator. Used to tag legacy rows that predate the kind field so the
+ * kind-scoped {@link pruneSlugsWithPrefixExcept} no longer leaves them as
+ * orphans.
+ *
+ * `allowedSuffixes` is required because `validateSlug` permits user-authored
+ * concept pages slugged like `skills/my-notes` — those rows also lack `kind`
+ * and would otherwise be tagged here and then deleted by the kind-scoped
+ * prune. Callers must pass the closed set of legitimate suffixes (e.g. the
+ * union of installed + remote-catalog skill IDs) so user pages stay untagged.
+ *
+ * The "missing kind" predicate is pushed to Qdrant via `is_empty`, so once
+ * every legacy row has been tagged the scroll returns the bounded set of
+ * other kindless concept pages without ever touching the already-tagged
+ * rows. Idempotent across retries: a row tagged by an earlier partial run
+ * no longer matches the filter and is silently skipped.
+ */
+export async function backfillKindOnPointsWithPrefix(
+  prefix: string,
+  kind: string,
+  allowedSuffixes: ReadonlySet<string>,
+): Promise<number> {
+  if (allowedSuffixes.size === 0) return 0;
+  await ensureConceptPageCollection();
+
+  const client = getClient();
+
+  const doBackfill = async (): Promise<number> => {
+    const pointIds: Array<string | number> = [];
+    let offset: string | number | undefined = undefined;
+    const maxIterations = 10_000;
+    const batchSize = 256;
+    for (let i = 0; i < maxIterations; i++) {
+      const result = await client.scroll(MEMORY_V2_COLLECTION, {
+        limit: batchSize,
+        with_payload: true,
+        with_vector: false,
+        filter: { must: [{ is_empty: { key: "kind" } }] },
+        ...(offset !== undefined ? { offset } : {}),
+      });
+      for (const point of result.points) {
+        const slug = (point.payload as { slug?: unknown } | null)?.slug;
+        if (typeof slug !== "string") continue;
+        if (!slug.startsWith(prefix)) continue;
+        const suffix = slug.slice(prefix.length);
+        if (!allowedSuffixes.has(suffix)) continue;
+        pointIds.push(point.id);
+      }
+      const next = result.next_page_offset;
+      if (next == null) break;
+      offset = typeof next === "string" ? next : (next as number);
+    }
+
+    if (pointIds.length === 0) return 0;
+
+    await client.setPayload(MEMORY_V2_COLLECTION, {
+      payload: { kind },
+      points: pointIds,
+      wait: true,
+    });
+    return pointIds.length;
+  };
+
+  try {
+    return await doBackfill();
+  } catch (err) {
+    if (isCollectionMissing(err)) {
+      _collectionReady = false;
+      await ensureConceptPageCollection();
+      return await doBackfill();
+    }
+    throw err;
+  }
+}
+
 /**
  * Approximate count of points in the v2 concept-page collection. Used by the
  * daemon-startup rebuild hook to detect "collection exists but empty" — the

package/src/memory/v2/router.ts

@@ -181,6 +181,7 @@ export async function runRouter(
 
   const systemPrompt = resolveRouterPrompt(
     config.memory?.v2?.router?.router_prompt_path ?? null,
+    workspaceDir,
     {
       assistantName: getAssistantName(),
       userName: resolveUserName(workspaceDir),
@@ -274,22 +275,22 @@ export async function runRouter(
     );
   }
 
-  const truncated = inRangeIds.length > maxPageIds;
-  const finalIds = truncated ? inRangeIds.slice(0, maxPageIds) : inRangeIds;
+  // De-duplicate BEFORE applying the cap — otherwise a duplicate-heavy
+  // model output like `[1, 1, 2]` with `max=2` slices to `[1, 1]` and
+  // dedupes to `[1]`, under-filling the cap.
+  const dedupedIds = Array.from(new Set(inRangeIds));
+
+  const truncated = dedupedIds.length > maxPageIds;
+  const finalIds = truncated ? dedupedIds.slice(0, maxPageIds) : dedupedIds;
   if (truncated) {
     log.warn(
-      { returned: inRangeIds.length, max: maxPageIds },
+      { returned: dedupedIds.length, max: maxPageIds },
       "Router returned more page IDs than max_page_ids; truncating",
     );
   }
 
-  // De-duplicate while preserving order — the index lookup alone wouldn't
-  // catch repeats from the model.
-  const seen = new Set<number>();
   const selectedSlugs: string[] = [];
   for (const id of finalIds) {
-    if (seen.has(id)) continue;
-    seen.add(id);
     const entry = pageIndex.byId.get(id);
     if (!entry) continue;
     selectedSlugs.push(entry.slug);