@vellumai/assistant 0.7.1 → 0.7.2

package/src/cli/commands/memory-v2.ts

@@ -36,6 +36,10 @@ import { cliIpcCall } from "../../ipc/cli-client.js";
 import type {
   MemoryV2BackfillOp,
   MemoryV2BackfillResult,
+  MemoryV2ExplainSimilarityResult,
+  MemoryV2ExplainSimilarityStats,
+  MemoryV2FitAnisotropyResult,
+  MemoryV2RebuildCorpusStatsResult,
   MemoryV2ReembedSkillsResult,
   MemoryV2ValidateResult,
 } from "../../runtime/routes/memory-v2-routes.js";
@@ -73,6 +77,74 @@ async function runBackfillOp(
   log.info(`Queued ${op} job: ${result.result!.jobId}`);
 }
 
+/** Format a number for table output. */
+function fmt(n: number | null, decimals: number): string {
+  if (n === null) return "—";
+  return n.toFixed(decimals);
+}
+
+/** Render the per-channel breakdown table + stats for the explain command. */
+function printExplainResult(result: MemoryV2ExplainSimilarityResult): void {
+  log.info(
+    `dense_weight=${result.config.dense_weight}  sparse_weight=${result.config.sparse_weight}`,
+  );
+
+  for (const channel of result.channels) {
+    log.info("");
+    log.info(`── channel: ${channel.channel} ──`);
+    log.info(`text: ${channel.textPreview}`);
+    log.info(
+      `maxSparse (used for normalization): ${channel.maxSparse.toFixed(4)}`,
+    );
+    log.info(
+      `effective: dw=${channel.effectiveDenseWeight.toFixed(3)} sw=${channel.effectiveSparseWeight.toFixed(3)} (sparseSpread=${channel.sparseSpread.toFixed(3)})`,
+    );
+    log.info("");
+    log.info(
+      "slug".padEnd(48) +
+        "dense".padStart(10) +
+        "sparseRaw".padStart(12) +
+        "sparseNorm".padStart(12) +
+        "fused".padStart(10),
+    );
+    log.info("─".repeat(92));
+    for (const row of channel.rows) {
+      const slugCol =
+        row.slug.length > 47 ? `${row.slug.slice(0, 46)}…` : row.slug;
+      log.info(
+        slugCol.padEnd(48) +
+          fmt(row.denseScore, 4).padStart(10) +
+          fmt(row.sparseRaw, 4).padStart(12) +
+          fmt(row.sparseNorm, 4).padStart(12) +
+          fmt(row.fused, 4).padStart(10),
+      );
+    }
+    log.info("");
+    log.info("Stats (per channel):");
+    log.info(`  ${formatStatLine("dense       ", channel.stats.dense)}`);
+    log.info(`  ${formatStatLine("sparseRaw   ", channel.stats.sparseRaw)}`);
+    log.info(`  ${formatStatLine("sparseNorm  ", channel.stats.sparseNorm)}`);
+    log.info(`  ${formatStatLine("fused       ", channel.stats.fused)}`);
+  }
+}
+
+function formatStatLine(
+  label: string,
+  stats: MemoryV2ExplainSimilarityStats,
+): string {
+  if (stats.count === 0) {
+    return `${label} n=0`;
+  }
+  const range = stats.max - stats.min;
+  return (
+    `${label} n=${String(stats.count).padStart(3)}` +
+    ` range=[${stats.min.toFixed(4)}, ${stats.max.toFixed(4)}]` +
+    ` (Δ=${range.toFixed(4)})` +
+    ` mean=${stats.mean.toFixed(4)}` +
+    ` std=${stats.stddev.toFixed(4)}`
+  );
+}
+
 // ---------------------------------------------------------------------------
 // Registration
 // ---------------------------------------------------------------------------
@@ -119,13 +191,18 @@ Subcommands fall into three groups:
     validate         Print a diagnostic report of orphan outgoing-edge
                      targets, oversized pages, and parse failures.
 
+  Calibration:
+    fit-anisotropy   Fit Mu & Viswanath's all-but-the-top correction so
+                     embedding cosines stop clustering in a narrow band.
+
 Examples:
   $ assistant memory v2 validate
   $ assistant memory v2 migrate
   $ assistant memory v2 migrate --force
   $ assistant memory v2 reembed
   $ assistant memory v2 reembed-skills
-  $ assistant memory v2 activation`,
+  $ assistant memory v2 activation
+  $ assistant memory v2 fit-anisotropy`,
   );
 
   // ── migrate ───────────────────────────────────────────────────────────
@@ -241,6 +318,224 @@ Examples:
       await runBackfillOp("activation-recompute");
     });
 
+  // ── explain ───────────────────────────────────────────────────────────
+
+  v2.command("explain")
+    .description(
+      "Diagnose dense vs sparse score distributions for a query (read-only)",
+    )
+    .requiredOption(
+      "--text <text>",
+      "Query text to embed and score against the concept-page collection (the user channel).",
+    )
+    .option(
+      "--assistant-text <text>",
+      "Optional second query text — scored independently as the assistant channel.",
+    )
+    .option(
+      "--now-text <text>",
+      "Optional third query text — scored independently as the now channel.",
+    )
+    .option(
+      "--top <n>",
+      "Number of top hits to fetch per channel (default 25)",
+      "25",
+    )
+    .addHelpText(
+      "after",
+      `
+Embeds the supplied text(s), runs the hybrid dense + sparse query against
+the v2 concept-page Qdrant collection, and prints per-slug raw dense, raw
+sparse, normalized sparse, and fused scores plus per-channel summary
+statistics (range, mean, stddev). Use this to identify whether dense
+embedding compression (anisotropy) or per-batch sparse normalization is
+the dominant cause of score compression at the head of the activation
+distribution.
+
+Read-only: does not mutate Qdrant, the workspace, or the activation log.
+
+Interpretation:
+  Dense range  < 0.1  AND sparseNorm range > 0.5 → embedding anisotropy
+  Dense range  > 0.2  AND sparseNorm range < 0.1 → sparse max-normalization
+  Both compressed                                → both contribute
+  Both wide                                      → channel mixing is the cause
+
+Examples:
+  $ assistant memory v2 explain --text "what's bothering me"
+  $ assistant memory v2 explain --text "..." --top 50
+  $ assistant memory v2 explain --text "..." --assistant-text "..." --now-text "..."`,
+    )
+    .action(
+      async (opts: {
+        text: string;
+        assistantText?: string;
+        nowText?: string;
+        top: string;
+      }) => {
+        const top = Number.parseInt(opts.top, 10);
+        if (!Number.isFinite(top) || top < 1) {
+          log.error("--top must be a positive integer");
+          process.exitCode = 1;
+          return;
+        }
+
+        const result = await cliIpcCall<MemoryV2ExplainSimilarityResult>(
+          "memory_v2_explain_similarity",
+          {
+            body: {
+              userText: opts.text,
+              assistantText: opts.assistantText,
+              nowText: opts.nowText,
+              top,
+            },
+          },
+        );
+
+        if (!result.ok) {
+          log.error(result.error ?? "Failed to run similarity diagnostic");
+          process.exitCode = 1;
+          return;
+        }
+
+        printExplainResult(result.result!);
+      },
+    );
+
+  // ── rebuild-corpus-stats ──────────────────────────────────────────────
+
+  v2.command("rebuild-corpus-stats")
+    .description(
+      "Rebuild the BM25 corpus stats (DF table + avg doc length) used by the sparse channel",
+    )
+    .addHelpText(
+      "after",
+      `
+Walks every concept page on disk and recomputes the document-frequency
+table and average document length used to weight BM25 sparse vectors.
+Atomic swap — the previous stats stay live until the new ones are ready.
+
+Run after bulk content imports, after manually editing many pages, or to
+recover from a startup rebuild that errored.
+
+Note: this only refreshes the in-memory stats used to *construct* new
+document-side sparse vectors. Existing sparse vectors stored in Qdrant
+are not refreshed by this command — pair with 'assistant memory v2
+reembed' if document-side weights need updating.
+
+Examples:
+  $ assistant memory v2 rebuild-corpus-stats`,
+    )
+    .action(async () => {
+      const result = await cliIpcCall<MemoryV2RebuildCorpusStatsResult>(
+        "memory_v2_rebuild_corpus_stats",
+        { body: {} },
+      );
+
+      if (!result.ok) {
+        log.error(result.error ?? "Failed to rebuild corpus stats");
+        process.exitCode = 1;
+        return;
+      }
+
+      const r = result.result!;
+      log.info(`Rebuilt BM25 corpus stats: ${r.totalDocs} docs.`);
+      log.info(`  avg doc length: ${r.avgDl.toFixed(2)} tokens`);
+      log.info(`  vocabulary buckets: ${r.vocabularyBuckets.toLocaleString()}`);
+    });
+
+  // ── fit-anisotropy ────────────────────────────────────────────────────
+
+  v2.command("fit-anisotropy")
+    .description(
+      "Fit the embedding anisotropy correction (Mu & Viswanath 'all-but-the-top')",
+    )
+    .option(
+      "--k <n>",
+      "Number of leading principal components to project out (default 1; raise to 2-3 only if PC2/PC3 also explain >5% variance)",
+      "1",
+    )
+    .option(
+      "--sample <n>",
+      "Maximum stored dense vectors to pull for the fit (default 5000)",
+      "5000",
+    )
+    .addHelpText(
+      "after",
+      `
+Modern transformer embeddings (Gemini in particular) are anisotropic — every
+vector lives in a narrow cone of the embedding space, which compresses cosine
+similarities into a tight band (typically 0.4-0.7) and lets a few dominant
+directions drown out semantic signal.
+
+This command samples stored dense vectors from the concept-page Qdrant
+collection, fits a corpus mean + top-k principal components, and persists the
+calibration. Subsequent embeds and queries apply the correction automatically:
+
+  vec' = vec - mean
+  for each pc_i: vec' = vec' - (vec' · pc_i) pc_i
+  vec' = vec' / ||vec'||
+
+After fitting, run 'assistant memory v2 reembed' to re-process every stored
+concept page through the new calibration. Until then, queries are corrected
+but stored vectors are not — the score range stays the same and retrieval may
+degrade. The calibration is keyed by (provider, model, dim), so changing
+embedding model invalidates it.
+
+Recommended k:
+  k=1   safest default. Removes the dominant common direction.
+  k=2-3 only when explained-variance ratios show PC2/PC3 each above ~5%.
+  k>3   not justified without a labeled retrieval eval set.
+
+Examples:
+  $ assistant memory v2 fit-anisotropy
+  $ assistant memory v2 fit-anisotropy --k 2
+  $ assistant memory v2 fit-anisotropy --sample 10000`,
+    )
+    .action(async (opts: { k: string; sample: string }) => {
+      const k = Number.parseInt(opts.k, 10);
+      const sample = Number.parseInt(opts.sample, 10);
+      if (!Number.isFinite(k) || k < 1) {
+        log.error("--k must be a positive integer");
+        process.exitCode = 1;
+        return;
+      }
+      if (!Number.isFinite(sample) || sample < 1) {
+        log.error("--sample must be a positive integer");
+        process.exitCode = 1;
+        return;
+      }
+
+      const result = await cliIpcCall<MemoryV2FitAnisotropyResult>(
+        "memory_v2_fit_anisotropy",
+        { body: { k, sample } },
+      );
+
+      if (!result.ok) {
+        log.error(result.error ?? "Failed to fit anisotropy calibration");
+        process.exitCode = 1;
+        return;
+      }
+
+      const r = result.result!;
+      log.info(
+        `Fit anisotropy calibration: ${r.provider}/${r.model} (dim=${r.dim})`,
+      );
+      log.info(`  samples: ${r.sampleCount}`);
+      log.info(`  total variance: ${r.totalVariance.toFixed(6)}`);
+      log.info("  explained variance per component:");
+      for (let i = 0; i < r.componentVariance.length; i++) {
+        const ratio = (r.explainedVarianceRatio[i] * 100).toFixed(2);
+        log.info(
+          `    PC${i + 1}: ${r.componentVariance[i].toFixed(6)} (${ratio}%)`,
+        );
+      }
+      log.info(`  written to: ${r.path}`);
+      log.info("");
+      log.info(
+        "Next step: run 'assistant memory v2 reembed' so every stored concept page is re-written under the new calibration. Until then, query-side correction operates on stored vectors that were embedded without it.",
+      );
+    });
+
   // ── validate ──────────────────────────────────────────────────────────
 
   v2.command("validate")

package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts

@@ -61,7 +61,6 @@ mock.module("../../../../config/loader.js", () => ({
   }),
   loadConfig: () => ({}),
   invalidateConfigCache: () => {},
-  saveConfig: () => {},
   loadRawConfig: () => ({}),
   saveRawConfig: () => {},
 }));

package/src/cli/commands/platform/__tests__/connect.test.ts

@@ -88,13 +88,11 @@ mock.module("../../../../config/loader.js", () => ({
   }),
   loadConfig: () => ({}),
   invalidateConfigCache: () => {},
-  saveConfig: () => {},
   loadRawConfig: () => ({}),
   saveRawConfig: () => {},
   getNestedValue: () => undefined,
   setNestedValue: () => {},
   applyNestedDefaults: (config: unknown) => config,
-  deepMergeMissing: () => false,
   deepMergeOverwrite: () => {},
   mergeDefaultWorkspaceConfig: () => {},
 }));

package/src/cli/commands/platform/__tests__/disconnect.test.ts

@@ -109,13 +109,11 @@ mock.module("../../../../config/loader.js", () => ({
   }),
   loadConfig: () => ({}),
   invalidateConfigCache: () => {},
-  saveConfig: () => {},
   loadRawConfig: () => ({}),
   saveRawConfig: () => {},
   getNestedValue: () => undefined,
   setNestedValue: () => {},
   applyNestedDefaults: (config: unknown) => config,
-  deepMergeMissing: () => false,
   deepMergeOverwrite: () => {},
   mergeDefaultWorkspaceConfig: () => {},
 }));

package/src/cli/commands/platform/__tests__/status.test.ts

@@ -75,6 +75,8 @@ mock.module("../../../../util/logger.js", () => ({
   LOG_FILE_PATTERN: /^assistant-(\d{4}-\d{2}-\d{2})\.log$/,
 }));
 
+// Also mock the CLI logger singleton so log.info calls do not write to stdout.
+
 mock.module("../../../../config/loader.js", () => ({
   API_KEY_PROVIDERS: [] as const,
   getConfig: () => ({
@@ -89,13 +91,11 @@ mock.module("../../../../config/loader.js", () => ({
   }),
   loadConfig: () => ({}),
   invalidateConfigCache: () => {},
-  saveConfig: () => {},
   loadRawConfig: () => ({}),
   saveRawConfig: () => {},
   getNestedValue: () => undefined,
   setNestedValue: () => {},
   applyNestedDefaults: (config: unknown) => config,
-  deepMergeMissing: () => false,
   deepMergeOverwrite: () => {},
   mergeDefaultWorkspaceConfig: () => {},
 }));
@@ -166,14 +166,15 @@ describe("assistant platform status", () => {
     process.exitCode = 0;
   });
 
-  test("connected platform returns full status with stored credentials", async () => {
+  test("platform pod returns full status from context", async () => {
     /**
-     * When the assistant has stored platform credentials and a valid
-     * registration context, the status command should report connected
-     * with all context fields populated.
+     * When the assistant is running as a platform-managed pod, the status
+     * command reports all fields from the registration context plus
+     * organizationId and userId from the keychain. The connected field
+     * is absent — platform status does not expose it.
      */
 
-    // GIVEN a containerized environment with platform configuration
+    // GIVEN a containerized platform environment
     mockResolvePlatformCallbackRegistrationContext = async () => ({
       isPlatform: true,
       platformBaseUrl: "https://platform.vellum.ai",
@@ -184,12 +185,9 @@ describe("assistant platform status", () => {
       enabled: true,
     });
 
-    // AND stored platform credentials exist
+    // AND credentials are stored in the keychain
     mockGetSecureKeyAsync = async (account: string) => {
-      if (account === "credential/vellum/platform_base_url")
-        return "https://platform.vellum.ai";
-      if (account === "credential/vellum/assistant_api_key")
-        return "sk-test-key";
+      if (account === "credential/vellum/webhook_secret") return "wh-secret";
       if (account === "credential/vellum/platform_organization_id")
         return "org-456";
       if (account === "credential/vellum/platform_user_id") return "user-789";
@@ -213,8 +211,8 @@ describe("assistant platform status", () => {
     expect(parsed.assistantId).toBe("asst-abc-123");
     expect(parsed.hasInternalApiKey).toBe(true);
     expect(parsed.hasAssistantApiKey).toBe(true);
+    expect(parsed.hasWebhookSecret).toBe(true);
     expect(parsed.available).toBe(true);
-    expect(parsed.connected).toBe(true);
     expect(parsed.organizationId).toBe("org-456");
     expect(parsed.userId).toBe("user-789");
   });
@@ -243,7 +241,7 @@ describe("assistant platform status", () => {
     // THEN the command succeeds
     expect(exitCode).toBe(0);
 
-    // AND stdout contains no JSON (writeOutput is skipped in plain text mode)
-    expect(stdout.trim()).toBe("");
+    // Plain-text mode logs via log.info — verify writeOutput (JSON) was NOT called
+    expect(() => JSON.parse(stdout.trim())).toThrow();
   });
 });

package/src/cli/commands/platform/disconnect.ts

@@ -95,17 +95,18 @@ Examples:
 
         const failedKeys: string[] = [];
         for (const key of keysToDelete) {
-          const result = await deleteSecureKeyViaDaemon(
+          const delResult = await deleteSecureKeyViaDaemon(
             "credential",
             `${key.service}:${key.field}`,
           );
-          if (result === "error") {
-            failedKeys.push(`${key.service}:${key.field}`);
+          if (delResult.result === "error") {
+            const detail = delResult.error ? `: ${delResult.error}` : "";
+            failedKeys.push(`${key.service}:${key.field}${detail}`);
           }
         }
 
         if (failedKeys.length > 0) {
-          writeError(`Failed to delete credentials: ${failedKeys.join(", ")}`);
+          writeError(`Failed to delete credentials: ${failedKeys.join("; ")}`);
           return;
         }
 

package/src/cli/commands/platform/index.ts

@@ -68,7 +68,6 @@ Fields:
   hasWebhookSecret    Whether a stored webhook secret is available (needed
                       for email and other inbound webhook channels)
   available           Whether callback registration prerequisites are satisfied
-  connected           Whether platform credentials are stored (boolean)
   organizationId      The platform organization ID (from stored credentials)
   userId              The platform user ID (from stored credentials)
 
@@ -80,19 +79,6 @@ Examples:
       try {
         const context = await resolvePlatformCallbackRegistrationContext();
 
-        const storedBaseUrl =
-          (await getSecureKeyAsync(
-            credentialKey(
-              CREDENTIAL_KEYS.baseUrl.service,
-              CREDENTIAL_KEYS.baseUrl.field,
-            ),
-          )) ?? "";
-        const hasStoredApiKey = !!(await getSecureKeyAsync(
-          credentialKey(
-            CREDENTIAL_KEYS.apiKey.service,
-            CREDENTIAL_KEYS.apiKey.field,
-          ),
-        ));
         const organizationId =
           (
             await getSecureKeyAsync(
@@ -116,8 +102,6 @@ Examples:
           credentialKey("vellum", "webhook_secret"),
         ));
 
-        const connected = !!storedBaseUrl && hasStoredApiKey;
-
         const result = {
           isPlatform: context.isPlatform,
           baseUrl: context.platformBaseUrl,
@@ -126,7 +110,6 @@ Examples:
           hasAssistantApiKey: context.hasAssistantApiKey,
           hasWebhookSecret,
           available: context.enabled,
-          connected,
           organizationId: organizationId || null,
           userId: userId || null,
         };
@@ -149,7 +132,6 @@ Examples:
           log.info(
             `Callback registration available: ${result.available ? "yes" : "no"}`,
           );
-          log.info(`Connected: ${connected}`);
           log.info(`Organization ID: ${organizationId || "(not set)"}`);
           log.info(`User ID: ${userId || "(not set)"}`);
         }