@vellumai/assistant 0.10.12 → 0.11.0-staging.2

package/src/telemetry/memory-corpus-size.ts

@@ -0,0 +1,184 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+import { getWorkspaceDir } from "../util/platform.js";
+
+/**
+ * Coarse on-disk size of the assistant's memory corpus, in the snake_case
+ * shape the `memory_tier` watchdog `detail` bag carries to the platform.
+ *
+ * Both corpora are measured on every tier rather than branching on the tier
+ * itself: an assistant part-way through a v1 -> v2 -> v3 migration has content
+ * in both trees, and a `memory.enabled: false` assistant still has whatever it
+ * accumulated before being switched off. Reporting both unconditionally keeps
+ * the numbers legible across a migration instead of blinking to zero the
+ * moment a tier flips.
+ *
+ * KNOWN GAP: this measures on-disk Markdown only. It does NOT count
+ * `memory_graph_nodes`, which is a real part of a v1 assistant's learned state
+ * — `v2/migration.ts` gathers graph nodes alongside PKB Markdown when it
+ * migrates v1 — so a v1 assistant rich in graph nodes but holding only the
+ * seeded PKB files reads as near-empty here. Counting them means opening the
+ * memory SQLite DB from the resource-monitor process, which this probe
+ * deliberately does not do. Treat `pkb_files` as a lower bound on v1 corpus
+ * size and confirm v1 assistants directly before sizing a migration off it.
+ */
+export interface MemoryCorpusSize {
+  /** `.md` files under `memory/concepts/` — the v2/v3 concept-page count. */
+  concept_pages: number;
+  /** Total bytes of those concept pages. */
+  concept_bytes: number;
+  /** `.md` files under `pkb/` — the v1 personal-knowledge-base corpus. */
+  pkb_files: number;
+  /** Total bytes of those PKB files. */
+  pkb_bytes: number;
+  /**
+   * Non-empty lines in `memory/buffer.md` — remembered facts still waiting on
+   * consolidation. Matches `countBufferLines` in the substrate's
+   * consolidation job: for a well-formed buffer, one non-empty line is one
+   * entry.
+   */
+  buffer_lines: number;
+}
+
+/**
+ * Ceiling on directory entries examined per tree. The largest corpus observed
+ * in the fleet is in the low hundreds of pages, so this never binds in
+ * practice; it exists so a pathological workspace cannot turn a telemetry
+ * probe into a long walk.
+ *
+ * It counts every dirent seen — directories and non-Markdown files included —
+ * not just the Markdown matches. A cap on matches alone would not bound the
+ * walk at all: a tree of a million empty directories contains zero Markdown
+ * files and would still be traversed in full by this synchronous probe.
+ *
+ * A tree that hits the cap returns what it counted so far, making the result
+ * a floor rather than an error.
+ */
+const MAX_ENTRIES_PER_TREE = 20_000;
+
+interface TreeSize {
+  files: number;
+  bytes: number;
+}
+
+const EMPTY_TREE: TreeSize = { files: 0, bytes: 0 };
+
+/**
+ * Recursively total the `.md` files under `dir`.
+ *
+ * Deliberately synchronous, and deliberately not routed through the memory
+ * plugin's `listPages` / `getPageIndex`. This runs in the resource-monitor
+ * process, where the plugin's page-index cache is cold — going through it
+ * would read and parse every concept page on each six-hour cycle, and would
+ * add an import out of `telemetry/` into the plugin tree that the plugin
+ * import-boundary guard exists to prevent. A bare readdir + stat is the whole
+ * job here.
+ *
+ * `withFileTypes` dirents carry lstat semantics, so a symlinked directory
+ * reports as a symlink rather than a directory and is skipped — the walk
+ * cannot follow a link into a cycle, and needs no visited-inode bookkeeping.
+ *
+ * `maxEntries` is a parameter only so the ceiling is testable without
+ * materializing twenty thousand files; production callers take the default.
+ */
+export function measureMarkdownTree(
+  dir: string,
+  maxEntries: number = MAX_ENTRIES_PER_TREE,
+): TreeSize {
+  let files = 0;
+  let bytes = 0;
+  let visited = 0;
+  const pending: string[] = [dir];
+
+  while (pending.length > 0) {
+    const current = pending.pop();
+    if (current === undefined) {
+      break;
+    }
+
+    let entries;
+    try {
+      entries = readdirSync(current, { withFileTypes: true });
+    } catch {
+      // A missing or unreadable subtree contributes nothing. The common case
+      // is ENOENT: the tier that owns this tree was never active here.
+      continue;
+    }
+
+    for (const entry of entries) {
+      visited += 1;
+      if (visited > maxEntries) {
+        return { files, bytes };
+      }
+      if (entry.isDirectory()) {
+        pending.push(join(current, entry.name));
+        continue;
+      }
+      if (!entry.isFile() || !entry.name.endsWith(".md")) {
+        continue;
+      }
+      files += 1;
+      try {
+        bytes += statSync(join(current, entry.name)).size;
+      } catch {
+        // Counted in `files` but contributes no bytes — a file that vanished
+        // between the readdir and the stat is a race, not a failure.
+      }
+    }
+  }
+
+  return { files, bytes };
+}
+
+/** Non-empty-line count of the file at `path`; 0 when it is absent or empty. */
+function countNonEmptyLines(path: string): number {
+  let content: string;
+  try {
+    content = readFileSync(path, "utf8");
+  } catch {
+    return 0;
+  }
+  if (content.length === 0) {
+    return 0;
+  }
+  return content.split("\n").filter((line) => line.trim().length > 0).length;
+}
+
+/**
+ * Measure the on-disk memory corpus for this assistant.
+ *
+ * Never throws: every probe degrades to zero independently, so a corpus that
+ * is partly unreadable still reports the part that is.
+ */
+export function measureMemoryCorpusSize(): MemoryCorpusSize {
+  const workspaceDir = getWorkspaceDir();
+
+  let concepts: TreeSize = EMPTY_TREE;
+  let pkb: TreeSize = EMPTY_TREE;
+  let bufferLines = 0;
+
+  try {
+    concepts = measureMarkdownTree(join(workspaceDir, "memory", "concepts"));
+  } catch {
+    concepts = EMPTY_TREE;
+  }
+  try {
+    pkb = measureMarkdownTree(join(workspaceDir, "pkb"));
+  } catch {
+    pkb = EMPTY_TREE;
+  }
+  try {
+    bufferLines = countNonEmptyLines(join(workspaceDir, "memory", "buffer.md"));
+  } catch {
+    bufferLines = 0;
+  }
+
+  return {
+    concept_pages: concepts.files,
+    concept_bytes: concepts.bytes,
+    pkb_files: pkb.files,
+    pkb_bytes: pkb.bytes,
+    buffer_lines: bufferLines,
+  };
+}

package/src/telemetry/memory-tier-reporter.test.ts

@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 import { memoryTier } from "../config/memory-tier.js";
 import type { AssistantConfig } from "../config/schema.js";
 import type { ConsentState } from "../platform/consent-cache.js";
+import type { MemoryCorpusSize } from "./memory-corpus-size.js";
 import type { WatchdogEventRecord } from "./watchdog-events-store.js";
 
 // Mutable stubs flipped per test. bun's `mock.module` patches retroactively
@@ -14,6 +15,27 @@ let consent: ConsentState = true;
 let currentConfig: AssistantConfig = {} as AssistantConfig;
 let recorded: WatchdogEventRecord[] = [];
 
+// Stubbed so the assertions below stay exact-match: the real probe walks the
+// live workspace, whose contents differ per machine. `measureMemoryCorpusSize`
+// has its own tests against a temp tree.
+const ZERO_CORPUS: MemoryCorpusSize = {
+  concept_pages: 0,
+  concept_bytes: 0,
+  pkb_files: 0,
+  pkb_bytes: 0,
+  buffer_lines: 0,
+};
+let corpus: MemoryCorpusSize = ZERO_CORPUS;
+let corpusError: Error | null = null;
+
+mock.module("./memory-corpus-size.js", () => ({
+  measureMemoryCorpusSize: () => {
+    if (corpusError) {
+      throw corpusError;
+    }
+    return corpus;
+  },
+}));
 mock.module("../platform/consent-cache.js", () => ({
   getRawShareAnalytics: () => consent,
 }));
@@ -51,6 +73,8 @@ describe("memory-tier-reporter", () => {
     consent = true;
     currentConfig = {} as AssistantConfig;
     recorded = [];
+    corpus = ZERO_CORPUS;
+    corpusError = null;
     // Clear any interval/boot-retry timer a prior test's start left behind.
     stopMemoryTierReporter();
   });
@@ -69,7 +93,7 @@ describe("memory-tier-reporter", () => {
     expect(recorded).toHaveLength(1);
     expect(recorded[0]).toEqual({
       checkName: "memory_tier",
-      detail: { tier: memoryTier(config) },
+      detail: { tier: memoryTier(config), ...ZERO_CORPUS },
     });
     expect(recorded[0]?.detail?.tier).toBe("v2");
   });
@@ -78,6 +102,33 @@ describe("memory-tier-reporter", () => {
     currentConfig = makeConfig(true, true, true); // v3 live wins over v2
     recordMemoryTierOnce();
 
+    expect(recorded[0]?.detail).toEqual({ tier: "v3", ...ZERO_CORPUS });
+  });
+
+  test("carries the measured corpus size alongside the tier", () => {
+    currentConfig = makeConfig(true, true, true);
+    corpus = {
+      concept_pages: 42,
+      concept_bytes: 133_700,
+      pkb_files: 3,
+      pkb_bytes: 900,
+      buffer_lines: 7,
+    };
+
+    recordMemoryTierOnce();
+
+    expect(recorded[0]?.detail).toEqual({ tier: "v3", ...corpus });
+  });
+
+  test("a failed corpus probe still emits the tier heartbeat", () => {
+    // The fleet-mix dashboards read `detail.tier`; a sizing failure must
+    // degrade the payload, never drop the event and gap the series.
+    currentConfig = makeConfig(true, true, true);
+    corpusError = new Error("workspace unreadable");
+
+    recordMemoryTierOnce();
+
+    expect(recorded).toHaveLength(1);
     expect(recorded[0]?.detail).toEqual({ tier: "v3" });
   });
 
@@ -91,7 +142,7 @@ describe("memory-tier-reporter", () => {
     for (const event of recorded) {
       expect(event).toEqual({
         checkName: "memory_tier",
-        detail: { tier: "v1" },
+        detail: { tier: "v1", ...ZERO_CORPUS },
       });
     }
   });
@@ -117,6 +168,20 @@ describe("memory-tier-reporter", () => {
     expect(recorded).toHaveLength(1);
   });
 
+  test("a confirmed opt-out skips the corpus probe entirely", () => {
+    // The event is dropped one layer down, so walking the workspace for it
+    // would be pure cost on an assistant that asked us not to collect. The
+    // throwing stub asserts the probe is never reached.
+    consent = false;
+    currentConfig = makeConfig(true, true);
+    corpusError = new Error("probe must not run");
+
+    recordMemoryTierOnce();
+
+    expect(recorded).toHaveLength(1);
+    expect(recorded[0]?.detail).toEqual({ tier: "v2" });
+  });
+
   test("startMemoryTierReporter is a no-op under VELLUM_DEV=1", () => {
     process.env.VELLUM_DEV = "1";
     consent = true; // consent known → a non-dev start would emit immediately

package/src/telemetry/memory-tier-reporter.ts

@@ -2,6 +2,7 @@ import { getConfigReadOnly } from "../config/loader.js";
 import { memoryTier } from "../config/memory-tier.js";
 import { getRawShareAnalytics } from "../platform/consent-cache.js";
 import { getLogger } from "../util/logger.js";
+import { measureMemoryCorpusSize } from "./memory-corpus-size.js";
 import { recordWatchdogEvent } from "./watchdog-events-store.js";
 
 const log = getLogger("memory-tier-reporter");
@@ -17,7 +18,14 @@ const log = getLogger("memory-tier-reporter");
  */
 const MEMORY_TIER_INTERVAL_MS = 6 * 60 * 60 * 1000;
 
-/** Watchdog `check_name` carrying the coarse memory tier in `detail.tier`. */
+/**
+ * Watchdog `check_name` carrying the coarse memory tier in `detail.tier`,
+ * alongside the on-disk corpus size (see {@link measureMemoryCorpusSize}).
+ *
+ * The check name and the `detail.tier` bucket strings are frozen — platform
+ * dashboards group on them. The rest of `detail` is an open bag, so the
+ * corpus-size keys are additive and need no wire, serializer, or dbt change.
+ */
 const MEMORY_TIER_CHECK_NAME = "memory_tier";
 
 // The boot-time emit races the first consent refresh (fire-and-forget in the
@@ -38,18 +46,37 @@ const BOOT_RETRY_INTERVAL_MS = 60_000;
  * no-ops on a `false` share_analytics, so the call still goes through and the
  * record layer drops it. Never throws: a storage failure is logged and
  * retried on the next cycle.
+ *
+ * The event also carries the on-disk corpus size. That probe is best-effort
+ * and isolated: if it fails the event still goes out with the tier alone,
+ * because the tier heartbeat is what the fleet-mix dashboards depend on and a
+ * sizing failure must not create a gap in that series. It is also skipped
+ * outright on a confirmed opt-out — the event is dropped one layer down, so
+ * walking the workspace for it would be pure cost on the assistants that
+ * asked us not to collect.
  */
 export function recordMemoryTierOnce(): void {
-  if (getRawShareAnalytics() === "unknown") {
+  const consent = getRawShareAnalytics();
+  if (consent === "unknown") {
     return;
   }
   try {
     // `getConfigReadOnly()` re-reads config.json on change (capturing live
     // edits) and never writes to disk — safe for the monitor process.
     const tier = memoryTier(getConfigReadOnly());
+
+    let corpus: Record<string, number> = {};
+    if (consent !== false) {
+      try {
+        corpus = { ...measureMemoryCorpusSize() };
+      } catch (err) {
+        log.warn({ err }, "memory corpus sizing failed (emitting tier only)");
+      }
+    }
+
     recordWatchdogEvent({
       checkName: MEMORY_TIER_CHECK_NAME,
-      detail: { tier },
+      detail: { tier, ...corpus },
     });
   } catch (err) {
     log.warn({ err }, "memory_tier watchdog emit failed (non-fatal)");

package/src/tools/__tests__/conversation-group-tool-input-schemas.test.ts

@@ -0,0 +1,61 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, test } from "bun:test";
+
+import type { z } from "zod";
+
+import { conversationGroupCreateInputSchema } from "../conversation-groups/group_create.js";
+import { conversationMoveToGroupInputSchema } from "../conversation-groups/move_to_group.js";
+import { toToolInputSchema } from "../shared/zod-tool-schema.js";
+import { expectNoSchemaDrift } from "./tool-schema-drift-harness.js";
+
+/**
+ * Drift guard between the conversation-groups skill's hand-written
+ * `TOOLS.json` `input_schema`s and the executors' runtime Zod schemas. See
+ * `tool-schema-drift-harness.ts` for what is (and deliberately is not)
+ * compared. `conversation_group_list` takes no input and has no runtime
+ * schema, so it has no entry here.
+ */
+
+const toolsJson = JSON.parse(
+  readFileSync(
+    join(
+      import.meta.dir,
+      "../../config/bundled-skills/conversation-groups/TOOLS.json",
+    ),
+    "utf8",
+  ),
+) as Parameters<typeof expectNoSchemaDrift>[1];
+
+const CASES: {
+  name: string;
+  schema: z.ZodType;
+  advertiseRequired?: string[];
+}[] = [
+  {
+    name: "conversation_group_create",
+    schema: conversationGroupCreateInputSchema,
+    advertiseRequired: ["name"],
+  },
+  {
+    name: "conversation_move_to_group",
+    schema: conversationMoveToGroupInputSchema,
+    advertiseRequired: ["group"],
+  },
+];
+
+describe("conversation-groups TOOLS.json ↔ runtime Zod schema drift guard", () => {
+  for (const c of CASES) {
+    test(c.name, () => {
+      expectNoSchemaDrift(
+        {
+          name: c.name,
+          derived: toToolInputSchema(c.schema, {
+            advertiseRequired: c.advertiseRequired,
+          }),
+        },
+        toolsJson,
+      );
+    });
+  }
+});

package/src/tools/__tests__/document-tool-input-schemas.test.ts

@@ -0,0 +1,103 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, test } from "bun:test";
+
+import type { z } from "zod";
+
+import {
+  commentListInputSchema,
+  commentReplyInputSchema,
+  commentResolveInputSchema,
+} from "../document/document-comment-tool.js";
+import {
+  documentCreateInputSchema,
+  documentDeleteInputSchema,
+  documentFindInputSchema,
+  documentListInputSchema,
+  documentOpenInputSchema,
+  documentReadInputSchema,
+  documentReplaceTextInputSchema,
+  documentUpdateInputSchema,
+} from "../document/document-tool.js";
+import { toToolInputSchema } from "../shared/zod-tool-schema.js";
+import { expectNoSchemaDrift } from "./tool-schema-drift-harness.js";
+
+/**
+ * Drift guard between the document-editor skill's hand-written `TOOLS.json`
+ * `input_schema`s (the advertised contract) and the executors' runtime Zod
+ * schemas. See `tool-schema-drift-harness.ts` for what is (and deliberately
+ * is not) compared.
+ */
+
+const TOOLS_JSON = JSON.parse(
+  readFileSync(
+    join(
+      import.meta.dir,
+      "../../config/bundled-skills/document-editor/TOOLS.json",
+    ),
+    "utf8",
+  ),
+) as Parameters<typeof expectNoSchemaDrift>[1];
+
+const CASES: {
+  name: string;
+  schema: z.ZodType;
+  advertiseRequired?: string[];
+  passthrough?: Record<string, string>;
+}[] = [
+  {
+    name: "document_open",
+    schema: documentOpenInputSchema,
+    advertiseRequired: ["surface_id"],
+  },
+  { name: "document_create", schema: documentCreateInputSchema },
+  {
+    name: "document_update",
+    schema: documentUpdateInputSchema,
+    advertiseRequired: ["content"],
+    passthrough: {
+      mode: "bespoke check owns the error message and the { mode: null } tolerance that mirrors validateInputAgainstSchema",
+    },
+  },
+  {
+    name: "document_read",
+    schema: documentReadInputSchema,
+    advertiseRequired: ["surface_id"],
+  },
+  { name: "document_list", schema: documentListInputSchema },
+  {
+    name: "document_delete",
+    schema: documentDeleteInputSchema,
+    advertiseRequired: ["surface_id"],
+  },
+  {
+    name: "document_find",
+    schema: documentFindInputSchema,
+    advertiseRequired: ["surface_id"],
+  },
+  {
+    name: "document_replace_text",
+    schema: documentReplaceTextInputSchema,
+    advertiseRequired: ["surface_id", "replace"],
+  },
+  { name: "comment_list", schema: commentListInputSchema },
+  { name: "comment_resolve", schema: commentResolveInputSchema },
+  { name: "comment_reply", schema: commentReplyInputSchema },
+];
+
+describe("document-editor TOOLS.json ↔ runtime Zod schema drift guard", () => {
+  for (const c of CASES) {
+    test(c.name, () => {
+      expectNoSchemaDrift(
+        {
+          name: c.name,
+          derived: toToolInputSchema(c.schema, {
+            advertiseRequired: c.advertiseRequired,
+          }),
+          passthrough: c.passthrough,
+        },
+        TOOLS_JSON,
+      );
+    });
+  }
+});

package/src/tools/__tests__/followup-call-acp-tool-input-schemas.test.ts

@@ -0,0 +1,125 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, test } from "bun:test";
+
+import type { z } from "zod";
+
+import { acpAbortInputSchema } from "../acp/abort.js";
+import { acpSpawnInputSchema } from "../acp/spawn.js";
+import { acpStatusInputSchema } from "../acp/status.js";
+import { acpSteerInputSchema } from "../acp/steer.js";
+import { callEndInputSchema } from "../calls/call-end.js";
+import { callStartInputSchema } from "../calls/call-start.js";
+import { callStatusInputSchema } from "../calls/call-status.js";
+import { followupCreateInputSchema } from "../followups/followup_create.js";
+import { followupListInputSchema } from "../followups/followup_list.js";
+import { followupResolveInputSchema } from "../followups/followup_resolve.js";
+import { toToolInputSchema } from "../shared/zod-tool-schema.js";
+import { expectNoSchemaDrift } from "./tool-schema-drift-harness.js";
+
+/**
+ * Drift guard between the followups / phone-calls / acp skills' hand-written
+ * `TOOLS.json` `input_schema`s (the advertised contract) and the executors'
+ * runtime Zod schemas. See `tool-schema-drift-harness.ts` for what is (and
+ * deliberately is not) compared. `acp_list_agents` takes no input and has no
+ * runtime schema, so it has no entry here.
+ */
+
+function loadToolsJson(
+  skill: string,
+): Parameters<typeof expectNoSchemaDrift>[1] {
+  return JSON.parse(
+    readFileSync(
+      join(import.meta.dir, `../../config/bundled-skills/${skill}/TOOLS.json`),
+      "utf8",
+    ),
+  ) as Parameters<typeof expectNoSchemaDrift>[1];
+}
+
+const CASES: {
+  skill: string;
+  name: string;
+  schema: z.ZodType;
+  advertiseRequired?: string[];
+  passthrough?: Record<string, string>;
+}[] = [
+  {
+    skill: "followups",
+    name: "followup_create",
+    schema: followupCreateInputSchema,
+    advertiseRequired: ["channel", "conversation_id"],
+    passthrough: {
+      expected_response_hours:
+        "bespoke positive-number check owns the error message for every malformed shape (per LUM-2856)",
+    },
+  },
+  {
+    skill: "followups",
+    name: "followup_list",
+    schema: followupListInputSchema,
+    passthrough: {
+      status:
+        "bespoke VALID_STATUSES check owns the 'Invalid status' error (test-asserted)",
+    },
+  },
+  {
+    skill: "followups",
+    name: "followup_resolve",
+    schema: followupResolveInputSchema,
+  },
+  {
+    skill: "phone-calls",
+    name: "call_start",
+    schema: callStartInputSchema,
+    passthrough: {
+      skip_disclosure: "deliberate `=== true` coercion owns malformed values",
+    },
+  },
+  {
+    skill: "phone-calls",
+    name: "call_status",
+    schema: callStatusInputSchema,
+  },
+  {
+    skill: "phone-calls",
+    name: "call_end",
+    schema: callEndInputSchema,
+    advertiseRequired: ["call_session_id"],
+  },
+  {
+    skill: "acp",
+    name: "acp_spawn",
+    schema: acpSpawnInputSchema,
+    advertiseRequired: ["task"],
+  },
+  { skill: "acp", name: "acp_status", schema: acpStatusInputSchema },
+  {
+    skill: "acp",
+    name: "acp_abort",
+    schema: acpAbortInputSchema,
+    advertiseRequired: ["acp_session_id"],
+  },
+  {
+    skill: "acp",
+    name: "acp_steer",
+    schema: acpSteerInputSchema,
+    advertiseRequired: ["acp_session_id", "instruction"],
+  },
+];
+
+describe("followups + phone-calls + acp TOOLS.json ↔ runtime Zod schema drift guard", () => {
+  for (const c of CASES) {
+    test(`${c.skill}:${c.name}`, () => {
+      expectNoSchemaDrift(
+        {
+          name: c.name,
+          derived: toToolInputSchema(c.schema, {
+            advertiseRequired: c.advertiseRequired,
+          }),
+          passthrough: c.passthrough,
+        },
+        loadToolsJson(c.skill),
+      );
+    });
+  }
+});