@vellumai/assistant 0.10.9 → 0.10.10-dev.202607162206.d08e98e

package/src/__tests__/telemetry-routes.test.ts

@@ -18,9 +18,7 @@ function pendingPayloads(): OnboardingResearchTelemetryEvent[] {
   );
 }
 
-const route = ROUTES.find(
-  (r) => r.operationId === "telemetry_onboarding_research",
-);
+const route = ROUTES.find((r) => r.operationId === "telemetry_ingest");
 
 function call(body: unknown) {
   if (!route) {
@@ -29,18 +27,28 @@ function call(body: unknown) {
   return route.handler({ body } as RouteHandlerArgs);
 }
 
-const VALID_BODY = {
+/** Well-formed onboarding_research `fields` (base fields stamped by the daemon). */
+const VALID_FIELDS = {
   conversation_id: "conv-xyz",
   status: "done",
+  self_reported_occupation: "engineer",
   claims: [{ claim: "Senior engineer", confidence: "confident", sources: [] }],
-  suggestions: [
-    { suggestion: "I'll find 3 papers", prompt: "Find me 3 papers" },
-  ],
+  claim_count: 1,
+  claims_confident: 1,
+  claims_maybe: 0,
+  claims_guessing: 0,
+  suggestions: [{ suggestion: "I'll find 3 papers", prompt: "Find me 3 papers" }],
+  suggestion_count: 1,
   plugins: ["marketing-expert"],
   installed_plugins: ["marketing-expert", "web-research"],
 };
 
-describe("telemetry-routes: onboarding-research", () => {
+const VALID_BODY = {
+  type: "onboarding_research",
+  fields: VALID_FIELDS,
+};
+
+describe("telemetry-routes: ingest", () => {
   beforeEach(() => {
     setShareAnalytics(true);
     setShareDiagnostics(true);
@@ -49,13 +57,13 @@ describe("telemetry-routes: onboarding-research", () => {
 
   test("route policy matches the other client-facing telemetry routes", () => {
     expect(route).toBeDefined();
-    expect(route?.endpoint).toBe("telemetry/onboarding-research");
+    expect(route?.endpoint).toBe("telemetry/ingest");
     expect(route?.method).toBe("POST");
     expect(route?.policy?.allowedPrincipalTypes).toEqual(ACTOR_PRINCIPALS);
     expect(route?.policy?.requiredScopes).toEqual(["settings.write"]);
   });
 
-  test("valid body is persisted to the outbox as a wire onboarding_research event", () => {
+  test("valid body is persisted to the outbox as a wire event", () => {
     const result = call(VALID_BODY);
     expect(result).toEqual({ id: expect.any(String) });
 
@@ -71,6 +79,26 @@ describe("telemetry-routes: onboarding-research", () => {
       plugins: ["marketing-expert"],
       installed_plugins: ["marketing-expert", "web-research"],
     });
+    // Base fields are stamped by the daemon, not the client.
+    expect(payloads[0]?.daemon_event_id).toEqual(expect.any(String));
+    expect(payloads[0]?.recorded_at).toEqual(expect.any(Number));
+    expect(payloads[0]?.assistant_version).toEqual(expect.any(String));
+  });
+
+  test("stamps a fresh daemon_event_id by default", () => {
+    call(VALID_BODY);
+    // No override → the collapse key falls back to the row id (a uuid), not the
+    // conversation-scoped key.
+    expect(pendingPayloads()[0]?.daemon_event_id).not.toBe(
+      "onboarding_research:conv-xyz",
+    );
+  });
+
+  test("honors a client-supplied daemon_event_id collapse key", () => {
+    call({ ...VALID_BODY, daemon_event_id: "onboarding_research:conv-xyz" });
+    expect(pendingPayloads()[0]?.daemon_event_id).toBe(
+      "onboarding_research:conv-xyz",
+    );
   });
 
   test("returns skipped and persists nothing under the analytics opt-out", () => {
@@ -79,19 +107,71 @@ describe("telemetry-routes: onboarding-research", () => {
     expect(pendingPayloads().length).toBe(0);
   });
 
-  test("returns skipped and persists nothing under the diagnostics opt-out", () => {
-    setShareDiagnostics(false);
-    expect(call(VALID_BODY)).toEqual({ skipped: true });
+  test("persists regardless of the diagnostics opt-out (rides analytics only)", () => {
+    setShareDiagnostics(false, "2000-01-01");
+    expect(call(VALID_BODY)).toEqual({ id: expect.any(String) });
+    expect(pendingPayloads().length).toBe(1);
+  });
+
+  test("rejects a non-outbox (watermark) or unknown type", () => {
+    // `turn` is a real wire type but watermark-flushed, not outbox-backed, so it
+    // has no ingest variant and a client can't inject it.
+    expect(() => call({ type: "turn", fields: {} })).toThrow(RouteError);
+    // An unknown type is rejected the same way.
+    expect(() => call({ type: "not_a_type", fields: {} })).toThrow(RouteError);
     expect(pendingPayloads().length).toBe(0);
   });
 
-  test("rejects a malformed body without persisting", () => {
-    expect(() => call({ ...VALID_BODY, status: "not-a-status" })).toThrow(
-      RouteError,
-    );
-    expect(() => call({ ...VALID_BODY, claims: [{ claim: "x" }] })).toThrow(
-      RouteError,
+  test("accepts any outbox-backed type, e.g. config_setting", () => {
+    const result = call({
+      type: "config_setting",
+      fields: { config_key: "voice.provider", config_value: "elevenlabs" },
+    });
+    expect(result).toEqual({ id: expect.any(String) });
+
+    const payloads = pendingOutboxPayloads<{ config_key: string }>(
+      "config_setting",
     );
+    expect(payloads.length).toBe(1);
+    expect(payloads[0]).toMatchObject({
+      type: "config_setting",
+      config_key: "voice.provider",
+      config_value: "elevenlabs",
+    });
+  });
+
+  test("rejects malformed fields (missing or mistyped) without persisting", () => {
+    // Missing the required derived counts.
+    const { claim_count: _c, suggestion_count: _s, ...missingCounts } =
+      VALID_FIELDS;
+    expect(() =>
+      call({ type: "onboarding_research", fields: missingCounts }),
+    ).toThrow(RouteError);
+
+    // Wrong type for a numeric field.
+    expect(() =>
+      call({
+        type: "onboarding_research",
+        fields: { ...VALID_FIELDS, claim_count: "one" },
+      }),
+    ).toThrow(RouteError);
+
+    expect(pendingPayloads().length).toBe(0);
+  });
+
+  test("rejects a structurally invalid request body", () => {
+    expect(() => call({ type: "onboarding_research" })).toThrow(RouteError);
+    expect(() => call({ fields: VALID_FIELDS })).toThrow(RouteError);
     expect(pendingPayloads().length).toBe(0);
   });
+
+  test("strips unknown fields keys (typed schema, not a passthrough record)", () => {
+    call({
+      type: "onboarding_research",
+      fields: { ...VALID_FIELDS, bogus_extra: "nope" },
+    });
+    const payloads = pendingPayloads();
+    expect(payloads.length).toBe(1);
+    expect(payloads[0]).not.toHaveProperty("bogus_extra");
+  });
 });

package/src/__tests__/tool-audit.test.ts

@@ -1,11 +1,11 @@
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
-// Toggle for the share_analytics opt-out gate consulted when populating the
-// telemetry columns.
-let shareAnalytics = true;
+// Tri-state share_analytics consent consulted when populating the telemetry
+// columns: only a confirmed `false` NULLs them.
+let shareAnalytics: boolean | "unknown" = true;
 
 mock.module("../platform/consent-cache.js", () => ({
-  getCachedShareAnalytics: () => shareAnalytics,
+  getRawShareAnalytics: () => shareAnalytics,
 }));
 
 // Capture the audit rows the terminals would insert, and the lifecycle-event
@@ -379,6 +379,36 @@ describe("tool audit terminals", () => {
     });
   });
 
+  test("populates telemetry columns under unknown consent (cold cache)", () => {
+    // NULL columns are permanently excluded by the projection's
+    // arg_bytes IS NOT NULL filter, so only a CONFIRMED opt-out may NULL
+    // them — unknown records, and flush-time consent gating decides later.
+    shareAnalytics = "unknown";
+
+    recordToolExecuted({
+      conversationId: "conv-unknown",
+      toolName: "file_read",
+      input: { path: "/tmp/a" },
+      resultContent: "ok",
+      resultBytes: 99,
+      decision: "allow",
+      riskLevel: "low",
+      durationMs: 12,
+      attribution: ATTRIBUTION,
+      wasPrompted: false,
+    });
+
+    expect(records).toHaveLength(1);
+    expect(records[0]).toMatchObject({
+      argBytes: Buffer.byteLength(JSON.stringify({ path: "/tmp/a" }), "utf8"),
+      resultBytes: 99,
+      provider: "anthropic",
+      model: "model-a",
+      inferenceProfile: "balanced",
+      inferenceProfileSource: "active",
+    });
+  });
+
   test("maps attribution onto executed rows and leaves denied rows null", () => {
     recordToolExecuted({
       conversationId: "conv-attr",

package/src/__tests__/tool-executor-lifecycle-events.test.ts

@@ -81,7 +81,7 @@ mock.module("../telemetry/tool-audit.js", () => ({
 // Analytics consent is granted so any consent-gated telemetry path the audit
 // terminals consult sees the opted-in state.
 mock.module("../platform/consent-cache.js", () => ({
-  getCachedShareAnalytics: () => true,
+  getRawShareAnalytics: () => true,
 }));
 
 mock.module("../permissions/checker.js", () => ({

package/src/__tests__/tool-profiler.test.ts

@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
 
 import {
   disposeToolProfiler,
@@ -99,6 +99,55 @@ describe("ToolProfiler", () => {
   });
 });
 
+/**
+ * A `process.memoryUsage` replacement that reproduces the Bun 1.3.11
+ * `SystemError: Failed to get memory usage, errno: 2` syscall failure. Cast to
+ * the real signature because an always-throwing arrow infers a `never` return.
+ */
+const throwingMemoryUsage = (() => {
+  throw new Error("Failed to get memory usage, errno: 2");
+}) as unknown as typeof process.memoryUsage;
+
+describe("ToolProfiler RSS sampling failures", () => {
+  let profiler: ToolProfiler;
+
+  beforeEach(() => {
+    profiler = new ToolProfiler();
+  });
+
+  afterEach(() => {
+    // Restore the real implementation between cases.
+    spyOn(process, "memoryUsage").mockRestore();
+  });
+
+  test("startRequest tolerates a throwing memoryUsage syscall", () => {
+    // Simulate the Bun 1.3.11 `SystemError: Failed to get memory usage` bug.
+    spyOn(process, "memoryUsage").mockImplementation(throwingMemoryUsage);
+
+    expect(() => profiler.startRequest()).not.toThrow();
+  });
+
+  test("recordToolCompletion still records stats when memoryUsage throws", () => {
+    profiler.startRequest();
+    spyOn(process, "memoryUsage").mockImplementation(throwingMemoryUsage);
+
+    // The RSS sample fails, but the tool's timing stats are still recorded and
+    // no exception escapes to the caller (the agent loop).
+    expect(() =>
+      profiler.recordToolCompletion("bash", 123, false),
+    ).not.toThrow();
+
+    const summary = profiler.getSummary();
+    expect(summary.toolCount).toBe(1);
+    expect(summary.tools["bash"]).toEqual({
+      count: 1,
+      totalMs: 123,
+      maxMs: 123,
+      errors: 0,
+    });
+  });
+});
+
 describe("conversation-keyed profiler registry", () => {
   test("recording before a request window is a silent no-op", () => {
     expect(() =>
@@ -125,6 +174,28 @@ describe("conversation-keyed profiler registry", () => {
     ).not.toThrow();
   });
 
+  test("recordToolCompletion swallows profiler failures instead of throwing", () => {
+    startToolProfilingRequest("conv-throwing");
+    const spy = spyOn(process, "memoryUsage").mockImplementation(
+      throwingMemoryUsage,
+    );
+
+    try {
+      // A throwing RSS syscall must never propagate to the executor/agent loop.
+      expect(() =>
+        recordToolCompletion("conv-throwing", "bash", 42, false),
+      ).not.toThrow();
+    } finally {
+      spy.mockRestore();
+    }
+
+    // The turn's timing stats survived even though the RSS sample failed.
+    expect(() =>
+      emitToolProfilingSummary("conv-throwing", "req-throwing"),
+    ).not.toThrow();
+    disposeToolProfiler("conv-throwing");
+  });
+
   test("startToolProfilingRequest resets a conversation's prior window", () => {
     startToolProfilingRequest("conv-reset");
     recordToolCompletion("conv-reset", "bash", 100, false);

package/src/__tests__/tool-result-spool.test.ts

@@ -110,13 +110,14 @@ describe("spoolAndStubOversizedToolResults", () => {
     expect(messages).toBe(history);
   });
 
-  test("skips below-threshold, error, exempt, file_read, host_file_read, marked, and ax-tree results", () => {
+  test("skips below-threshold, error, exempt, file_read, host_file_read, web_fetch, marked, and ax-tree results", () => {
     const blocks = [
       makeToolResult(SHORT, "tu_short"),
       makeToolResult(LONG, "tu_err", true),
       makeToolResult(LONG, "tu_skill"),
       makeToolResult(LONG, "tu_read"),
       makeToolResult(LONG, "tu_host_read"),
+      makeToolResult(LONG, "tu_web_fetch"),
       makeToolResult(`${LONG}${TRUNCATION_MARKER}`, "tu_marked"),
       makeToolResult(`<ax-tree>${LONG}</ax-tree>`, "tu_ax"),
       { type: "text" as const, text: LONG },
@@ -126,9 +127,18 @@ describe("spoolAndStubOversizedToolResults", () => {
     const count = spoolAndStubOversizedToolResults(blocks, {
       conversationDir: convDir,
       toolNameById: (id) => {
-        if (id === "tu_skill") return "skill_load";
-        if (id === "tu_read") return "file_read";
-        if (id === "tu_host_read") return "host_file_read";
+        if (id === "tu_skill") {
+          return "skill_load";
+        }
+        if (id === "tu_read") {
+          return "file_read";
+        }
+        if (id === "tu_host_read") {
+          return "host_file_read";
+        }
+        if (id === "tu_web_fetch") {
+          return "web_fetch";
+        }
         return undefined;
       },
     });
@@ -312,6 +322,41 @@ describe("AgentLoop result-time spooling", () => {
     expect(readFileSync(filePath, "utf-8")).toBe(HUGE);
   });
 
+  test("web_fetch result is sent to the provider in full mid-turn (explicit self-sized read)", async () => {
+    const webFetchTool: ToolDefinition[] = [
+      {
+        name: "web_fetch",
+        description: "Fetch a web page",
+        input_schema: { type: "object", properties: {} },
+      },
+    ];
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("tu_wf", "web_fetch", {}),
+      textResponse("Done."),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: webFetchTool,
+      toolExecutor: async () => ({ content: LONG, isError: false }),
+      resolveConversationDir: () => convDir,
+    });
+
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // The second provider call carries the full requested window, not a stub;
+    // the post-turn pass (conversation-turn-finalize) owns truncating it.
+    const sentResult = calls[1].messages[2].content[0] as { content: string };
+    expect(sentResult.content).toBe(LONG);
+    expect(existsSync(join(convDir, TOOL_RESULT_DIR))).toBe(false);
+  });
+
   test("without resolveConversationDir the full content is sent and post-turn handles it", async () => {
     const { provider, calls } = createMockProvider([
       toolUseResponse("tu_1", "fetch_transcript", {}),

package/src/__tests__/tool-side-effects-slack-dm.test.ts

@@ -31,7 +31,6 @@ mock.module("../apps/app-store.js", () => ({
   getApp: mock(() => null),
   getAppDirPath: mock(() => ""),
   getAppsDir: mock(() => ""),
-  isMultifileApp: mock(() => false),
   resolveAppIdFromPath: mock(() => null),
   resolveAppIdByDirName: mock(() => null),
   resolveAppDir: mock(() => ({ dirName: "", dirPath: "" })),

package/src/__tests__/ui-channel-variants.test.ts

@@ -0,0 +1,108 @@
+import { describe, expect, test } from "bun:test";
+
+import type { ToolDefinition } from "../providers/types.js";
+import { ACTIVATION_MOMENT_PARAMS } from "../telemetry/activation-funnel.js";
+import {
+  injectActivationMomentParam,
+  projectUiToolsForChannel,
+} from "../tools/ui-surface/channel-variants.js";
+import {
+  uiDismissTool,
+  uiShowTool,
+  uiUpdateTool,
+} from "../tools/ui-surface/definitions.js";
+
+const defs: ToolDefinition[] = [
+  uiShowTool,
+  uiUpdateTool,
+  uiDismissTool,
+  {
+    name: "bash",
+    description: "run a command",
+    input_schema: { type: "object" },
+  },
+];
+
+function surfaceTypeEnum(def: ToolDefinition): string[] {
+  return (
+    def.input_schema as { properties: { surface_type: { enum: string[] } } }
+  ).properties.surface_type.enum;
+}
+
+describe("projectUiToolsForChannel", () => {
+  test("returns the same array for channels without a variant", () => {
+    for (const channel of [undefined, "macos", "telegram", "phone"]) {
+      expect(projectUiToolsForChannel(defs, channel)).toBe(defs);
+    }
+  });
+
+  test("slack gets a task_progress-only ui_show", () => {
+    const projected = projectUiToolsForChannel(defs, "slack");
+    const uiShow = projected.find((d) => d.name === "ui_show")!;
+
+    expect(surfaceTypeEnum(uiShow)).toEqual(["card"]);
+    expect(uiShow.description).toContain("task_progress");
+    expect(uiShow.description).toContain(
+      "the only surface this channel renders",
+    );
+    expect(uiShow.description).not.toContain("oauth_connect");
+    expect(uiShow.description).not.toContain("dynamic_page");
+    expect(uiShow.description!.length).toBeLessThan(
+      uiShowTool.description.length / 2,
+    );
+  });
+
+  test("slack projection keeps execute and leaves other tools untouched", () => {
+    const projected = projectUiToolsForChannel(defs, "slack");
+    const uiShow = projected.find((d) => d.name === "ui_show") as {
+      execute?: unknown;
+    };
+
+    expect(uiShow.execute).toBe(uiShowTool.execute);
+    expect(projected.find((d) => d.name === "ui_update")).toBe(uiUpdateTool);
+    expect(projected.find((d) => d.name === "ui_dismiss")).toBe(uiDismissTool);
+    expect(projected.find((d) => d.name === "bash")).toBe(defs[3]);
+  });
+
+  test("projection never mutates the shared definitions", () => {
+    const descriptionBefore = uiShowTool.description;
+    const schemaBefore = JSON.stringify(uiShowTool.input_schema);
+
+    projectUiToolsForChannel(defs, "slack");
+
+    expect(uiShowTool.description).toBe(descriptionBefore);
+    expect(JSON.stringify(uiShowTool.input_schema)).toBe(schemaBefore);
+  });
+});
+
+describe("injectActivationMomentParam", () => {
+  function properties(def: ToolDefinition): Record<string, unknown> {
+    return (def.input_schema as { properties: Record<string, unknown> })
+      .properties;
+  }
+
+  test("the standard ui_show schema does not carry activation_moment", () => {
+    expect(properties(uiShowTool).activation_moment).toBeUndefined();
+  });
+
+  test("adds the optional param to ui_show only, leaving required unchanged", () => {
+    const projected = injectActivationMomentParam(defs);
+    const uiShow = projected.find((d) => d.name === "ui_show")!;
+
+    const param = properties(uiShow).activation_moment as { enum: string[] };
+    expect(param).toBeDefined();
+    expect(param.enum).toEqual([...ACTIVATION_MOMENT_PARAMS]);
+    expect((uiShow.input_schema as { required: string[] }).required).toEqual([
+      "surface_type",
+      "data",
+    ]);
+    expect(projected.find((d) => d.name === "ui_update")).toBe(uiUpdateTool);
+    expect(projected.find((d) => d.name === "bash")).toBe(defs[3]);
+  });
+
+  test("injection never mutates the shared definition", () => {
+    const before = JSON.stringify(uiShowTool.input_schema);
+    injectActivationMomentParam(defs);
+    expect(JSON.stringify(uiShowTool.input_schema)).toBe(before);
+  });
+});