@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/telemetry/__tests__/phone-call-funnel.test.ts

@@ -0,0 +1,72 @@
+import { describe, expect, it } from "bun:test";
+
+import {
+  PHONE_CALL_STEPS,
+  phoneCallEndScreen,
+  phoneCallSilenceReason,
+  phoneCallStartScreen,
+} from "../phone-call-funnel.js";
+
+describe("phoneCallStartScreen", () => {
+  it("stamps the direction and the call mode", () => {
+    expect(phoneCallStartScreen("inbound", "normal")).toBe(
+      "started_inbound:normal",
+    );
+    expect(phoneCallStartScreen("outbound", "verification")).toBe(
+      "started_outbound:verification",
+    );
+  });
+
+  it("reads an unwritten mode as the ordinary call it is", () => {
+    // Only verification and invite calls write a mode, so a null there is a
+    // normal conversation rather than a dimension nobody filled in.
+    expect(phoneCallStartScreen("inbound", null)).toBe(
+      "started_inbound:normal",
+    );
+    expect(phoneCallStartScreen("inbound")).toBe("started_inbound:normal");
+  });
+
+  it("keeps the longest stamp inside the wire field's 64-char bound", () => {
+    const longest = phoneCallStartScreen("outbound", "verification");
+    expect(longest.length).toBeLessThanOrEqual(64);
+  });
+});
+
+describe("phoneCallSilenceReason", () => {
+  it("blames the connection when the call never reached the caller", () => {
+    // Nothing that happens on a live call can be held against one that never
+    // connected: nobody was there to speak.
+    expect(phoneCallSilenceReason({ connected: false })).toBe("no_connect");
+  });
+
+  it("reports a connected call that took no turn as silence on the line", () => {
+    expect(phoneCallSilenceReason({ connected: true })).toBe("no_turn");
+  });
+});
+
+describe("phoneCallEndScreen", () => {
+  it("stamps the terminal status on a call that took a turn", () => {
+    expect(phoneCallEndScreen("completed")).toBe("ended_completed");
+    expect(phoneCallEndScreen("failed", null)).toBe("ended_failed");
+  });
+
+  it("carries the silence classification when no turn was taken", () => {
+    expect(phoneCallEndScreen("failed", "no_connect")).toBe(
+      "ended_failed:silent_no_connect",
+    );
+    expect(phoneCallEndScreen("completed", "no_turn")).toBe(
+      "ended_completed:silent_no_turn",
+    );
+  });
+});
+
+describe("PHONE_CALL_STEPS", () => {
+  it("orders the pair the warehouse subtracts", () => {
+    // Duration is the gap between these two rows, so their order is part of
+    // the contract rather than a label.
+    expect(PHONE_CALL_STEPS.callStarted.stepIndex).toBe(0);
+    expect(PHONE_CALL_STEPS.callEnded.stepIndex).toBe(1);
+    expect(PHONE_CALL_STEPS.callStarted.stepName).toBe("phone_call_started");
+    expect(PHONE_CALL_STEPS.callEnded.stepName).toBe("phone_call_ended");
+  });
+});

package/src/telemetry/phone-call-funnel.ts

@@ -0,0 +1,144 @@
+/**
+ * Phone-call telemetry vocabulary: the single source of truth for the step
+ * names, funnel version, and dimension values the call store's emitter uses.
+ *
+ * The phone half of the live-voice funnel, and deliberately its mirror image:
+ * same onboarding substrate (`type: "onboarding"`), same open-string
+ * `step_name` / `funnel_version`, same reason for both. See
+ * `live-voice-funnel.ts` for the full argument; the short version is that the
+ * backend stores those two as open strings, so a new funnel needs no platform
+ * serializer and no wire-contract change.
+ *
+ * **Duration is not a field**, for the same reason it is not one on live
+ * voice: the onboarding event shape has no numeric slot. The two events below
+ * are keyed by the call session id in `session_id` and the warehouse subtracts
+ * their `recorded_at` stamps.
+ *
+ * **Turn count is not a field either.** Phone turns already carry the call
+ * session id in their `client` bag (`voiceTelemetry` at the bridge call site),
+ * so a turn count is a count of those rows.
+ *
+ * A call that starts and never ends is **censored, not infinite**: a daemon
+ * crash between the two writes emits no end event, so duration math must drop
+ * unmatched `started` rows rather than treat the call as still running.
+ */
+
+import type { CallMode, CallStatus } from "../calls/types.js";
+
+/** Funnel version stamped on every phone-call event. */
+export const PHONE_CALL_FUNNEL_VERSION = "phone_call_v1_2026_09";
+
+/**
+ * Phone-call funnel steps. `stepName` is the wire value; `stepIndex` is the
+ * ordinal position.
+ *
+ * `callStarted` fires when the call session row is created, before the
+ * provider has dialled: a call that never connects is exactly the one worth
+ * counting, so the start event precedes no-answer, busy, provider failure and
+ * the inbound credential preflight (an inbound session exists before that
+ * preflight runs and records against it).
+ *
+ * It does **not** cover an outbound attempt rejected before a session exists:
+ * `startCall`, `startVerificationCall` and `startInviteCall` all return on a
+ * failed `preflightVoiceIngress()` without creating a row, and this funnel is
+ * keyed by the session id it would have had. Counting those needs a key that
+ * does not depend on a session row, which is its own piece of work.
+ */
+export const PHONE_CALL_STEPS = {
+  callStarted: { stepName: "phone_call_started", stepIndex: 0 },
+  callEnded: { stepName: "phone_call_ended", stepIndex: 1 },
+} as const;
+
+export type PhoneCallStepName =
+  (typeof PHONE_CALL_STEPS)[keyof typeof PHONE_CALL_STEPS]["stepName"];
+
+/**
+ * Which way the call was placed. Inbound is someone dialling the assistant's
+ * number; outbound is the assistant dialling out on a task.
+ */
+export type PhoneCallDirection = "inbound" | "outbound";
+
+/**
+ * How a call ended, stamped as `outcome`.
+ *
+ * `failed` means the call died on an error: no answer, a provider or
+ * credential failure, a dropped media stream. A call the caller or the
+ * assistant hung up, including one cancelled before it connected, is
+ * `completed`: it ran its course and the close reason on `screen` says how.
+ */
+export type PhoneCallOutcome = "completed" | "failed";
+
+/**
+ * How far a call that produced NO caller turn actually got.
+ *
+ * The phone analogue of the live-voice silence taxonomy, and the closest thing
+ * call telemetry has to a "the phone path didn't work" rate. A bare
+ * started/ended pair cannot separate a call that never connected from one that
+ * connected to silence, and those have completely different fixes.
+ *
+ * - `no_connect` the call never reached `in_progress`: it failed, was
+ *                cancelled, or was never answered, so nobody could have
+ *                spoken.
+ * - `no_turn`    the call connected and then no caller utterance was ever
+ *                transcribed: a silent line, a failed STT leg, or a caller who
+ *                hung up on the disclosure.
+ */
+export type PhoneCallSilenceReason = "no_connect" | "no_turn";
+
+/**
+ * Classify a call that produced no caller turn. Call only when the call really
+ * produced none: the second branch asserts silence, so a call that did take a
+ * turn would be mislabelled by it.
+ */
+export function phoneCallSilenceReason(signals: {
+  connected: boolean;
+}): PhoneCallSilenceReason {
+  return signals.connected ? "no_turn" : "no_connect";
+}
+
+/**
+ * The mode an ordinary call carries. Only the verification and invite flows
+ * write a mode on the session row, so a null there is not a missing dimension
+ * but the ordinary conversation every other path creates.
+ */
+const PHONE_CALL_DEFAULT_MODE: CallMode = "normal";
+
+/**
+ * The `screen` dimension for a started call: which way it was placed and what
+ * kind of call it is, as `started_<direction>:<mode>`.
+ *
+ * The mode separates the three things a phone call can be (a normal
+ * conversation, a callee verification, an invite redemption) because they have
+ * completely different shapes and success criteria, and a funnel that mixed
+ * them would report the verification flow's brevity as a conversation problem.
+ */
+export function phoneCallStartScreen(
+  direction: PhoneCallDirection,
+  mode?: CallMode | null,
+): string {
+  return `started_${direction}:${mode ?? PHONE_CALL_DEFAULT_MODE}`;
+}
+
+/**
+ * The `screen` dimension for an ended call: the terminal status it reached,
+ * plus a detail half carrying the silence classification when the call took no
+ * caller turn at all.
+ *
+ * The status is the daemon's own `CallStatus` verbatim rather than a mapping,
+ * so a rename surfaces here as a compile error instead of a silently-empty
+ * dashboard facet.
+ *
+ * The silence value carries a `silent_` prefix to match the live-voice stamp,
+ * which shares its detail slot with failure codes and renders in an admin
+ * column labelled "failure code". Reading the two funnels side by side is
+ * worth more than a shorter string here.
+ */
+export function phoneCallEndScreen(
+  status: CallStatus,
+  silenceReason?: PhoneCallSilenceReason | null,
+): string {
+  if (silenceReason) {
+    return `ended_${status}:silent_${silenceReason}`;
+  }
+  return `ended_${status}`;
+}

package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts

@@ -0,0 +1,81 @@
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, test } from "bun:test";
+
+import { parseToolManifestFile } from "../../skills/tool-manifest.js";
+import { explicitTools } from "../tool-manifest.js";
+
+/**
+ * Tool input schemas must be plain objects at the root.
+ *
+ * Anthropic's Messages API rejects a tool whose `input_schema` carries
+ * `oneOf`, `anyOf`, or `allOf` at the top level ("input_schema does not
+ * support oneOf, allOf, or anyOf at the top level"). The rejection is a 400
+ * for the whole request, so one offending definition takes down every call
+ * that advertises it. Other hosts accept the same schema, which lets the
+ * mistake hide until a request routes to Anthropic directly. Either/or rules
+ * between fields belong in the tool description and the tool's own input
+ * validation instead.
+ *
+ * Covers the core manifest and every bundled skill's `TOOLS.json`.
+ * Combinators nested under `properties` are accepted by Anthropic and stay
+ * out of scope.
+ */
+
+const ROOT_COMBINATORS = ["oneOf", "anyOf", "allOf"] as const;
+
+const BUNDLED_SKILLS_DIR = resolve(
+  dirname(fileURLToPath(import.meta.url)),
+  "../../config/bundled-skills",
+);
+
+interface SchemaCase {
+  /** Tool name, prefixed with the skill directory for bundled skill tools. */
+  label: string;
+  schema: Record<string, unknown>;
+}
+
+function bundledSkillSchemas(): SchemaCase[] {
+  const cases: SchemaCase[] = [];
+  for (const relative of new Bun.Glob("*/TOOLS.json").scanSync({
+    cwd: BUNDLED_SKILLS_DIR,
+  })) {
+    const manifest = parseToolManifestFile(join(BUNDLED_SKILLS_DIR, relative));
+    for (const tool of manifest.tools) {
+      cases.push({
+        label: `${dirname(relative)}/${tool.name}`,
+        schema: tool.input_schema,
+      });
+    }
+  }
+  return cases;
+}
+
+function coreSchemas(): SchemaCase[] {
+  return explicitTools.map((tool) => {
+    if (!tool.name) {
+      throw new Error("core manifest entries carry explicit names");
+    }
+    return {
+      label: tool.name,
+      schema: tool.input_schema as Record<string, unknown>,
+    };
+  });
+}
+
+const CASES: SchemaCase[] = [...coreSchemas(), ...bundledSkillSchemas()];
+
+describe("tool input schema root", () => {
+  test("covers the core manifest and the bundled skills", () => {
+    expect(CASES.length).toBeGreaterThan(explicitTools.length);
+  });
+
+  for (const { label, schema } of CASES) {
+    test(`${label} keeps combinators out of the schema root`, () => {
+      for (const keyword of ROOT_COMBINATORS) {
+        expect(schema).not.toHaveProperty(keyword);
+      }
+      expect(schema.type).toBe("object");
+    });
+  }
+});

package/src/tools/ask-question/ask-question-tool.test.ts

@@ -15,6 +15,13 @@ import type { ToolContext } from "../types.js";
 // `mock.module` is hoisted by bun before any static import of the tool
 // runs, so the import below sees the stubbed prompter even though
 // `askQuestionTool` captures the symbol at module-eval time.
+const finishDesktopHelp = mock(async (_resume: boolean) => {});
+const prepareDesktopHelp = mock(
+  async (_context: ToolContext) => finishDesktopHelp,
+);
+let promptError: Error | undefined;
+mock.module("../../desktop/desktop-help.js", () => ({ prepareDesktopHelp }));
+
 const calls: QuestionPromptParams[] = [];
 let nextResult: QuestionPromptResult = {
   entries: [{ questionId: "q1", decision: "skipped" }],
@@ -28,6 +35,9 @@ mock.module("../../permissions/question-prompter.js", () => ({
   QuestionPrompter: class {
     async prompt(params: QuestionPromptParams): Promise<QuestionPromptOutcome> {
       calls.push(params);
+      if (promptError) {
+        throw promptError;
+      }
       // Mirror the real prompter: it mints the request id and assigns the
       // per-question `q1..qN` ids, then returns them alongside the resolution.
       return {
@@ -61,6 +71,9 @@ function makeContext(overrides: Partial<ToolContext> = {}): ToolContext {
 // `setNextResult()` before invoking `askQuestionTool.execute(...)`.
 beforeEach(() => {
   calls.length = 0;
+  prepareDesktopHelp.mockClear();
+  finishDesktopHelp.mockClear();
+  promptError = undefined;
   nextResult = {
     entries: [{ questionId: "q1", decision: "skipped" }],
     overall: "completed",
@@ -100,7 +113,7 @@ describe("askQuestionTool definition", () => {
     expect(def.description).toContain("up to 5");
     expect(def.description).toContain("Skip button");
 
-    const schema = def.input_schema as {
+    const schema = def.input_schema as unknown as {
       properties: Record<
         string,
         {
@@ -670,7 +683,7 @@ describe("AskQuestionTool batched input", () => {
 });
 
 describe("askQuestionTool definition (batched schema)", () => {
-  test("exposes `questions[]` shape, requires it, and drops the flat fields", () => {
+  test("exposes batched questions and a desktop help alternative", () => {
     const def = askQuestionTool;
     const schema = def.input_schema as unknown as {
       properties: Record<
@@ -708,9 +721,7 @@ describe("askQuestionTool definition (batched schema)", () => {
 
     expect(questions?.items?.required).toEqual(["question", "options"]);
 
-    // `questions` is the only top-level input now.
-    expect(schema.required).toEqual(["questions"]);
-    expect(Object.keys(schema.properties)).toEqual(["questions"]);
+    expect(schema.properties.desktopHelp?.type).toBe("object");
 
     // The legacy flat fields are gone.
     expect(schema.properties.question).toBeUndefined();
@@ -825,3 +836,154 @@ describe("answered-question record", () => {
     ).toBeUndefined();
   });
 });
+
+describe("virtual desktop help", () => {
+  test("reserves the desktop before waiting and resumes control after Done", async () => {
+    setNextResult(singleCompleted({ decision: "option", optionId: "done" }));
+    const result = await askQuestionTool.execute(
+      {
+        desktopHelp: {
+          message: "Please complete the CAPTCHA.",
+          doneLabel: "Done",
+          skipLabel: "Skip",
+        },
+      },
+      makeContext(),
+    );
+    expect(prepareDesktopHelp).toHaveBeenCalledTimes(1);
+    expect(finishDesktopHelp).toHaveBeenCalledWith(true);
+    expect(calls[0]?.questions[0]?.presentation).toBe("virtual_desktop");
+    expect(result.content).toContain("fresh browser snapshot");
+    expect(result.answeredQuestion?.responses[0]).toEqual({
+      questionId: "q1",
+      decision: "option",
+      optionId: "done",
+    });
+  });
+
+  test.each([
+    { decision: "skipped" as const },
+    { decision: "option" as const, optionId: "skip" },
+  ])("does not claim the obstacle was solved after Skip: %j", async (entry) => {
+    setNextResult(singleCompleted(entry));
+    const result = await askQuestionTool.execute(
+      {
+        desktopHelp: {
+          message: "Please complete the CAPTCHA.",
+          doneLabel: "Done",
+          skipLabel: "Skip",
+        },
+      },
+      makeContext(),
+    );
+    expect(finishDesktopHelp).toHaveBeenCalledWith(false);
+    expect(result.content).toContain("obstacle may still be present");
+    expect(result.isError).toBe(false);
+  });
+
+  test("does not wait for a user in a background turn", async () => {
+    await askQuestionTool.execute(
+      {
+        desktopHelp: {
+          message: "Please complete the CAPTCHA.",
+          doneLabel: "Done",
+          skipLabel: "Skip",
+        },
+      },
+      makeContext({ isInteractive: false }),
+    );
+    expect(prepareDesktopHelp).not.toHaveBeenCalled();
+    expect(calls).toHaveLength(0);
+  });
+
+  test("rejects mixing desktop help with a question batch", async () => {
+    const result = await askQuestionTool.execute(
+      {
+        ...validInput,
+        desktopHelp: {
+          message: "Please complete the CAPTCHA.",
+          doneLabel: "Done",
+          skipLabel: "Skip",
+        },
+      },
+      makeContext(),
+    );
+    expect(result.isError).toBe(true);
+    expect(calls).toHaveLength(0);
+  });
+});
+
+test("desktop help preserves model-localized fallback labels in history", async () => {
+  setNextResult(singleCompleted({ decision: "option", optionId: "done" }));
+  const result = await askQuestionTool.execute(
+    {
+      desktopHelp: {
+        message: "Completa la verificación.",
+        doneLabel: "Listo",
+        skipLabel: "Omitir",
+      },
+    },
+    makeContext(),
+  );
+  expect(calls[0]?.questions[0]?.options).toEqual([
+    { id: "done", label: "Listo" },
+    { id: "skip", label: "Omitir" },
+  ]);
+  expect(result.answeredQuestion?.questions[0]?.options).toEqual(
+    calls[0]?.questions[0]?.options,
+  );
+});
+
+test("desktop help does not park on a guardian channel without desktop controls", async () => {
+  const result = await askQuestionTool.execute(
+    {
+      desktopHelp: {
+        message: "Please sign in.",
+        doneLabel: "Done",
+        skipLabel: "Skip",
+      },
+    },
+    makeContext({
+      supportsDynamicUi: false,
+      supportsGuardianQuestionCards: true,
+    }),
+  );
+  expect(result.isError).toBe(true);
+  expect(result.content).toContain("continue in the Vellum app");
+  expect(prepareDesktopHelp).not.toHaveBeenCalled();
+  expect(calls).toHaveLength(0);
+});
+
+for (const overall of ["closed", "timed_out", "aborted"] as const) {
+  test(`desktop reservation ends when the question is ${overall}`, async () => {
+    setNextResult({ overall, entries: [] });
+    await askQuestionTool.execute(
+      {
+        desktopHelp: {
+          message: "Complete verification.",
+          doneLabel: "Done",
+          skipLabel: "Skip",
+        },
+      },
+      makeContext(),
+    );
+    expect(finishDesktopHelp).toHaveBeenCalledWith(false);
+  });
+}
+
+test("desktop reservation ends when presenting the question throws", async () => {
+  promptError = new Error("delivery failed");
+  await expect(
+    askQuestionTool.execute(
+      {
+        desktopHelp: {
+          message: "Complete verification.",
+          doneLabel: "Done",
+          skipLabel: "Skip",
+        },
+      },
+      makeContext(),
+    ),
+  ).rejects.toThrow("delivery failed");
+  expect(finishDesktopHelp).toHaveBeenCalledWith(false);
+});